| commit | d399e65f7697d9be783e0ad36c2e9f3f67d6e640 | [log] [tgz] |
|---|---|---|
| author | Wail Alkowaileet <wael.y.k@gmail.com> | Wed Aug 23 12:59:33 2023 -0700 |
| committer | Wail Alkowaileet <wael.y.k@gmail.com> | Wed Aug 23 23:08:33 2023 +0000 |
| tree | fd44e725284555e83a3ea1fedb29482dc8b20ae3 | |
| parent | 505a96bce384239498d263b85124dd6de2fcfca3 [diff] |
[ASTERIXDB-3250][STO] Ensure writing indexes' empty files to S3 - user model changes: no - storage format changes: no - interface changes: yes Details: Empty indexes files are not written to S3. This patch fixes this issue. Interface changes: - Add isEmpty() in ICloudBufferedWriter Change-Id: I740c0d9e82bd0f933a37c4f68c3d95dc715cb46b Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17737 Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu> Reviewed-by: Murtadha Al Hubail <mhubail@apache.org> Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java index 3a7504a..7ba95c5 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java
@@ -100,7 +100,13 @@ public void finish() throws HyracksDataException { open(); try { - if (writeBuffer.position() > 0) { + if (writeBuffer.position() > 0 || bufferedWriter.isEmpty()) { + /* + * upload if: + * (1) the writeBuffer is not empty + * OR + * (2) nothing was written to the file at all to ensure writing empty file + */ uploadAndWait(); } bufferedWriter.finish();
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudBufferedWriter.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudBufferedWriter.java index 4f08111..16e79f9 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudBufferedWriter.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudBufferedWriter.java
@@ -34,6 +34,13 @@ int upload(InputStream stream, int length); /** + * Checks whether the writer has not written anything + * + * @return true if nothing was written, false otherwise + */ + boolean isEmpty(); + + /** * Finishes the upload * * @throws HyracksDataException HyracksDataException
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java index 71246f1..2efd214 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
@@ -73,10 +73,17 @@ } @Override + public boolean isEmpty() { + return uploadId == null; + } + + @Override public void finish() throws HyracksDataException { if (uploadId == null) { - return; + throw new IllegalStateException("Cannot finish without writing any bytes"); } + + // A non-empty files, proceed with completing the multipart upload CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(partQueue).build(); CompleteMultipartUploadRequest completeMultipartUploadRequest = CompleteMultipartUploadRequest.builder() .bucket(bucket).key(path).uploadId(uploadId).multipartUpload(completedMultipartUpload).build();