commit d3a31a6bf9e1f1ad2dde81b877079132b85a38a5
author Wail Alkowaileet <wael.y.k@gmail.com> Tue Sep 05 15:01:33 2023 -0700
committer Wail Alkowaileet <wael.y.k@gmail.com> Wed Sep 06 01:31:44 2023 +0000
tree 5fa0f88e4d1c5f8a55f94d3c009d810283be8f30
parent e6d1859c5ab1798c11757e07b58aa467a002d2e1

[ASTERIXDB-3257][OTH} Fix S3 list in Cloud I/O Manager

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
Fix list to return the entire objects stored in the cloud.

Change-Id: I500aab9a51376b63128c78feece334116b5215ce
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17762
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>

asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java

diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
index 2faba79..8901dec 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
@@ -23,6 +23,7 @@
 import java.net.URISyntaxException;
 import java.net.URLDecoder;
 import java.nio.charset.Charset;
+import java.util.ArrayList;
 import java.util.List;
 
 import software.amazon.awssdk.services.s3.S3Client;
@@ -41,6 +42,7 @@
         ListObjectsV2Response listObjectsResponse;
         ListObjectsV2Request.Builder listObjectsBuilder = ListObjectsV2Request.builder().bucket(bucket);
         listObjectsBuilder.prefix(toCloudPrefix(path));
+        List<S3Object> files = new ArrayList<>();
         while (true) {
             // List the objects from the start, or from the last marker in case of truncated result
             if (newMarker == null) {
@@ -49,6 +51,8 @@
                 listObjectsResponse = s3Client.listObjectsV2(listObjectsBuilder.continuationToken(newMarker).build());
             }
 
+            files.addAll(listObjectsResponse.contents());
+
             // Mark the flag as done if done, otherwise, get the marker of the previous response for the next request
             if (Boolean.FALSE.equals(listObjectsResponse.isTruncated())) {
                 break;
@@ -56,7 +60,7 @@
                 newMarker = listObjectsResponse.nextContinuationToken();
             }
         }
-        return listObjectsResponse.contents();
+        return files;
     }
 
     public static String encodeURI(String path) {

asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java

diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
index 4394361..aee916c 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
@@ -66,6 +66,8 @@
         // First get the set of local files
         Set<FileReference> localFiles = ioManager.list(partitionPath);
         Iterator<FileReference> localFilesIter = localFiles.iterator();
+        LOGGER.info("Cleaning partition {}. Total number of unchecked cloud files {}", partitionPath.getRelativePath(),
+                cloudFiles.size());
 
         // Reconcile local files and cloud files
         while (localFilesIter.hasNext()) {
@@ -131,7 +133,8 @@
 
     private static void logDeleteFile(FileReference fileReference) {
         if (LOGGER.isDebugEnabled()) {
-            LOGGER.debug("Deleting {} from the local cache as it doesn't exists in the cloud", fileReference);
+            LOGGER.debug("Deleting {} from the local cache as {} doesn't exist in the cloud", fileReference,
+                    fileReference.getRelativePath());
         }
     }
 }