commit 619b8c53b95bb4a67a6a527fbf41a63b0522173e
author Hussain Towaileb <HussainHT@Gmail.com> Thu Jun 11 17:49:45 2020 +0300
committer Hussain Towaileb <hussainht@gmail.com> Fri Jun 12 05:42:25 2020 +0000
tree d3f8ee8d769f5f86c9f7f530c09803e3b27ad2a9
parent 8ef98896877ee33d0480a916ed070487634f29d7

[ASTERIXDB-2743][EXT] Skip not found files when querying S3 external dataset

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- When querying an S3 external dataset, if some files are not found
  due to possiblity getting deleted while querying, skip the filies
  and continue to the next ones instead of failing.

Change-Id: I6d3e691a1714228f1844faf05095cf8aa439ea1d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6723
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>

asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java

diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
index bcbf540..9e10e6a 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
@@ -33,13 +33,19 @@
 import org.apache.asterix.external.util.ExternalDataUtils;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.util.CleanupUtils;
+import org.apache.hyracks.util.LogRedactionUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 import software.amazon.awssdk.core.exception.SdkException;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
 
 public class AwsS3InputStream extends AbstractMultipleInputStream {
 
+    private static final Logger LOGGER = LogManager.getLogger();
+
     // Configuration
     private final Map<String, String> configuration;
 
@@ -83,6 +89,11 @@
         // the header, then the S3 stream gets closed in the close method
         try {
             in = s3Client.getObject(getObjectRequest);
+        } catch (NoSuchKeyException ex) {
+            LOGGER.debug(() -> "Key " + LogRedactionUtil.userData(getObjectRequest.key()) + " was not found in bucket "
+                    + getObjectRequest.bucket());
+            nextFileIndex++;
+            return advance();
         } catch (SdkException ex) {
             throw new RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
         }