commit 6b099b0fb0fbea84d237514b6035d84334921197
author Hussain Towaileb <Hussain.Towaileb@Couchbase.com> Fri Dec 17 17:25:11 2021 +0300
committer Hussain Towaileb <hussainht@gmail.com> Fri Dec 17 15:28:30 2021 +0000
tree dbd5e291e72f4e0fde1a7f8029e777ff4f93094c
parent 311d6f7dd3a4e1273dbd615cc7b19f23232f6d84

[NO ISSUE][EXT][AZDL]: Skip instead of failing when json file not found

Details:
- When a JSON file is not found while reading external azure
  datalake dataset, skip the file and continue reading, do
  not fail.

Change-Id: Ic9b04e418483cc245379e35c9a20f1a4c4389e87
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14583
Tested-by: Hussain Towaileb <hussainht@gmail.com>
Integration-Tests: Hussain Towaileb <hussainht@gmail.com>
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>

asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java

diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java
index f0c185e..b7d142f 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java
@@ -35,10 +35,10 @@
 import org.apache.hyracks.util.LogRedactionUtil;
 
 import com.azure.storage.blob.models.BlobErrorCode;
+import com.azure.storage.blob.models.BlobStorageException;
 import com.azure.storage.file.datalake.DataLakeFileClient;
 import com.azure.storage.file.datalake.DataLakeFileSystemClient;
 import com.azure.storage.file.datalake.DataLakeServiceClient;
-import com.azure.storage.file.datalake.models.DataLakeStorageException;
 
 public class AzureDataLakeInputStream extends AbstractExternalInputStream {
 
@@ -67,8 +67,7 @@
             if (lowerCaseFileName.endsWith(".gz") || lowerCaseFileName.endsWith(".gzip")) {
                 in = new GZIPInputStream(in, ExternalDataConstants.DEFAULT_BUFFER_SIZE);
             }
-        } catch (DataLakeStorageException ex) {
-            // TODO(htowaileb): need to find the right error for Azure Data Lake
+        } catch (BlobStorageException ex) {
             if (ex.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND)) {
                 LOGGER.debug(() -> "Key " + LogRedactionUtil.userData(filePaths.get(nextFileIndex)) + " was not "
                         + "found in container " + container);