commit 5f2c24f4db895fe6d91f90336ab2502783b2a8f7
author Hussain Towaileb <Hussain.Towaileb@couchbase.com> Mon Mar 06 21:41:16 2023 +0300
committer Hussain Towaileb <hussainht@gmail.com> Tue Mar 07 04:18:02 2023 +0000
tree 0b2ddb8e8aba2884c62b79e8783422768be28b2a
parent 1edc94dd592216f37eeabbf820a0bdd2cf02b15b

[NO ISSUE][EXT]: Properly handle failures and retries for GCS

Change-Id: I0bd42c391db603040d0470e976fa561649024992
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17409
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>

asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java

diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
index 652fa3e..5da4583 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
@@ -47,7 +47,7 @@
 
     private final Storage client;
     private final String container;
-    private static final int MAX_RETRIES = 5; // We will retry 5 times in case of retryable errors
+    private static final int MAX_ATTEMPTS = 5; // We try a total of 5 times in case of retryable errors
 
     public GCSInputStream(Map<String, String> configuration, List<String> filePaths) throws HyracksDataException {
         super(configuration, filePaths);
@@ -78,10 +78,10 @@
      * @return true
      */
     private boolean doGetInputStream(String fileName) throws RuntimeDataException {
-        int retries = 0;
+        int attempt = 0;
         BlobId blobId = BlobId.of(container, fileName);
 
-        while (retries < MAX_RETRIES) {
+        while (attempt < MAX_ATTEMPTS) {
             try {
                 Blob blob = client.get(blobId);
                 if (blob == null) {
@@ -93,14 +93,14 @@
                 in = new ByteArrayInputStream(blob.getContent());
                 break;
             } catch (BaseServiceException ex) {
-                if (!shouldRetry(retries++) && ex.isRetryable()) {
+                if (!ex.isRetryable() || !shouldRetry(++attempt)) {
                     throw new RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
                 }
-                LOGGER.debug(() -> "Retryable error: " + LogRedactionUtil.userData(ex.getMessage()));
+                LOGGER.debug(() -> "Retryable error: " + getMessageOrToString(ex));
 
-                // Backoff for 1 sec for the first 2 retries, and 2 seconds from there onward
+                // Backoff for 1 sec for the first 3 attempts, and 2 seconds from there onward
                 try {
-                    Thread.sleep(TimeUnit.SECONDS.toMillis(retries < 3 ? 1 : 2));
+                    Thread.sleep(TimeUnit.SECONDS.toMillis(attempt < 3 ? 1 : 2));
                 } catch (InterruptedException e) {
                     Thread.currentThread().interrupt();
                 }
@@ -111,8 +111,8 @@
         return true;
     }
 
-    private boolean shouldRetry(int currentRetry) {
-        return currentRetry < MAX_RETRIES;
+    private boolean shouldRetry(int nextAttempt) {
+        return nextAttempt < MAX_ATTEMPTS;
     }
 
     @Override