commit 89a2905adf6673241389269a0c2e6d761541c826
author Murtadha Hubail <mhubail@apache.org> Tue Jun 11 18:04:32 2024 -0700
committer Murtadha Hubail <mhubail@apache.org> Wed Jun 12 20:42:44 2024 +0000
tree 46a9b17232706f0e6b48e603a0ecd1f94db3858b
parent 3580069c212d8cac8483fe2ae40d5e5cf04dc6e6

[NO ISSUE][TEST] Catch all failures on cc/nc startup

- user model changes: no
- storage format changes: no
- interface changes: no

Details:

- Catch all nc/cc startup failures in AsterixHyracksIntegrationUtil.

Change-Id: Ifcbe1a0dff7f3dbb7e6b258d43bcbd5a2deb7020
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18363
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Murtadha Hubail <mhubail@apache.org>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Reviewed-by: Peeyush Gupta <peeyush.gupta@couchbase.com>

asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java

diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
index 506e82a..e312b01 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
@@ -31,6 +31,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.BiPredicate;
 import java.util.stream.Stream;
 
@@ -61,7 +62,6 @@
 import org.apache.hyracks.ipc.impl.HyracksConnection;
 import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtreeLocalResource;
 import org.apache.hyracks.test.support.TestUtils;
-import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
@@ -110,8 +110,8 @@
         try {
             integrationUtil.run(Boolean.getBoolean("cleanup.start"), Boolean.getBoolean("cleanup.shutdown"),
                     getConfPath());
-        } catch (Exception e) {
-            LOGGER.fatal("Unexpected exception", e);
+        } catch (Throwable t) {
+            LOGGER.fatal("Unexpected exception", t);
             System.exit(1);
         }
     }
@@ -155,19 +155,27 @@
         }
 
         opts.forEach(opt -> configManager.set(opt.getLeft(), opt.getRight()));
-        cc.start();
+        try {
+            cc.start();
+        } catch (Throwable t) {
+            LOGGER.error("failed to start cc", t);
+            throw t;
+        }
 
         // Starts ncs.
         nodeNames = ccConfig.getConfigManager().getNodeNames();
         List<Thread> startupThreads = new ArrayList<>();
+        AtomicBoolean ncFailedToStart = new AtomicBoolean(false);
         for (NodeControllerService nc : nodeControllers) {
             Thread ncStartThread = new Thread("IntegrationUtil-" + nc.getId()) {
                 @Override
                 public void run() {
                     try {
                         nc.start();
-                    } catch (Exception e) {
-                        LOGGER.log(Level.ERROR, e.getMessage(), e);
+                        LOGGER.info("started node {}", nc.getId());
+                    } catch (Throwable t) {
+                        LOGGER.error("failed to start node {}", nc.getId(), t);
+                        ncFailedToStart.set(true);
                     }
                 }
             };
@@ -178,11 +186,14 @@
         for (Thread thread : startupThreads) {
             thread.join();
         }
+        if (ncFailedToStart.get()) {
+            throw new Exception("some node failed to start");
+        }
         // Wait until cluster becomes active
         ((ICcApplicationContext) cc.getApplicationContext()).getClusterStateManager().waitForState(ClusterState.ACTIVE);
         hcc = new HyracksConnection(cc.getConfig().getClientListenAddress(), cc.getConfig().getClientListenPort(),
                 cc.getNetworkSecurityManager().getSocketChannelFactory());
-        this.ncs = nodeControllers.toArray(new NodeControllerService[nodeControllers.size()]);
+        this.ncs = nodeControllers.toArray(new NodeControllerService[0]);
     }
 
     @NotNull
@@ -297,8 +308,8 @@
                     public void run() {
                         try {
                             nodeControllerService.stop();
-                        } catch (Exception e) {
-                            e.printStackTrace();
+                        } catch (Throwable t) {
+                            LOGGER.error("failed to stop node {}", nodeControllerService.getId(), t);
                         }
                     }
                 };
@@ -367,8 +378,8 @@
             public void run() {
                 try {
                     deinit(cleanupOnShutdown);
-                } catch (Exception e) {
-                    LOGGER.log(Level.WARN, "Unexpected exception on shutdown", e);
+                } catch (Throwable t) {
+                    LOGGER.warn("Unexpected exception on shutdown", t);
                 }
             }
         });
@@ -385,8 +396,8 @@
             public void run() {
                 try {
                     deinit(cleanupOnShutdown);
-                } catch (Exception e) {
-                    LOGGER.log(Level.WARN, "Unexpected exception on shutdown", e);
+                } catch (Throwable t) {
+                    LOGGER.warn("Unexpected exception on shutdown", t);
                 }
             }
         });