commit de892d7c075c6341ccfbb7a4d0090487d82dd5b6
author Michael Blow <mblow@apache.org> Tue Aug 22 00:38:08 2017 -0400
committer Michael Blow <mblow@apache.org> Mon Aug 21 21:39:29 2017 -0700
tree 24034ff67c03f6096b93d9caa41ec71f985a8805
parent 7def53f4e75abbb50012823ce6cb44fad911e117

[ASTERIXDB-2055][HYR][CLUS] Avoid double shutdown

Change-Id: If50651a4c46178f191966e09d365d2015df295bc
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1957
Reviewed-by: Michael Blow <mblow@apache.org>
Tested-by: Michael Blow <mblow@apache.org>

hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java

diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java
index 162d912..1cd45b2 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCShutdownHook.java
@@ -68,7 +68,7 @@
             shutdownHookThread = Thread.currentThread();
             watchDog.start();
             nodeControllerService.stop();
-        } catch (Throwable th) { // NOSONAR... This is fine since this is shutdwon hook
+        } catch (Throwable th) { // NOSONAR... This is fine since this is shutdown hook
             LOGGER.log(Level.WARNING, "Exception in executing shutdown hook", th);
         }
     }

hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java

diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
index 6b97b31..5601f9c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
@@ -29,6 +29,7 @@
 import java.lang.management.ThreadMXBean;
 import java.lang.reflect.Field;
 import java.net.InetSocketAddress;
+import java.util.Arrays;
 import java.util.Hashtable;
 import java.util.List;
 import java.util.Map;
@@ -153,7 +154,7 @@
 
     private final MemoryManager memoryManager;
 
-    private boolean shuttedDown = false;
+    private StackTraceElement[] shutdownCallStack;
 
     private IIOCounter ioCounter;
 
@@ -369,7 +370,8 @@
 
     @Override
     public synchronized void stop() throws Exception {
-        if (!shuttedDown) {
+        if (shutdownCallStack == null) {
+            shutdownCallStack = new Throwable().getStackTrace();
             LOGGER.log(Level.INFO, "Stopping NodeControllerService");
             application.preStop();
             executor.shutdownNow();
@@ -391,9 +393,9 @@
              */
             heartbeatTask.cancel();
             LOGGER.log(Level.INFO, "Stopped NodeControllerService");
-            shuttedDown = true;
         } else {
-            LOGGER.log(Level.SEVERE, "Double shutdown calls!!", new Exception("Double shutdown calls"));
+            LOGGER.log(Level.SEVERE, "Duplicate shutdown call; original: " + Arrays.toString(shutdownCallStack),
+                    new Exception("Duplicate shutdown call"));
         }
     }
 

hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java

diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
index cdbd4ad..c3aa5f4 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
@@ -36,6 +36,7 @@
     }
 
     @Override
+    @SuppressWarnings("squid:S1147") // Runtime.exit()
     public void run() {
         IClusterController ccs = ncs.getClusterController();
         try {
@@ -45,19 +46,12 @@
             // proceed with shutdown
         }
 
-        LOGGER.info("JVM Exiting.. Bye!");
         //run the shutdown in a new thread, so we don't block this last work task
         Thread t = new Thread("NC " + ncs.getId() + " Shutdown") {
             @Override
             public void run() {
-                try {
-                    ncs.stop();
-                } catch (Exception e) {
-                    LOGGER.log(Level.SEVERE, "Exception stopping node controller service", e);
-                } finally {
-                    Runtime rt = Runtime.getRuntime();
-                    rt.exit(terminateNCService ? 99 : 0);
-                }
+                LOGGER.info("JVM Exiting.. Bye!");
+                Runtime.getRuntime().exit(terminateNCService ? 99 : 0);
             }
         };
         t.start();