Fix Hang At CC Shutdown When NC Has Exited
Also fix a few mispellings in log mesages:
/UNSABLE/UNUSABLE/
/Recieved/Received/
Change-Id: I6892cfb7b7253b1e0900abaa21705ee06da477d0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1130
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixClusterProperties.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixClusterProperties.java
index abfcf67..5acf9ae 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixClusterProperties.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixClusterProperties.java
@@ -205,7 +205,7 @@
for (ClusterPartition p : clusterPartitions.values()) {
if (!p.isActive()) {
state = ClusterState.UNUSABLE;
- LOGGER.info("Cluster is in UNSABLE state");
+ LOGGER.info("Cluster is in UNUSABLE state");
return;
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
index b05a3d6..6acab12 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
@@ -22,6 +22,7 @@
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
+import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.hyracks.control.cc.ClusterControllerService;
@@ -60,9 +61,8 @@
/**
* Shutdown all the nodes...
*/
- for (NodeControllerState ncs : nodeControllerStateMap.values()) {
- ncs.getNodeController().shutDown();
- }
+ nodeControllerStateMap.forEach(this::shutdownNode);
+
ccs.getExecutor().execute(new Runnable() {
@Override
public void run() {
@@ -72,7 +72,7 @@
*/
boolean cleanShutdown = shutdownStatus.waitForCompletion();
if (cleanShutdown) {
- callback.setValue(new Boolean(true));
+ callback.setValue(true);
ccs.stop();
LOGGER.info("JVM Exiting.. Bye!");
Runtime rt = Runtime.getRuntime();
@@ -85,10 +85,10 @@
LOGGER.severe("Clean shutdown of NCs timed out- CC bailing out!");
StringBuilder unresponsive = new StringBuilder();
for (String s : shutdownStatus.getRemainingNodes()) {
- unresponsive.append(s + " ");
+ unresponsive.append(s).append(' ');
}
LOGGER.severe("Unresponsive Nodes: " + unresponsive);
- callback.setValue(new Boolean(false));
+ callback.setValue(false);
ccs.stop();
LOGGER.info("JVM Exiting.. Bye!");
Runtime rt = Runtime.getRuntime();
@@ -103,4 +103,13 @@
callback.setException(e);
}
}
+
+ protected void shutdownNode(String key, NodeControllerState ncState) {
+ try {
+ ncState.getNodeController().shutdown();
+ } catch (Exception e) {
+ LOGGER.log(
+ Level.INFO, "Exception shutting down NC " + key + " (possibly dead?), continuing shutdown...", e);
+ }
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyShutdownWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyShutdownWork.java
index 9e290d6..1efbc6a 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyShutdownWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyShutdownWork.java
@@ -21,10 +21,7 @@
import java.util.logging.Logger;
-import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.control.cc.ClusterControllerService;
-import org.apache.hyracks.control.common.deployment.DeploymentRun;
-import org.apache.hyracks.control.common.deployment.DeploymentStatus;
import org.apache.hyracks.control.common.shutdown.ShutdownRun;
import org.apache.hyracks.control.common.work.SynchronizableWork;
@@ -44,7 +41,7 @@
public void doRun() {
/** triggered remotely by a NC to notify that the NC is shutting down */
ShutdownRun sRun = ccs.getShutdownRun();
- LOGGER.info("Recieved shutdown acknowledgement from NC ID:" + nodeId);
+ LOGGER.info("Received shutdown acknowledgement from NC ID:" + nodeId);
sRun.notifyShutdown(nodeId);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/INodeController.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/INodeController.java
index 7fe2f58..75c3127 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/INodeController.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/INodeController.java
@@ -51,7 +51,7 @@
public void dumpState(String stateDumpId) throws Exception;
- public void shutDown() throws Exception;
+ public void shutdown() throws Exception;
public void sendApplicationMessageToNC(byte[] data, DeploymentId deploymentId, String nodeId) throws Exception;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/NodeControllerRemoteProxy.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/NodeControllerRemoteProxy.java
index f7f73d7..841c889 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/NodeControllerRemoteProxy.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/NodeControllerRemoteProxy.java
@@ -90,7 +90,7 @@
}
@Override
- public void shutDown() throws Exception {
+ public void shutdown() throws Exception {
CCNCFunctions.ShutdownRequestFunction sdrf = new CCNCFunctions.ShutdownRequestFunction();
ipcHandle.send(-1, sdrf, null);
}