[NO ISSUE][HYR][*DB] Exit JVM from independent thread to avoid deadlocks
Change-Id: I21b2090ea3ef85e95ae90de04b08b4a6d22ebe42
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1973
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java
index 76fa322..f5b4417 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java
@@ -28,6 +28,8 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.hyracks.util.ExitUtil;
+
public class LifeCycleComponentManager implements ILifeCycleComponentManager {
public static final class Config {
@@ -54,7 +56,7 @@
try {
LOGGER.log(Level.SEVERE, "Uncaught Exception from thread " + t.getName() + ". Calling shutdown hook", e);
} finally {
- Runtime.getRuntime().exit(99);// NOSONAR: It is really required
+ ExitUtil.exit(99);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml
index b28cc79..fce37dd 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml
@@ -43,6 +43,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</dependency>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
index d32e577..a243bf8 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
@@ -83,6 +83,7 @@
import org.apache.hyracks.ipc.api.IIPCI;
import org.apache.hyracks.ipc.impl.IPCSystem;
import org.apache.hyracks.ipc.impl.JavaSerializationBasedPayloadSerializerDeserializer;
+import org.apache.hyracks.util.ExitUtil;
import org.xml.sax.InputSource;
public class ClusterControllerService implements IControllerService {
@@ -138,6 +139,10 @@
private ShutdownRun shutdownCallback;
+ static {
+ ExitUtil.init();
+ }
+
public ClusterControllerService(final CCConfig config) throws Exception {
this(config, getApplication(config));
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
index 0b89f55..613efad 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
@@ -30,6 +30,7 @@
import org.apache.hyracks.control.common.work.IResultCallback;
import org.apache.hyracks.control.common.work.SynchronizableWork;
import org.apache.hyracks.ipc.exceptions.IPCException;
+import org.apache.hyracks.util.ExitUtil;
public class ClusterShutdownWork extends SynchronizableWork {
private static final Logger LOGGER = Logger.getLogger(ClusterShutdownWork.class.getName());
@@ -53,41 +54,36 @@
}
INodeManager nodeManager = ccs.getNodeManager();
Collection<String> nodeIds = nodeManager.getAllNodeIds();
- /**
+ /*
* set up our listener for the node ACKs
*/
final ShutdownRun shutdownStatus = new ShutdownRun(nodeIds);
// set up the CC to listen for it
ccs.setShutdownRun(shutdownStatus);
- /**
+ /*
* Shutdown all the nodes...
*/
nodeManager.apply(this::shutdownNode);
- ccs.getExecutor().execute(new Runnable() {
- @Override
- public void run() {
- try {
+ ccs.getExecutor().execute(() -> {
+ try {
+ /*
+ * wait for all our acks
+ */
+ LOGGER.info("Waiting for NCs to shutdown...");
+ boolean cleanShutdown = shutdownStatus.waitForCompletion();
+ if (!cleanShutdown) {
/*
- * wait for all our acks
+ * best effort - just exit, user will have to kill misbehaving NCs
*/
- LOGGER.info("Waiting for NCs to shutdown...");
- boolean cleanShutdown = shutdownStatus.waitForCompletion();
- if (!cleanShutdown) {
- /*
- * best effort - just exit, user will have to kill misbehaving NCs
- */
- LOGGER.severe("Clean shutdown of NCs timed out- giving up; unresponsive nodes: " +
- shutdownStatus.getRemainingNodes());
- }
- callback.setValue(cleanShutdown);
- ccs.stop(terminateNCService);
- LOGGER.info("JVM Exiting.. Bye!");
- Runtime rt = Runtime.getRuntime();
- rt.exit(cleanShutdown ? 0 : 1);
- } catch (Exception e) {
- callback.setException(e);
+ LOGGER.severe("Clean shutdown of NCs timed out- giving up; unresponsive nodes: " +
+ shutdownStatus.getRemainingNodes());
}
+ callback.setValue(cleanShutdown);
+ ccs.stop(terminateNCService);
+ ExitUtil.exit(cleanShutdown ? 0 : 1);
+ } catch (Exception e) {
+ callback.setException(e);
}
});
} catch (Exception e) {
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
index 548ed31..b5e96e3 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
@@ -54,6 +54,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
index 5601f9c..b52675c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
@@ -89,6 +89,7 @@
import org.apache.hyracks.ipc.impl.IPCSystem;
import org.apache.hyracks.net.protocols.muxdemux.FullFrameChannelInterfaceFactory;
import org.apache.hyracks.net.protocols.muxdemux.MuxDemuxPerformanceCounters;
+import org.apache.hyracks.util.ExitUtil;
import org.kohsuke.args4j.CmdLineException;
public class NodeControllerService implements IControllerService {
@@ -166,6 +167,10 @@
private final AtomicLong maxJobId = new AtomicLong(-1);
+ static {
+ ExitUtil.init();
+ }
+
public NodeControllerService(NCConfig config) throws Exception {
this(config, getApplication(config));
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
index c3aa5f4..e9cf3cb 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.control.common.base.IClusterController;
import org.apache.hyracks.control.nc.NodeControllerService;
+import org.apache.hyracks.util.ExitUtil;
public class ShutdownTask implements Runnable {
private static final Logger LOGGER = Logger.getLogger(ShutdownTask.class.getName());
@@ -46,15 +47,7 @@
// proceed with shutdown
}
- //run the shutdown in a new thread, so we don't block this last work task
- Thread t = new Thread("NC " + ncs.getId() + " Shutdown") {
- @Override
- public void run() {
- LOGGER.info("JVM Exiting.. Bye!");
- Runtime.getRuntime().exit(terminateNCService ? 99 : 0);
- }
- };
- t.start();
+ ExitUtil.exit(terminateNCService ? 99 : 0);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
new file mode 100644
index 0000000..b039227
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.util;
+
+import java.util.logging.Logger;
+
+@SuppressWarnings("squid:S1147")
+public class ExitUtil {
+
+ private static final Logger LOGGER = Logger.getLogger(ExitUtil.class.getName());
+
+ private static final ExitThread exitThread = new ExitThread();
+
+ private ExitUtil() {
+ }
+
+ public static void init() {
+ // no-op, the clinit does the work
+ }
+
+ public static void exit(int status) {
+ exitThread.setStatus(status);
+ exitThread.start();
+ }
+
+ private static class ExitThread extends Thread {
+ private int status;
+
+ ExitThread() {
+ super("JVM exit thread");
+ setDaemon(true);
+ }
+
+ @Override
+ public void run() {
+ try {
+ LOGGER.info("JVM exiting with status " + status + "; bye!");
+ } finally {
+ Runtime.getRuntime().exit(status);
+ }
+ }
+
+ public void setStatus(int status) {
+ this.status = status;
+ }
+ }
+}