[NO ISSUE] Ensure shutdown watchdog is started
Relying on the shutdown watchdog to be triggered by a shutdown hook
is problematic, as it is undefined when the JVM will run the shutdown
hook.
Change-Id: Ia3ed2c88ec4c93a298f89f12f29448370cae9136
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2860
Reviewed-by: Michael Blow <mblow@apache.org>
Tested-by: Michael Blow <mblow@apache.org>
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
index 6aa708d..9604c30 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
@@ -18,6 +18,9 @@
*/
package org.apache.hyracks.util;
+import java.lang.reflect.Field;
+import java.util.IdentityHashMap;
+import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.mutable.MutableLong;
@@ -58,7 +61,7 @@
private static final MutableLong shutdownHaltDelay = new MutableLong(10 * 60 * 1000L); // 10 minutes default
static {
- Runtime.getRuntime().addShutdownHook(new Thread(watchdogThread::start));
+ watchdogThread.start();
}
private ExitUtil() {
@@ -94,6 +97,8 @@
private static class ShutdownWatchdog extends Thread {
+ private final Semaphore startSemaphore = new Semaphore(0);
+
private ShutdownWatchdog() {
super("ShutdownWatchdog");
setDaemon(true);
@@ -101,11 +106,14 @@
@Override
public void run() {
+ startSemaphore.acquireUninterruptibly();
+ LOGGER.info("starting shutdown watchdog- system will halt if shutdown is not completed within {} seconds",
+ TimeUnit.MILLISECONDS.toSeconds(shutdownHaltDelay.getValue()));
try {
- exitThread.join(shutdownHaltDelay.getValue()); // 10 min
+ exitThread.join(shutdownHaltDelay.getValue());
if (exitThread.isAlive()) {
try {
- LOGGER.warn("Watchdog is angry. Killing shutdown hook");
+ LOGGER.fatal("shutdown did not complete within configured delay; halting");
} finally {
ExitUtil.halt(EC_HALT_SHUTDOWN_TIMED_OUT);
}
@@ -114,6 +122,10 @@
ExitUtil.halt(EC_HALT_WATCHDOG_FAILED);
}
}
+
+ public void beginWatch() {
+ startSemaphore.release();
+ }
}
private static class ExitThread extends Thread {
@@ -127,8 +139,10 @@
@Override
public void run() {
+ watchdogThread.beginWatch();
try {
LOGGER.warn("JVM exiting with status " + status + "; bye!", callstack);
+ logShutdownHooks();
} finally {
Runtime.getRuntime().exit(status);
}
@@ -138,5 +152,17 @@
this.status = status;
this.callstack = callstack;
}
+
+ private static void logShutdownHooks() {
+ try {
+ Class clazz = Class.forName("java.lang.ApplicationShutdownHooks");
+ Field hooksField = clazz.getDeclaredField("hooks");
+ hooksField.setAccessible(true);
+ IdentityHashMap hooks = (IdentityHashMap) hooksField.get(null);
+ LOGGER.info("the following ({}) shutdown hooks have been registered: {}", hooks::size, hooks::toString);
+ } catch (Exception e) {
+ LOGGER.warn("ignoring exception trying to determine number of shutdown hooks", e);
+ }
+ }
}
}