[NO ISSUE] Minor active refactoring
- Remove unused ActiveRuntimeManager
- Rename StatsRequestMessage -> ActiveStatsRequestMessage
- Add ActiveManager API to return all active runtimes
- Interrupt running HTTP requests after 5s upon shutdown
- Log thread dump when HTTP requests do not complete after interruption
Change-Id: I79249f7cd42496d6679eb9b0acbe8cda1892f9d3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2021
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>
Tested-by: Michael Blow <mblow@apache.org>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
index b5388c2..407f9cd 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.control.cc.work;
-import java.lang.management.ManagementFactory;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -29,7 +28,7 @@
import org.apache.hyracks.control.cc.ClusterControllerService;
import org.apache.hyracks.control.cc.NodeControllerState;
import org.apache.hyracks.control.cc.cluster.INodeManager;
-import org.apache.hyracks.control.common.utils.ThreadDumpHelper;
+import org.apache.hyracks.util.ThreadDumpUtil;
import org.apache.hyracks.control.common.work.AbstractWork;
import org.apache.hyracks.control.common.work.IResultCallback;
@@ -55,7 +54,7 @@
if (nodeId == null) {
// null nodeId means the request is for the cluster controller
try {
- callback.setValue(ThreadDumpHelper.takeDumpJSONString(ManagementFactory.getThreadMXBean()));
+ callback.setValue(ThreadDumpUtil.takeDumpJSONString());
} catch (Exception e) {
LOGGER.log(Level.WARNING, "Exception taking CC thread dump", e);
callback.setException(e);
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
index ed5598b..a426d47 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
@@ -210,7 +210,7 @@
osMXBean = ManagementFactory.getOperatingSystemMXBean();
getNodeControllerInfosAcceptor = new MutableObject<>();
memoryManager = new MemoryManager((long) (memoryMXBean.getHeapMemoryUsage().getMax() * MEMORY_FUDGE_FACTOR));
- ioCounter = new IOCounterFactory().getIOCounter();
+ ioCounter = IOCounterFactory.INSTANCE.getIOCounter();
}
public IOManager getIoManager() {
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java
index 1b7cf8f..2301ae6 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java
@@ -21,6 +21,11 @@
public class IOCounterFactory {
+ public static final IOCounterFactory INSTANCE = new IOCounterFactory();
+
+ private IOCounterFactory() {
+ }
+
/**
* Get the IOCounter for the specific underlying OS
*
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ThreadDumpTask.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ThreadDumpTask.java
index abde87f..e23aaaa 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ThreadDumpTask.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ThreadDumpTask.java
@@ -21,7 +21,7 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.hyracks.control.common.utils.ThreadDumpHelper;
+import org.apache.hyracks.util.ThreadDumpUtil;
import org.apache.hyracks.control.nc.NodeControllerService;
public class ThreadDumpTask implements Runnable {
@@ -38,7 +38,7 @@
public void run() {
String result;
try {
- result = ThreadDumpHelper.takeDumpJSONString(ncs.getThreadMXBean());
+ result = ThreadDumpUtil.takeDumpJSONString();
} catch (Exception e) {
LOGGER.log(Level.WARNING, "Exception taking thread dump", e);
result = null;
diff --git a/hyracks-fullstack/hyracks/hyracks-http/pom.xml b/hyracks-fullstack/hyracks/hyracks-http/pom.xml
index ed0e8c8..09bf513 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-http/pom.xml
@@ -66,5 +66,10 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
index 44d4dfe..645bc01 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
@@ -31,6 +31,7 @@
import java.util.logging.Logger;
import org.apache.hyracks.http.api.IServlet;
+import org.apache.hyracks.util.ThreadDumpUtil;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.buffer.PooledByteBufAllocator;
@@ -218,11 +219,22 @@
}
protected void doStop() throws InterruptedException {
+ // stop taking new requests
executor.shutdown();
try {
- executor.awaitTermination(1, TimeUnit.MINUTES);
+ // wait 5s before interrupting existing requests
+ executor.awaitTermination(5, TimeUnit.SECONDS);
+ // interrupt
+ executor.shutdownNow();
+ // wait 30s for interrupted requests to unwind
+ executor.awaitTermination(30, TimeUnit.SECONDS);
if (!executor.isTerminated()) {
- LOGGER.log(Level.SEVERE, "Failed to shutdown http server executor");
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.log(Level.SEVERE, "Failed to shutdown http server executor; thread dump: " +
+ ThreadDumpUtil.takeDumpString());
+ } else {
+ LOGGER.log(Level.SEVERE, "Failed to shutdown http server executor");
+ }
}
} catch (Exception e) {
LOGGER.log(Level.SEVERE, "Error while shutting down http server executor", e);
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/ThreadDumpHelper.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadDumpUtil.java
similarity index 84%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/ThreadDumpHelper.java
rename to hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadDumpUtil.java
index 62c6586..ec1a0b2 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/ThreadDumpHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadDumpUtil.java
@@ -16,9 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.control.common.utils;
+package org.apache.hyracks.util;
import java.io.IOException;
+import java.lang.management.ManagementFactory;
import java.lang.management.ThreadInfo;
import java.lang.management.ThreadMXBean;
import java.util.ArrayList;
@@ -26,24 +27,26 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.Stream;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.node.ObjectNode;
-public class ThreadDumpHelper {
+public class ThreadDumpUtil {
private static final ObjectMapper om = new ObjectMapper();
+ private static final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
- private ThreadDumpHelper() {
+ private ThreadDumpUtil() {
om.enable(SerializationFeature.INDENT_OUTPUT);
}
- public static String takeDumpJSONString(ThreadMXBean threadMXBean) throws IOException {
- ObjectNode json = takeDumpJSON(threadMXBean);
+ public static String takeDumpJSONString() throws IOException {
+ ObjectNode json = takeDumpJSON();
return om.writerWithDefaultPrettyPrinter().writeValueAsString(json);
}
- public static ObjectNode takeDumpJSON(ThreadMXBean threadMXBean) {
+ public static ObjectNode takeDumpJSON() {
ThreadInfo[] threadInfos = threadMXBean.dumpAllThreads(true, true);
List<Map<String, Object>> threads = new ArrayList<>();
@@ -92,4 +95,10 @@
}
return json;
}
+
+ public static String takeDumpString() {
+ StringBuilder buf = new StringBuilder(2048);
+ Stream.of(threadMXBean.dumpAllThreads(true, true)).forEach(buf::append);
+ return buf.toString();
+ }
}