[NO ISSUE][HYR][*DB][HTTP] Stop accepting web connections at shutdown
Avoid returning 503s while shutting down, by stopping web
listeners at start of shutdown process
Ext-ref: MB-67428
Change-Id: I7cacccec8c44d6bac55d8e46129eb7ad1e6c251c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20012
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
index 6928b64..4e35ade 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
@@ -300,6 +300,7 @@
public void stop() throws Exception {
LOGGER.info("Stopping Asterix cluster controller");
super.stop();
+ webManager.closeChannels(); // stop accepting new requests
appCtx.getClusterStateManager().setState(SHUTTING_DOWN);
((ActiveNotificationHandler) appCtx.getActiveNotificationHandler()).stop();
AsterixStateProxy.unregisterRemoteObject();
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index 0da4e0d..0e6a04b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -299,6 +299,7 @@
@Override
public void preStop() throws Exception {
+ webManager.closeChannels();
runtimeContext.preStop();
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
index a5c8073..6c0982a 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
@@ -248,6 +248,8 @@
Map<String, Map<IOption, Object>> getActiveNcConfiguration();
+ Set<String> getNodesPendingRemoval();
+
boolean isPendingRemoval(String nodeId);
/**
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
index 7c9fd5b..668decf 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
@@ -474,6 +474,7 @@
return ncConfigMap;
}
+ @Override
public synchronized Set<String> getNodesPendingRemoval() {
return new HashSet<>(pendingRemoval);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
index 271de53..1198945 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
@@ -433,12 +433,12 @@
return addresses.iterator().next();
}
- private void closeChannels() throws InterruptedException {
+ void closeChannels() throws InterruptedException {
synchronized (lock) {
for (Channel channel : channels) {
channel.closeFuture().removeListener(channelCloseListener);
- channel.close();
- channel.closeFuture().sync();
+ channel.close().sync();
+ LOGGER.info("channel {} closed", channel);
}
channels.clear();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/WebManager.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/WebManager.java
index 3b17c06..9a0aa75 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/WebManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/WebManager.java
@@ -113,4 +113,27 @@
public void add(HttpServer server) {
servers.add(server);
}
+
+ /**
+ * Closes all channels associated with the servers in this WebManager.
+ * This prevents any additional connections from being established.
+ */
+ public void closeChannels() throws Exception {
+ List<Exception> closeExceptions = Collections.synchronizedList(new ArrayList<>());
+ servers.parallelStream().forEach(server -> {
+ try {
+ server.closeChannels();
+ } catch (Exception e) {
+ closeExceptions.add(e);
+ }
+ });
+ if (!closeExceptions.isEmpty()) {
+ Exception ex = null;
+ for (Exception closeException : closeExceptions) {
+ ex = ExceptionUtils.suppress(ex, closeException);
+ }
+ throw ex;
+ }
+
+ }
}