[NO ISSUE][CLUS] Request NC Startup Tasks After Startup Completion
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Wait for NC startup completion before requesting startup
tasks from CC to ensure the NCs web servers are up before
declaring it active.
- Check node status before validations on adding replica.
Change-Id: I7c58d006546f3ebca91333c2a4bc8ced68fdaf39
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2830
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
index a0e3c8b..ad70cf4 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
@@ -68,6 +68,13 @@
@Override
public synchronized void addReplica(ReplicaIdentifier id) {
+ final NodeControllerService controllerService =
+ (NodeControllerService) appCtx.getServiceContext().getControllerService();
+ final NodeStatus nodeStatus = controllerService.getNodeStatus();
+ if (nodeStatus != NodeStatus.ACTIVE) {
+ LOGGER.warn("Ignoring request to add replica. Node is not ACTIVE yet. Current status: {}", nodeStatus);
+ return;
+ }
if (!partitions.contains(id.getPartition())) {
throw new IllegalStateException(
"This node is not the current master of partition(" + id.getPartition() + ")");
@@ -76,13 +83,6 @@
LOGGER.info("ignoring request to add replica to ourselves");
return;
}
- final NodeControllerService controllerService =
- (NodeControllerService) appCtx.getServiceContext().getControllerService();
- final NodeStatus nodeStatus = controllerService.getNodeStatus();
- if (nodeStatus != NodeStatus.ACTIVE) {
- LOGGER.warn("Ignoring request to add replica. Node is not ACTIVE yet. Current status: {}", nodeStatus);
- return;
- }
replicas.computeIfAbsent(id, k -> new PartitionReplica(k, appCtx));
replicas.get(id).sync();
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index 019f54d..fbafc2e 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -83,6 +83,7 @@
private INcApplicationContext runtimeContext;
private String nodeId;
private boolean stopInitiated;
+ private boolean startupCompleted;
protected WebManager webManager;
@Override
@@ -211,14 +212,19 @@
}
@Override
- public void startupCompleted() throws Exception {
+ public synchronized void startupCompleted() throws Exception {
// configure servlets after joining the cluster, so we can create HyracksClientConnection
configureServers();
webManager.start();
+ startupCompleted = true;
+ notifyAll();
}
@Override
- public synchronized void onRegisterNode(CcId ccId) throws Exception {
+ public synchronized void tasksCompleted(CcId ccId) throws Exception {
+ while (!startupCompleted) {
+ this.wait();
+ }
final NodeControllerService ncs = (NodeControllerService) ncServiceCtx.getControllerService();
final NodeStatus currentStatus = ncs.getNodeStatus();
final SystemState systemState = isPendingStartupTasks(currentStatus, ncs.getPrimaryCcId(), ccId)
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCApplication.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCApplication.java
index af6cb92..919722d 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCApplication.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCApplication.java
@@ -34,5 +34,5 @@
*/
IFileDeviceResolver getFileDeviceResolver();
- void onRegisterNode(CcId ccId) throws Exception;
+ void tasksCompleted(CcId ccId) throws Exception;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/BaseNCApplication.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/BaseNCApplication.java
index ea16032..800721c9 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/BaseNCApplication.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/BaseNCApplication.java
@@ -60,7 +60,7 @@
}
@Override
- public void onRegisterNode(CcId ccId) throws Exception {
+ public void tasksCompleted(CcId ccId) throws Exception {
// no-op
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
index 169e5ea..a189ac5 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
@@ -655,7 +655,7 @@
public void notifyTasksCompleted(CcId ccId) throws Exception {
partitionManager.jobsCompleted(ccId);
- application.onRegisterNode(ccId);
+ application.tasksCompleted(ccId);
}
private static INCApplication getApplication(NCConfig config)
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/TestNCApplication.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/TestNCApplication.java
index 15248e7..b13feda 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/TestNCApplication.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/TestNCApplication.java
@@ -46,7 +46,7 @@
}
@Override
- public void onRegisterNode(CcId ccs) throws Exception {
+ public void tasksCompleted(CcId ccs) throws Exception {
// No-op
}