[ASTERIXDB-2065][ING] Make stop timeout configurable
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Make timeout of a graceful stop of an active job
configurable.
Change-Id: Id8d0c205a6959967dbce2b7223061ffa2f26059c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2845
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index 22c9b5d..39a8402 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -74,10 +74,7 @@
private static final EnumSet<ActivityState> TRANSITION_STATES = EnumSet.of(ActivityState.RESUMING,
ActivityState.STARTING, ActivityState.STOPPING, ActivityState.RECOVERING, ActivityState.CANCELLING);
private static final String DEFAULT_ACTIVE_STATS = "{\"Stats\":\"N/A\"}";
- // TODO: Make configurable https://issues.apache.org/jira/browse/ASTERIXDB-2065
- protected static final long STOP_MESSAGE_TIMEOUT = 5L;
- protected static final long SUSPEND_MESSAGE_TIMEOUT = 10L;
- protected static final TimeUnit TIMEOUT_UNIT = TimeUnit.MINUTES;
+ protected static final TimeUnit TIMEOUT_UNIT = TimeUnit.SECONDS;
protected final IClusterStateManager clusterStateManager;
protected final ActiveNotificationHandler handler;
protected final List<IActiveEntityEventSubscriber> subscribers = new ArrayList<>();
@@ -521,7 +518,7 @@
} else if (state == ActivityState.RUNNING) {
setState(ActivityState.STOPPING);
try {
- doStop(metadataProvider, STOP_MESSAGE_TIMEOUT, TIMEOUT_UNIT);
+ doStop(metadataProvider, appCtx.getActiveProperties().getActiveStopTimeout(), TIMEOUT_UNIT);
} catch (Exception e) {
setState(ActivityState.STOPPED);
LOGGER.log(Level.ERROR, "Failed to stop the entity " + entityId, e);
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
index 6c95958..64520a4 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
@@ -37,6 +37,7 @@
import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
import org.apache.asterix.common.api.IMetadataLockManager;
import org.apache.asterix.common.cluster.IClusterStateManager;
+import org.apache.asterix.common.config.ActiveProperties;
import org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.external.feed.watch.WaitForStateSubscriber;
@@ -121,6 +122,7 @@
Mockito.when(appCtx.getMetadataLockManager()).thenReturn(lockManager);
Mockito.when(appCtx.getServiceContext()).thenReturn(ccServiceCtx);
Mockito.when(appCtx.getClusterStateManager()).thenReturn(clusterStateManager);
+ Mockito.when(appCtx.getActiveProperties()).thenReturn(Mockito.mock(ActiveProperties.class));
componentProvider = new StorageComponentProvider();
Mockito.when(appCtx.getStorageComponentProvider()).thenReturn(componentProvider);
Mockito.when(ccServiceCtx.getControllerService()).thenReturn(ccService);
@@ -375,7 +377,7 @@
testStartWhenStartSucceed();
// suspend
Assert.assertEquals(ActivityState.RUNNING, listener.getState());
- listener.onStop(Behavior.FAIL_COMPILE);
+ listener.onStop(Behavior.RUNNING_JOB_FAIL);
Action suspension = users[1].suspendActivity(listener);
suspension.sync();
Assert.assertFalse(suspension.hasFailed());
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestEventsListener.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestEventsListener.java
index 80806f3..2143404 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestEventsListener.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestEventsListener.java
@@ -196,6 +196,9 @@
}
public void onStop(Behavior behavior) {
+ if (behavior == Behavior.FAIL_COMPILE) {
+ throw new IllegalArgumentException("Test framework is not designed for this case");
+ }
this.onStop = behavior;
}
@@ -215,7 +218,7 @@
@Override
protected void doSuspend(MetadataProvider metadataProvider) throws HyracksDataException {
- doStop(metadataProvider, SUSPEND_MESSAGE_TIMEOUT, TIMEOUT_UNIT);
+ doStop(metadataProvider, appCtx.getActiveProperties().getActiveSuspendTimeout(), TIMEOUT_UNIT);
}
@Override
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 572323a..ccb8c45 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -6,6 +6,8 @@
\},
"config" : \{
"active\.memory\.global\.budget" : 67108864,
+ "active\.stop\.timeout" : 3600,
+ "active\.suspend\.timeout" : 3600,
"compiler\.framesize" : 32768,
"compiler\.groupmemory" : 163840,
"compiler\.joinmemory" : 262144,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 622f8fd..56fdf91 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -6,6 +6,8 @@
\},
"config" : \{
"active\.memory\.global\.budget" : 67108864,
+ "active\.stop\.timeout" : 3600,
+ "active\.suspend\.timeout" : 3600,
"compiler\.framesize" : 32768,
"compiler\.groupmemory" : 163840,
"compiler\.joinmemory" : 262144,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 9ade575..658f4b1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -6,6 +6,8 @@
\},
"config" : \{
"active\.memory\.global\.budget" : 67108864,
+ "active\.stop\.timeout" : 3600,
+ "active\.suspend\.timeout" : 3600,
"compiler\.framesize" : 32768,
"compiler\.groupmemory" : 163840,
"compiler\.joinmemory" : 262144,
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ActiveProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ActiveProperties.java
index 8455a6f..c64c287 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ActiveProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ActiveProperties.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.common.config;
+import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER;
import static org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT;
import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE;
@@ -32,7 +33,9 @@
ACTIVE_MEMORY_GLOBAL_BUDGET(
LONG_BYTE_UNIT,
StorageUtil.getLongSizeInBytes(64L, MEGABYTE),
- "The memory budget (in bytes) for the active runtime");
+ "The memory budget (in bytes) for the active runtime"),
+ ACTIVE_STOP_TIMEOUT(INTEGER, 3600, "The maximum time to wait for a graceful stop of an active runtime"),
+ ACTIVE_SUSPEND_TIMEOUT(INTEGER, 3600, "The maximum time to wait for a graceful suspend of an active runtime");
private final IOptionType type;
private final Object defaultValue;
@@ -72,4 +75,12 @@
public long getMemoryComponentGlobalBudget() {
return accessor.getLong(Option.ACTIVE_MEMORY_GLOBAL_BUDGET);
}
+
+ public int getActiveStopTimeout() {
+ return accessor.getInt(Option.ACTIVE_STOP_TIMEOUT);
+ }
+
+ public int getActiveSuspendTimeout() {
+ return accessor.getInt(Option.ACTIVE_SUSPEND_TIMEOUT);
+ }
}