[ASTERIXDB-2065][ING] Make stop timeout configurable

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Make timeout of a graceful stop of an active job
  configurable.

Change-Id: Id8d0c205a6959967dbce2b7223061ffa2f26059c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2845
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index 22c9b5d..39a8402 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -74,10 +74,7 @@
     private static final EnumSet<ActivityState> TRANSITION_STATES = EnumSet.of(ActivityState.RESUMING,
             ActivityState.STARTING, ActivityState.STOPPING, ActivityState.RECOVERING, ActivityState.CANCELLING);
     private static final String DEFAULT_ACTIVE_STATS = "{\"Stats\":\"N/A\"}";
-    // TODO: Make configurable https://issues.apache.org/jira/browse/ASTERIXDB-2065
-    protected static final long STOP_MESSAGE_TIMEOUT = 5L;
-    protected static final long SUSPEND_MESSAGE_TIMEOUT = 10L;
-    protected static final TimeUnit TIMEOUT_UNIT = TimeUnit.MINUTES;
+    protected static final TimeUnit TIMEOUT_UNIT = TimeUnit.SECONDS;
     protected final IClusterStateManager clusterStateManager;
     protected final ActiveNotificationHandler handler;
     protected final List<IActiveEntityEventSubscriber> subscribers = new ArrayList<>();
@@ -521,7 +518,7 @@
         } else if (state == ActivityState.RUNNING) {
             setState(ActivityState.STOPPING);
             try {
-                doStop(metadataProvider, STOP_MESSAGE_TIMEOUT, TIMEOUT_UNIT);
+                doStop(metadataProvider, appCtx.getActiveProperties().getActiveStopTimeout(), TIMEOUT_UNIT);
             } catch (Exception e) {
                 setState(ActivityState.STOPPED);
                 LOGGER.log(Level.ERROR, "Failed to stop the entity " + entityId, e);
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
index 6c95958..64520a4 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
@@ -37,6 +37,7 @@
 import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
 import org.apache.asterix.common.api.IMetadataLockManager;
 import org.apache.asterix.common.cluster.IClusterStateManager;
+import org.apache.asterix.common.config.ActiveProperties;
 import org.apache.asterix.common.context.IStorageComponentProvider;
 import org.apache.asterix.common.exceptions.ErrorCode;
 import org.apache.asterix.external.feed.watch.WaitForStateSubscriber;
@@ -121,6 +122,7 @@
         Mockito.when(appCtx.getMetadataLockManager()).thenReturn(lockManager);
         Mockito.when(appCtx.getServiceContext()).thenReturn(ccServiceCtx);
         Mockito.when(appCtx.getClusterStateManager()).thenReturn(clusterStateManager);
+        Mockito.when(appCtx.getActiveProperties()).thenReturn(Mockito.mock(ActiveProperties.class));
         componentProvider = new StorageComponentProvider();
         Mockito.when(appCtx.getStorageComponentProvider()).thenReturn(componentProvider);
         Mockito.when(ccServiceCtx.getControllerService()).thenReturn(ccService);
@@ -375,7 +377,7 @@
         testStartWhenStartSucceed();
         // suspend
         Assert.assertEquals(ActivityState.RUNNING, listener.getState());
-        listener.onStop(Behavior.FAIL_COMPILE);
+        listener.onStop(Behavior.RUNNING_JOB_FAIL);
         Action suspension = users[1].suspendActivity(listener);
         suspension.sync();
         Assert.assertFalse(suspension.hasFailed());
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestEventsListener.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestEventsListener.java
index 80806f3..2143404 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestEventsListener.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/TestEventsListener.java
@@ -196,6 +196,9 @@
     }
 
     public void onStop(Behavior behavior) {
+        if (behavior == Behavior.FAIL_COMPILE) {
+            throw new IllegalArgumentException("Test framework is not designed for this case");
+        }
         this.onStop = behavior;
     }
 
@@ -215,7 +218,7 @@
 
     @Override
     protected void doSuspend(MetadataProvider metadataProvider) throws HyracksDataException {
-        doStop(metadataProvider, SUSPEND_MESSAGE_TIMEOUT, TIMEOUT_UNIT);
+        doStop(metadataProvider, appCtx.getActiveProperties().getActiveSuspendTimeout(), TIMEOUT_UNIT);
     }
 
     @Override
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 572323a..ccb8c45 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -6,6 +6,8 @@
   \},
   "config" : \{
     "active\.memory\.global\.budget" : 67108864,
+    "active\.stop\.timeout" : 3600,
+    "active\.suspend\.timeout" : 3600,
     "compiler\.framesize" : 32768,
     "compiler\.groupmemory" : 163840,
     "compiler\.joinmemory" : 262144,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 622f8fd..56fdf91 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -6,6 +6,8 @@
   \},
   "config" : \{
     "active\.memory\.global\.budget" : 67108864,
+    "active\.stop\.timeout" : 3600,
+    "active\.suspend\.timeout" : 3600,
     "compiler\.framesize" : 32768,
     "compiler\.groupmemory" : 163840,
     "compiler\.joinmemory" : 262144,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 9ade575..658f4b1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -6,6 +6,8 @@
   \},
   "config" : \{
     "active\.memory\.global\.budget" : 67108864,
+    "active\.stop\.timeout" : 3600,
+    "active\.suspend\.timeout" : 3600,
     "compiler\.framesize" : 32768,
     "compiler\.groupmemory" : 163840,
     "compiler\.joinmemory" : 262144,
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ActiveProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ActiveProperties.java
index 8455a6f..c64c287 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ActiveProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ActiveProperties.java
@@ -18,6 +18,7 @@
  */
 package org.apache.asterix.common.config;
 
+import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER;
 import static org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT;
 import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE;
 
@@ -32,7 +33,9 @@
         ACTIVE_MEMORY_GLOBAL_BUDGET(
                 LONG_BYTE_UNIT,
                 StorageUtil.getLongSizeInBytes(64L, MEGABYTE),
-                "The memory budget (in bytes) for the active runtime");
+                "The memory budget (in bytes) for the active runtime"),
+        ACTIVE_STOP_TIMEOUT(INTEGER, 3600, "The maximum time to wait for a graceful stop of an active runtime"),
+        ACTIVE_SUSPEND_TIMEOUT(INTEGER, 3600, "The maximum time to wait for a graceful suspend of an active runtime");
 
         private final IOptionType type;
         private final Object defaultValue;
@@ -72,4 +75,12 @@
     public long getMemoryComponentGlobalBudget() {
         return accessor.getLong(Option.ACTIVE_MEMORY_GLOBAL_BUDGET);
     }
+
+    public int getActiveStopTimeout() {
+        return accessor.getInt(Option.ACTIVE_STOP_TIMEOUT);
+    }
+
+    public int getActiveSuspendTimeout() {
+        return accessor.getInt(Option.ACTIVE_SUSPEND_TIMEOUT);
+    }
 }