Add Diagnostics Endpoint

Add an endpoint (http://<host>:19001/admin/diagnostics) to gather common
diagnostics information on the cluster for cc and each ncs:
- threaddump
- config
- stats

Change-Id: I664b713f6614acd5ce4f8bf8ce6f8f71345cab06
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1310
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
index 7e7b068..2b32c60 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
@@ -48,6 +48,7 @@
     public static final String SHUTDOWN_URI_KEY = "shutdownUri";
     public static final String FULL_SHUTDOWN_URI_KEY = "fullShutdownUri";
     public static final String VERSION_URI_KEY = "versionUri";
+    public static final String DIAGNOSTICS_URI_KEY = "diagnosticsUri";
     public static final Pattern PARENT_DIR = Pattern.compile("/[^./]+/\\.\\./");
 
     @Override
@@ -69,8 +70,6 @@
             json = getClusterStateJSON(request, "");
             response.setStatus(HttpServletResponse.SC_OK);
             responseWriter.write(json.toString(4));
-        } catch (IllegalArgumentException e) { // NOSONAR - exception not logged or rethrown
-            response.sendError(HttpServletResponse.SC_BAD_REQUEST);
         } catch (Exception e) {
             LOGGER.log(Level.INFO, "exception thrown for " + request, e);
             response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.toString());
@@ -125,6 +124,7 @@
         json.put(SHUTDOWN_URI_KEY, analyticsURL + "shutdown");
         json.put(FULL_SHUTDOWN_URI_KEY, analyticsURL + "shutdown?all=true");
         json.put(VERSION_URI_KEY, analyticsURL + "version");
+        json.put(DIAGNOSTICS_URI_KEY, analyticsURL + "diagnostics");
         return json;
     }
 
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterNodeDetailsAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterNodeDetailsAPIServlet.java
index f5509c2..1489e55 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterNodeDetailsAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterNodeDetailsAPIServlet.java
@@ -118,7 +118,7 @@
         }
     }
 
-    private void fixupKeys(JSONObject json) throws JSONException {
+    protected JSONObject fixupKeys(JSONObject json) throws JSONException {
         // TODO (mblow): generate the keys with _ to begin with
         List<String> keys = new ArrayList<>();
         for (Iterator iter = json.keys(); iter.hasNext(); ) {
@@ -130,9 +130,10 @@
                 json.put(newKey, json.remove(key));
             }
         }
+        return json;
     }
 
-    private JSONObject processNodeStats(IHyracksClientConnection hcc, String node) throws Exception {
+    protected JSONObject processNodeStats(IHyracksClientConnection hcc, String node) throws Exception {
         final String details = hcc.getNodeDetailsJSON(node, true, false);
         if (details == null) {
             throw new IllegalArgumentException();
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DiagnosticsAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DiagnosticsAPIServlet.java
new file mode 100644
index 0000000..fd5547c
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DiagnosticsAPIServlet.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.api.http.servlet;
+
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.asterix.runtime.util.AsterixAppContextInfo;
+import org.apache.hyracks.api.client.IHyracksClientConnection;
+import org.json.JSONObject;
+
+public class DiagnosticsAPIServlet extends ClusterNodeDetailsAPIServlet {
+    private static final long serialVersionUID = 1L;
+    private static final Logger LOGGER = Logger.getLogger(DiagnosticsAPIServlet.class.getName());
+
+    @Override
+    protected void getUnsafe(HttpServletRequest request, HttpServletResponse response) throws IOException {
+        response.setContentType("application/json");
+        response.setCharacterEncoding("utf-8");
+        PrintWriter responseWriter = response.getWriter();
+        JSONObject json;
+        try {
+            if (request.getPathInfo() != null) {
+                throw new IllegalArgumentException();
+            }
+            json = getClusterDiagnosticsJSON();
+            response.setStatus(HttpServletResponse.SC_OK);
+            responseWriter.write(json.toString(4));
+        } catch (IllegalStateException e) { // NOSONAR - exception not logged or rethrown
+            response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
+        } catch (IllegalArgumentException e) { // NOSONAR - exception not logged or rethrown
+            response.sendError(HttpServletResponse.SC_NOT_FOUND);
+        } catch (Exception e) {
+            LOGGER.log(Level.INFO, "exception thrown for " + request, e);
+            response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.toString());
+        }
+        responseWriter.flush();
+    }
+
+    private JSONObject getClusterDiagnosticsJSON() throws Exception {
+        final ServletContext context = getServletContext();
+        IHyracksClientConnection hcc =
+                (IHyracksClientConnection) context.getAttribute(HYRACKS_CONNECTION_ATTR);
+        ExecutorService executor = (ExecutorService) context.getAttribute(ServletConstants.EXECUTOR_SERVICE);
+
+        Map<String, Future<JSONObject>> ccFutureData = new HashMap<>();
+        ccFutureData.put("threaddump", executor.submit(() -> fixupKeys(new JSONObject(hcc.getThreadDump(null)))));
+        ccFutureData.put("config", executor.submit(() ->
+                fixupKeys(new JSONObject(hcc.getNodeDetailsJSON(null, false, true)))));
+        ccFutureData.put("stats", executor.submit(() ->
+                fixupKeys(new JSONObject(hcc.getNodeDetailsJSON(null, true, false)))));
+
+        Map<String, Map<String, Future<JSONObject>>> ncDataMap = new HashMap<>();
+        for (String nc : AsterixAppContextInfo.INSTANCE.getMetadataProperties().getNodeNames()) {
+            Map<String, Future<JSONObject>> ncData = new HashMap<>();
+            ncData.put("threaddump", executor.submit(() ->
+                    fixupKeys(new JSONObject(hcc.getThreadDump(nc)))));
+            ncData.put("config", executor.submit(() ->
+                    fixupKeys(new JSONObject(hcc.getNodeDetailsJSON(nc, false, true)))));
+            ncData.put("stats", executor.submit(() ->
+                    fixupKeys(processNodeStats(hcc, nc))));
+            ncDataMap.put(nc, ncData);
+        }
+        JSONObject result = new JSONObject();
+        result.put("cc", resolveFutures(ccFutureData));
+        List<Map<String, ?>> ncList = new ArrayList<>();
+        for (Map.Entry<String, Map<String, Future<JSONObject>>> entry : ncDataMap.entrySet()) {
+            final Map<String, Object> ncMap = resolveFutures(entry.getValue());
+            ncMap.put("node_id", entry.getKey());
+            ncList.add(ncMap);
+        }
+        result.put("ncs", ncList);
+        result.put("date", new Date());
+        return result;
+    }
+
+    private Map<String, Object> resolveFutures(Map<String, Future<JSONObject>> futureMap)
+            throws ExecutionException, InterruptedException {
+        Map<String, Object> result = new HashMap<>();
+        for (Map.Entry<String, Future<JSONObject>> entry : futureMap.entrySet()) {
+            result.put(entry.getKey(), entry.getValue().get());
+        }
+        return result;
+    }
+}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ServletConstants.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ServletConstants.java
index 9e28b02..d5f31ff 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ServletConstants.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ServletConstants.java
@@ -22,6 +22,7 @@
     public static final String HYRACKS_CONNECTION_ATTR = "org.apache.asterix.HYRACKS_CONNECTION";
     public static final String HYRACKS_DATASET_ATTR = "org.apache.asterix.HYRACKS_DATASET";
     public static final String ASTERIX_BUILD_PROP_ATTR = "org.apache.asterix.PROPS";
+    public static final String EXECUTOR_SERVICE = "org.apache.asterix.EXECUTOR_SERVICE";
 
     private ServletConstants() {
     }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
index 919af33..b097244 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
@@ -36,12 +36,14 @@
 import org.apache.asterix.api.http.servlet.ClusterNodeDetailsAPIServlet;
 import org.apache.asterix.api.http.servlet.ConnectorAPIServlet;
 import org.apache.asterix.api.http.servlet.DDLAPIServlet;
+import org.apache.asterix.api.http.servlet.DiagnosticsAPIServlet;
 import org.apache.asterix.api.http.servlet.FeedServlet;
 import org.apache.asterix.api.http.servlet.QueryAPIServlet;
 import org.apache.asterix.api.http.servlet.QueryResultAPIServlet;
 import org.apache.asterix.api.http.servlet.QueryServiceServlet;
 import org.apache.asterix.api.http.servlet.QueryStatusAPIServlet;
 import org.apache.asterix.api.http.servlet.QueryWebInterfaceServlet;
+import org.apache.asterix.api.http.servlet.ServletConstants;
 import org.apache.asterix.api.http.servlet.ShutdownAPIServlet;
 import org.apache.asterix.api.http.servlet.UpdateAPIServlet;
 import org.apache.asterix.api.http.servlet.VersionAPIServlet;
@@ -49,14 +51,11 @@
 import org.apache.asterix.app.cc.CompilerExtensionManager;
 import org.apache.asterix.app.external.ExternalLibraryUtils;
 import org.apache.asterix.common.api.AsterixThreadFactory;
-import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
 import org.apache.asterix.common.config.AsterixExtension;
 import org.apache.asterix.common.config.AsterixExternalProperties;
 import org.apache.asterix.common.config.AsterixMetadataProperties;
-import org.apache.asterix.common.config.ClusterProperties;
 import org.apache.asterix.common.library.ILibraryManager;
 import org.apache.asterix.common.utils.ServletUtil.Servlets;
-import org.apache.asterix.event.service.ILookupService;
 import org.apache.asterix.external.library.ExternalLibraryManager;
 import org.apache.asterix.messaging.CCMessageBroker;
 import org.apache.asterix.metadata.MetadataManager;
@@ -199,6 +198,8 @@
         IHyracksClientConnection hcc = getNewHyracksClientConnection();
         context.setAttribute(HYRACKS_CONNECTION_ATTR, hcc);
         context.setAttribute(ASTERIX_BUILD_PROP_ATTR, AsterixAppContextInfo.INSTANCE);
+        context.setAttribute(ServletConstants.EXECUTOR_SERVICE,
+                ((ClusterControllerService) appCtx.getControllerService()).getExecutor());
 
         jsonAPIServer.setHandler(context);
 
@@ -208,7 +209,7 @@
         addServlet(context, Servlets.AQL_DDL);
         addServlet(context, Servlets.AQL);
 
-        // SQL++ rest APIs.
+        // SQL+x+ rest APIs.
         addServlet(context, Servlets.SQLPP_QUERY);
         addServlet(context, Servlets.SQLPP_UPDATE);
         addServlet(context, Servlets.SQLPP_DDL);
@@ -224,6 +225,7 @@
         addServlet(context, Servlets.CLUSTER_STATE);
         addServlet(context, Servlets.CLUSTER_STATE_NODE_DETAIL);
         addServlet(context, Servlets.CLUSTER_STATE_CC_DETAIL);
+        addServlet(context, Servlets.DIAGNOSTICS);
 
         return jsonAPIServer;
     }
@@ -301,6 +303,8 @@
                 return new ClusterNodeDetailsAPIServlet();
             case CLUSTER_STATE_CC_DETAIL:
                 return new ClusterCCDetailsAPIServlet();
+            case DIAGNOSTICS:
+                return new DiagnosticsAPIServlet();
             default:
                 throw new IllegalStateException(String.valueOf(key));
         }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/APIQueries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/APIQueries.xml
index 5f60bce..67797b2 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/APIQueries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/APIQueries.xml
@@ -58,4 +58,9 @@
       <output-dir compare="Text">version_1</output-dir>
     </compilation-unit>
   </test-case>
+  <test-case FilePath="api">
+    <compilation-unit name="diagnostics_1">
+      <output-dir compare="Text">diagnostics_1</output-dir>
+    </compilation-unit>
+  </test-case>
 </test-group>
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/diagnostics_1/diagnostics_1.1.httpapi.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/diagnostics_1/diagnostics_1.1.httpapi.aql
new file mode 100644
index 0000000..c309110
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/diagnostics_1/diagnostics_1.1.httpapi.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : diagnostics_1
+ * Description     : Diagnostics
+ * Expected Result : Positive
+ * Date            : 8th September 2016
+ */
+/admin/diagnostics
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm
index 0db4561..9898b2c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm
@@ -89,6 +89,7 @@
         "web.queryinterface.port": 19006,
         "web.secondary.port": 19005
     },
+    "diagnosticsUri": "http://127.0.0.1:19002/admin/diagnostics",
     "fullShutdownUri": "http://127.0.0.1:19002/admin/shutdown?all=true",
     "metadata_node": "asterix_nc1",
     "ncs": [
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/diagnostics_1/diagnostics_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/diagnostics_1/diagnostics_1.1.regexadm
new file mode 100644
index 0000000..29c03bf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/diagnostics_1/diagnostics_1.1.regexadm
@@ -0,0 +1,10 @@
+.*"cc": \{
+        "config": \{.*
+        \},
+        "stats": \{.*
+        \},
+        "threaddump": \{.*
+        \}
+.*"ncs".*"node_id": "asterix_nc1",
+.*"threaddump".*"node_id": "asterix_nc2",
+.*"threaddump".*
\ No newline at end of file
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ServletUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ServletUtil.java
index 5436dc7..45c9ed7 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ServletUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ServletUtil.java
@@ -37,7 +37,8 @@
         VERSION("/admin/version"),
         CLUSTER_STATE("/admin/cluster"),
         CLUSTER_STATE_NODE_DETAIL("/admin/cluster/node/*"),
-        CLUSTER_STATE_CC_DETAIL("/admin/cluster/cc/*");
+        CLUSTER_STATE_CC_DETAIL("/admin/cluster/cc/*"),
+        DIAGNOSTICS("/admin/diagnostics");
 
         private final String path;
 
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
index 741518a..9c764e9 100644
--- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
@@ -138,7 +138,8 @@
                 runScriptAndCompareWithResultRegex(scriptFile, expectedFile, actualFile);
                 return;
             } else if (actualFile.toString().endsWith(".regexadm")) {
-                regex = true;
+                runScriptAndCompareWithResultRegexAdm(scriptFile, expectedFile, actualFile);
+                return;
             }
             String lineExpected, lineActual;
             int num = 1;
@@ -356,6 +357,18 @@
 
     }
 
+    public void runScriptAndCompareWithResultRegexAdm(File scriptFile, File expectedFile, File actualFile)
+            throws Exception {
+        StringWriter actual = new StringWriter();
+        StringWriter expected = new StringWriter();
+        IOUtils.copy(new FileInputStream(actualFile), actual, StandardCharsets.UTF_8);
+        IOUtils.copy(new FileInputStream(expectedFile), expected, StandardCharsets.UTF_8);
+        Pattern pattern = Pattern.compile(expected.toString(), Pattern.DOTALL | Pattern.MULTILINE);
+        if (!pattern.matcher(actual.toString()).matches()) {
+            throw new Exception("Result for " + scriptFile + ": actual file did not match expected result");
+        }
+    }
+
     // For tests where you simply want the byte-for-byte output.
     private static void writeOutputToFile(File actualFile, InputStream resultStream) throws Exception {
         try (FileOutputStream out = new FileOutputStream(actualFile)) {
@@ -499,7 +512,7 @@
         return builder.build();
     }
 
-    public InputStream executeClusterStateQuery(OutputFormat fmt, String url) throws Exception {
+    public InputStream executeJSONGet(OutputFormat fmt, String url) throws Exception {
         HttpUriRequest request = RequestBuilder.get(url).setHeader("Accept", fmt.mimeType()).build();
 
         HttpResponse response = executeAndCheckHttpRequest(request);
@@ -849,7 +862,7 @@
             case "cstate": // cluster state query
                 fmt = OutputFormat.forCompilationUnit(cUnit);
                 String extra = stripJavaComments(statement).trim();
-                resultStream = executeClusterStateQuery(fmt, getEndpoint(Servlets.CLUSTER_STATE) + extra);
+                resultStream = executeJSONGet(fmt, getEndpoint(Servlets.CLUSTER_STATE) + extra);
                 expectedResultFile = expectedResultFileCtxs.get(queryCount.intValue()).getFile();
                 actualResultFile = testCaseCtx.getActualResultFile(cUnit, expectedResultFile, new File(actualPath));
                 actualResultFile.getParentFile().mkdirs();
@@ -860,7 +873,19 @@
                 break;
             case "version": // version servlet
                 fmt = OutputFormat.forCompilationUnit(cUnit);
-                resultStream = executeClusterStateQuery(fmt, getEndpoint(Servlets.VERSION));
+                resultStream = executeJSONGet(fmt, getEndpoint(Servlets.VERSION));
+                expectedResultFile = expectedResultFileCtxs.get(queryCount.intValue()).getFile();
+                actualResultFile = testCaseCtx.getActualResultFile(cUnit, expectedResultFile, new File(actualPath));
+                actualResultFile.getParentFile().mkdirs();
+                writeOutputToFile(actualResultFile, resultStream);
+                runScriptAndCompareWithResult(testFile, new PrintWriter(System.err), expectedResultFile,
+                        actualResultFile);
+                queryCount.increment();
+                break;
+            case "httpapi": // http api
+                fmt = OutputFormat.forCompilationUnit(cUnit);
+                extra = stripJavaComments(statement).trim();
+                resultStream = executeJSONGet(fmt, "http://" + host + ":" + port + extra);
                 expectedResultFile = expectedResultFileCtxs.get(queryCount.intValue()).getFile();
                 actualResultFile = testCaseCtx.getActualResultFile(cUnit, expectedResultFile, new File(actualPath));
                 actualResultFile.getParentFile().mkdirs();
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
index 4fde1f6..ac38523 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
@@ -251,6 +251,7 @@
     private void startApplication() throws Exception {
         appCtx = new CCApplicationContext(this, serverCtx, ccContext, ccConfig.getAppConfig());
         appCtx.addJobLifecycleListener(datasetDirectoryService);
+        executor = Executors.newCachedThreadPool(appCtx.getThreadFactory());
         String className = ccConfig.appCCMainClass;
         if (className != null) {
             Class<?> c = Class.forName(className);
@@ -259,7 +260,6 @@
                     : ccConfig.appArgs.toArray(new String[ccConfig.appArgs.size()]);
             aep.start(appCtx, args);
         }
-        executor = Executors.newCachedThreadPool(appCtx.getThreadFactory());
     }
 
     private void connectNCs() throws Exception {