[NO ISSUE][HYR][TEST] Refactor hyracks-server tests to avoid dependency issues

Change-Id: I4f504f3137f843a2340e4b4558ec55e4f0fd9436
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17874
Reviewed-by: Michael Blow <mblow@apache.org>
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
Tested-by: Michael Blow <mblow@apache.org>
Integration-Tests: Michael Blow <mblow@apache.org>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/pom.xml
new file mode 100644
index 0000000..bd7bf75
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/pom.xml
@@ -0,0 +1,150 @@
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements.  See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership.  The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License.  You may obtain a copy of the License at
+ !
+ !   http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied.  See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>hyracks-server-test</artifactId>
+  <name>hyracks-server-test</name>
+  <parent>
+    <groupId>org.apache.hyracks</groupId>
+    <artifactId>hyracks-tests</artifactId>
+    <version>0.3.8.2-SNAPSHOT</version>
+  </parent>
+
+  <licenses>
+    <license>
+      <name>Apache License, Version 2.0</name>
+      <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+      <distribution>repo</distribution>
+      <comments>A business-friendly OSS license</comments>
+    </license>
+  </licenses>
+
+  <properties>
+    <root.dir>${basedir}/../../..</root.dir>
+  </properties>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <executions>
+          <execution>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <configuration>
+          <usedDependencies combine.children="append">
+            <usedDependency>org.apache.hyracks:hyracks-server</usedDependency>
+          </usedDependencies>
+        </configuration>
+        <executions>
+          <execution>
+            <phase>process-test-classes</phase>
+            <goals>
+              <goal>analyze-only</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-failsafe-plugin</artifactId>
+        <configuration>
+          <runOrder>alphabetical</runOrder>
+          <forkMode>pertest</forkMode>
+        </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>integration-test</goal>
+              <goal>verify</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hyracks</groupId>
+      <artifactId>hyracks-control-cc</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hyracks</groupId>
+      <artifactId>hyracks-nc-service</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hyracks</groupId>
+      <artifactId>hyracks-server</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+<!--
+    <dependency>
+      <groupId>org.apache.hyracks</groupId>
+      <artifactId>hyracks-control-nc</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+-->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpcore</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hyracks</groupId>
+      <artifactId>hyracks-util</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/java/org/apache/hyracks/test/server/NCServiceIT.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/java/org/apache/hyracks/test/server/NCServiceIT.java
new file mode 100644
index 0000000..5ae57cd
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/java/org/apache/hyracks/test/server/NCServiceIT.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.test.server;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.Iterator;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.apache.hyracks.test.server.process.HyracksVirtualCluster;
+import org.apache.hyracks.util.file.FileUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+
+public class NCServiceIT {
+
+    private static final String TARGET_DIR = FileUtil.joinPath(".", "target");
+    private static final String LOG_DIR = FileUtil.joinPath(TARGET_DIR, "failsafe-reports");
+    private static final String RESOURCE_DIR = FileUtil.joinPath(TARGET_DIR, "test-classes", "NCServiceIT");
+    private static final String APP_HOME = FileUtil.joinPath("..", "..", "hyracks-server", "target", "appassembler");
+    private static final Logger LOGGER = LogManager.getLogger();
+
+    private static HyracksVirtualCluster cluster = null;
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        cluster = new HyracksVirtualCluster(new File(APP_HOME), null);
+
+        cluster.addNCService(new File(RESOURCE_DIR, "nc-red.conf"), null);
+        cluster.addNCService(new File(RESOURCE_DIR, "nc-blue.conf"), null);
+
+        try {
+            Thread.sleep(2000);
+        } catch (InterruptedException ignored) {
+        }
+
+        // Start CC
+        cluster.start(new File(RESOURCE_DIR, "cc.conf"), null);
+
+        try {
+            Thread.sleep(10000);
+        } catch (InterruptedException ignored) {
+        }
+    }
+
+    @AfterClass
+    public static void tearDown() throws IOException {
+        cluster.stop();
+    }
+
+    private static String getHttp(String url) throws Exception {
+        HttpClient client = HttpClients.createDefault();
+        HttpGet get = new HttpGet(url);
+        int statusCode;
+        final HttpResponse httpResponse;
+        try {
+            httpResponse = client.execute(get);
+            statusCode = httpResponse.getStatusLine().getStatusCode();
+        } catch (Exception e) {
+            e.printStackTrace();
+            throw e;
+        }
+        String response = EntityUtils.toString(httpResponse.getEntity());
+        if (statusCode == HttpStatus.SC_OK) {
+            return response;
+        } else {
+            throw new Exception("HTTP error " + statusCode + ":\n" + response);
+        }
+    }
+
+    private JsonNode getEndpoint(String endpoint) throws Exception {
+        ObjectMapper om = new ObjectMapper();
+        String localhost = InetAddress.getLoopbackAddress().getHostAddress();
+        String response = getHttp("http://" + localhost + ":12345" + endpoint);
+        JsonNode result = om.readTree(response);
+        JsonNode nodes = result.get("result");
+        return nodes;
+    }
+
+    @Test
+    public void IsNodelistCorrect() throws Exception {
+        // Ping the nodelist HTTP API
+
+        JsonNode nodes = getEndpoint("/rest/nodes");
+        int numNodes = nodes.size();
+        Assert.assertEquals("Wrong number of nodes!", 2, numNodes);
+        for (int i = 0; i < nodes.size(); i++) {
+            JsonNode node = nodes.get(i);
+            String id = node.get("node-id").asText();
+            if (id.equals("red") || id.equals("blue")) {
+                continue;
+            }
+            Assert.fail("Unexpected node ID '" + id + "'!");
+        }
+    }
+
+    @Test
+    public void isXmxOverrideCorrect() throws Exception {
+        ArrayNode inputArgs = (ArrayNode) getEndpoint("/rest/nodes/red").get("input-arguments");
+        for (Iterator<JsonNode> it = inputArgs.elements(); it.hasNext();) {
+            String s = it.next().asText();
+            if (s.startsWith("-Xmx") && s.endsWith("m")) {
+                String digits = s.substring(4, 8);
+                Assert.assertEquals("1234", digits);
+            }
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        try {
+            setUp();
+        } catch (Exception e) {
+            e.printStackTrace();
+            LOGGER.error("TEST CASE(S) FAILED");
+        } finally {
+            tearDown();
+        }
+    }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/java/org/apache/hyracks/test/server/process/HyracksCCProcess.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/java/org/apache/hyracks/test/server/process/HyracksCCProcess.java
new file mode 100644
index 0000000..0b529fb
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/java/org/apache/hyracks/test/server/process/HyracksCCProcess.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.test.server.process;
+
+import java.io.File;
+import java.util.List;
+
+import org.apache.hyracks.control.cc.CCDriver;
+
+public class HyracksCCProcess extends HyracksServerProcess {
+
+    HyracksCCProcess(File configFile, File logFile, File appHome, File workingDir) {
+        super(" cc");
+        this.configFile = configFile;
+        this.logFile = logFile;
+        this.appHome = appHome;
+        this.workingDir = workingDir;
+    }
+
+    @Override
+    protected String getMainClassName() {
+        return CCDriver.class.getName();
+    }
+
+    @Override
+    @SuppressWarnings("squid:CommentedOutCodeLine")
+    protected void addJvmArgs(List<String> cList) {
+        // CC needs more than default memory
+        args.add("-Xmx1024m");
+        cList.addAll(args);
+        // cList.add("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005");
+        cList.add("-Dfile.encoding=us-ascii");
+    }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/java/org/apache/hyracks/test/server/process/HyracksNCServiceProcess.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/java/org/apache/hyracks/test/server/process/HyracksNCServiceProcess.java
new file mode 100644
index 0000000..958eb9a
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/java/org/apache/hyracks/test/server/process/HyracksNCServiceProcess.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.test.server.process;
+
+import java.io.File;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hyracks.control.nc.service.NCService;
+
+public class HyracksNCServiceProcess extends HyracksServerProcess {
+    private static final AtomicInteger ncServiceCounter = new AtomicInteger();
+
+    HyracksNCServiceProcess(File configFile, File logFile, File appHome, File workingDir) {
+        super("nc" + ncServiceCounter.incrementAndGet());
+        this.configFile = configFile;
+        this.logFile = logFile;
+        this.appHome = appHome;
+        this.workingDir = workingDir;
+    }
+
+    @Override
+    protected String getMainClassName() {
+        return NCService.class.getName();
+    }
+
+    @Override
+    protected void addJvmArgs(List<String> cList) {
+        // NCService needs little memory
+        args.add("-Xmx128m");
+        cList.addAll(args);
+    }
+
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/java/org/apache/hyracks/test/server/process/HyracksServerProcess.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/java/org/apache/hyracks/test/server/process/HyracksServerProcess.java
new file mode 100644
index 0000000..59ab4f7
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/java/org/apache/hyracks/test/server/process/HyracksServerProcess.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.test.server.process;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+abstract class HyracksServerProcess {
+    private static final Logger LOGGER = LogManager.getLogger();
+
+    protected final String processName;
+    protected Process process;
+    protected Thread pipeThread;
+    protected File configFile = null;
+    protected File logFile = null;
+    protected File appHome = null;
+    protected File workingDir = null;
+    protected List<String> args = new ArrayList<>();
+
+    protected HyracksServerProcess(String processName) {
+        this.processName = processName;
+    }
+
+    public void start() throws IOException {
+
+        String[] cmd = buildCommand();
+        if (LOGGER.isInfoEnabled()) {
+            LOGGER.info("Starting command: " + Arrays.toString(cmd));
+        }
+
+        ProcessBuilder pb = new ProcessBuilder(cmd);
+        pb.redirectErrorStream(true);
+        pb.directory(workingDir);
+        if (logFile != null) {
+            LOGGER.info("Logging to: " + logFile.getCanonicalPath());
+            logFile.getParentFile().mkdirs();
+            try (FileWriter writer = new FileWriter(logFile, true)) {
+                writer.write("---------------------\n");
+            }
+            pb.redirectOutput(ProcessBuilder.Redirect.appendTo(logFile));
+            process = pb.start();
+        } else {
+            pb.redirectOutput(ProcessBuilder.Redirect.PIPE);
+            process = pb.start();
+            pipeThread = new Thread(() -> {
+                try (BufferedReader reader =
+                        new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) {
+                    String line;
+                    while ((line = reader.readLine()) != null) {
+                        System.out.println(processName + ": " + line);
+                    }
+                } catch (IOException e) {
+                    LOGGER.debug("exception reading process pipe", e);
+                }
+            });
+            pipeThread.start();
+        }
+    }
+
+    public void stop() {
+        process.destroy();
+        try {
+            boolean success = process.waitFor(30, TimeUnit.SECONDS);
+            if (!success) {
+                LOGGER.warn("Killing unresponsive NC Process");
+                process.destroyForcibly();
+            }
+            if (pipeThread != null) {
+                pipeThread.interrupt();
+                pipeThread.join();
+            }
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
+    }
+
+    public void stop(boolean forcibly) {
+        if (forcibly) {
+            process.destroyForcibly();
+        } else {
+            process.destroy();
+        }
+        try {
+            process.waitFor();
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
+    }
+
+    private String[] buildCommand() {
+        List<String> cList = new ArrayList<String>();
+        cList.add(getJavaCommand());
+        addJvmArgs(cList);
+        cList.add("-Dapp.home=" + appHome.getAbsolutePath());
+        cList.add("-classpath");
+        cList.add(getClasspath());
+        cList.add(getMainClassName());
+        if (configFile != null) {
+            cList.add("-config-file");
+            cList.add(configFile.getAbsolutePath());
+        }
+        addCmdLineArgs(cList);
+        return cList.toArray(new String[0]);
+    }
+
+    protected void addJvmArgs(List<String> cList) {
+    }
+
+    protected void addCmdLineArgs(List<String> cList) {
+    }
+
+    public void addArg(String arg) {
+        args.add(arg);
+    }
+
+    protected abstract String getMainClassName();
+
+    private final String getClasspath() {
+        return System.getProperty("java.class.path");
+    }
+
+    private final String getJavaCommand() {
+        return System.getProperty("java.home") + File.separator + "bin" + File.separator + "java";
+    }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/java/org/apache/hyracks/test/server/process/HyracksVirtualCluster.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/java/org/apache/hyracks/test/server/process/HyracksVirtualCluster.java
new file mode 100644
index 0000000..062d429
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/java/org/apache/hyracks/test/server/process/HyracksVirtualCluster.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.test.server.process;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Starts a local hyracks-based cluster (NC and CC child processes).
+ */
+public class HyracksVirtualCluster {
+    private final File appHome;
+    private final File workingDir;
+    private List<HyracksNCServiceProcess> ncProcs = new ArrayList<>(3);
+    private HyracksCCProcess ccProc = null;
+
+    /**
+     * Construct a Hyracks-based cluster.
+     *
+     * @param appHome
+     *            - path to the installation root of the Hyracks application.
+     *            At least bin/hyracksnc (or the equivalent NC script for
+     *            the application) must exist in this directory.
+     * @param workingDir
+     *            - directory to use as CWD for all child processes. May
+     *            be null, in which case the CWD of the invoking process is used.
+     */
+    public HyracksVirtualCluster(File appHome, File workingDir) {
+        this.appHome = appHome;
+        this.workingDir = workingDir;
+    }
+
+    /**
+     * Creates and starts an NCService.
+     *
+     * @param configFile
+     *            - full path to an ncservice.conf. May be null to accept all defaults.
+     * @throws IOException
+     *             - if there are errors starting the process.
+     */
+    public HyracksNCServiceProcess addNCService(File configFile, File logFile) throws IOException {
+        HyracksNCServiceProcess proc = new HyracksNCServiceProcess(configFile, logFile, appHome, workingDir);
+        proc.start();
+        ncProcs.add(proc);
+        return proc;
+    }
+
+    /**
+     * Starts the CC, initializing the cluster. Expects that any NCs referenced
+     * in the cluster configuration have already been started with addNCService().
+     *
+     * @param ccConfigFile
+     *            - full path to a cluster conf file. May be null to accept all
+     *            defaults, although this is seldom useful since there are no NCs.
+     * @throws IOException
+     *             - if there are errors starting the process.
+     */
+    public HyracksCCProcess start(File ccConfigFile, File logFile) throws IOException {
+        ccProc = new HyracksCCProcess(ccConfigFile, logFile, appHome, workingDir);
+        ccProc.start();
+        return ccProc;
+    }
+
+    /**
+     * Stops all processes in the cluster.
+     * QQQ Someday this should probably do a graceful stop of NCs rather than
+     * killing the NCService.
+     */
+    public void stop() {
+        ccProc.stop();
+        for (HyracksNCServiceProcess proc : ncProcs) {
+            proc.stop();
+        }
+    }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/resources/NCServiceIT/cc.conf b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/resources/NCServiceIT/cc.conf
new file mode 100644
index 0000000..9b1a1cd
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/resources/NCServiceIT/cc.conf
@@ -0,0 +1,33 @@
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements.  See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership.  The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License.  You may obtain a copy of the License at
+;
+;   http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing,
+; software distributed under the License is distributed on an
+; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+; KIND, either express or implied.  See the License for the
+; specific language governing permissions and limitations
+; under the License.
+
+[nc/red]
+address = 127.0.0.1
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006
+jvm.args= -Xmx1234m
+
+[nc/blue]
+address = 127.0.0.1
+ncservice.port = 9091
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007
+
+[cc]
+address = 127.0.0.1
+console.listen.port = 12345
+
+[common]
+log.dir=target/NCServiceIT
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/resources/NCServiceIT/nc-blue.conf b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/resources/NCServiceIT/nc-blue.conf
new file mode 100644
index 0000000..baccd46
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/resources/NCServiceIT/nc-blue.conf
@@ -0,0 +1,21 @@
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements.  See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership.  The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License.  You may obtain a copy of the License at
+;
+;   http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing,
+; software distributed under the License is distributed on an
+; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+; KIND, either express or implied.  See the License for the
+; specific language governing permissions and limitations
+; under the License.
+
+[ncservice]
+address=127.0.0.1
+port=9091
+logdir=-
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/resources/NCServiceIT/nc-red.conf b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/resources/NCServiceIT/nc-red.conf
new file mode 100644
index 0000000..7616b37
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/resources/NCServiceIT/nc-red.conf
@@ -0,0 +1,21 @@
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements.  See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership.  The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License.  You may obtain a copy of the License at
+;
+;   http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing,
+; software distributed under the License is distributed on an
+; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+; KIND, either express or implied.  See the License for the
+; specific language governing permissions and limitations
+; under the License.
+
+[ncservice]
+address=127.0.0.1
+port=9090
+logdir=-
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/resources/logging.properties b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/resources/logging.properties
new file mode 100644
index 0000000..e9f84796
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-server-test/src/test/resources/logging.properties
@@ -0,0 +1,76 @@
+#/*
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+############################################################
+# Default Logging Configuration File
+#
+# You can use a different file by specifying a filename
+# with the java.util.logging.config.file system property.
+# For example java -Djava.util.logging.config.file=myfile
+############################################################
+
+############################################################
+# Global properties
+############################################################
+
+# "handlers" specifies a comma separated list of log Handler
+# classes.  These handlers will be installed during VM startup.
+# Note that these classes must be on the system classpath.
+# By default we only configure a ConsoleHandler, which will only
+# show messages at the INFO and above levels.
+
+handlers= java.util.logging.ConsoleHandler
+
+# To also add the FileHandler, use the following line instead.
+
+# handlers= java.util.logging.FileHandler, java.util.logging.ConsoleHandler
+
+# Default global logging level.
+# This specifies which kinds of events are logged across
+# all loggers.  For any given facility this global level
+# can be overriden by a facility specific level
+# Note that the ConsoleHandler also has a separate level
+# setting to limit messages printed to the console.
+
+# .level= WARNING
+.level= INFO
+# .level= FINE
+# .level = FINEST
+
+############################################################
+# Handler specific properties.
+# Describes specific configuration info for Handlers.
+############################################################
+
+# default file output is in user's home directory.
+
+# java.util.logging.FileHandler.pattern = %h/java%u.log
+# java.util.logging.FileHandler.limit = 50000
+# java.util.logging.FileHandler.count = 1
+# java.util.logging.FileHandler.formatter = java.util.logging.XMLFormatter
+
+# Limit the message that are printed on the console to FINE and above.
+
+java.util.logging.ConsoleHandler.level = FINE
+java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter
+
+
+############################################################
+# Facility specific properties.
+# Provides extra control for each logger.
+############################################################
+
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/pom.xml
index a26adea..71bb0e9 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/pom.xml
@@ -52,5 +52,6 @@
     <module>hyracks-storage-am-lsm-invertedindex-test</module>
     <module>hyracks-storage-am-bloomfilter-test</module>
     <module>hyracks-dataflow-common-test</module>
+    <module>hyracks-server-test</module>
   </modules>
 </project>