[NO ISSUE] Fix logging for ncservice-based tests

Change-Id: Id264aede0f62558ad6e34355047c623a1d594692
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3311
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <imaxon@uci.edu>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java
index 22dea9f..a635084 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java
@@ -23,6 +23,8 @@
 
 import org.apache.hyracks.control.common.controllers.CCConfig;
 import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.core.LoggerContext;
 import org.apache.logging.log4j.core.config.Configuration;
 import org.apache.logging.log4j.core.config.ConfigurationFactory;
@@ -34,6 +36,7 @@
 import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration;
 
 public class CCLogConfigurationFactory extends ConfigurationFactory {
+    private static final Logger LOGGER = LogManager.getLogger();
     private CCConfig config;
 
     public CCLogConfigurationFactory(CCConfig config) {
@@ -42,6 +45,8 @@
 
     public Configuration createConfiguration(ConfigurationBuilder<BuiltConfiguration> builder) {
         File logDir = new File(config.getLogDir());
+        File ccLog = new File(logDir, "cc.log");
+        LOGGER.warn("logs are being redirected to: {}", ccLog::getAbsolutePath);
         builder.setStatusLevel(Level.WARN);
         builder.setConfigurationName("RollingBuilder");
         // create a rolling file appender
@@ -50,10 +55,9 @@
         ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
                 .addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?"))
                 .addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "50M"));
-        AppenderComponentBuilder defaultRoll =
-                builder.newAppender("default", "RollingFile").addAttribute("fileName", new File(logDir, "cc.log"))
-                        .addAttribute("filePattern", new File(logDir, "cc-%d{MM-dd-yy}.log.gz")).add(defaultLayout)
-                        .addComponent(triggeringPolicy);
+        AppenderComponentBuilder defaultRoll = builder.newAppender("default", "RollingFile")
+                .addAttribute("fileName", ccLog).addAttribute("filePattern", new File(logDir, "cc-%d{MM-dd-yy}.log.gz"))
+                .add(defaultLayout).addComponent(triggeringPolicy);
         builder.add(defaultRoll);
 
         // create the new logger
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java
index 24a3e57..c37acab 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java
@@ -79,7 +79,7 @@
 
     private final Map<String, String> systemProperties;
 
-    private final long pid;
+    private final int pid;
 
     private final HeartbeatSchema hbSchema;
 
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java
index 474bc0a..437b001 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java
@@ -75,7 +75,7 @@
 
     private final NetworkAddress messagingPort;
 
-    private final long pid;
+    private final int pid;
 
     private final NodeCapacity capacity;
 
@@ -190,7 +190,7 @@
         return messagingPort;
     }
 
-    public long getPid() {
+    public int getPid() {
         return pid;
     }
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/resources/log4j2.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/resources/log4j2.xml
new file mode 100644
index 0000000..c0197a4
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/resources/log4j2.xml
@@ -0,0 +1,47 @@
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements.  See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership.  The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License.  You may obtain a copy of the License at
+ !
+ !   http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied.  See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<Configuration status="WARN">
+   <CustomLevels>
+    <CustomLevel name="ACCESS" intLevel="550" />
+    <CustomLevel name="TRACER" intLevel="570" />
+  </CustomLevels>
+  <Appenders>
+    <Console name="Console" target="SYSTEM_OUT">
+      <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+    </Console>
+    <Console name="AccessLog" target="SYSTEM_OUT">
+      <PatternLayout pattern="%m%n"/>
+    </Console>
+    <Console name="TracerLog" target="SYSTEM_OUT">
+      <PatternLayout pattern="%m%n"/>
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Root level="WARN">
+      <AppenderRef ref="Console"/>
+    </Root>
+    <Logger name="org.apache.hyracks.control.nc.service" level="INFO"/>
+    <Logger name="org.apache.hyracks.http.server.CLFLogger" level="ACCESS" additivity="false">
+        <AppenderRef ref="AccessLog"/>
+    </Logger>
+    <Logger name="org.apache.hyracks.util.trace.Tracer.Traces" level="TRACER" additivity="false">
+      <AppenderRef ref="TracerLog"/>
+    </Logger>
+  </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java
index a657d6b..fad6b3e 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java
@@ -23,6 +23,8 @@
 
 import org.apache.hyracks.control.common.controllers.NCConfig;
 import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.core.LoggerContext;
 import org.apache.logging.log4j.core.config.Configuration;
 import org.apache.logging.log4j.core.config.ConfigurationFactory;
@@ -34,6 +36,7 @@
 import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration;
 
 public class NCLogConfigurationFactory extends ConfigurationFactory {
+    private static final Logger LOGGER = LogManager.getLogger();
     private NCConfig config;
 
     public NCLogConfigurationFactory(NCConfig config) {
@@ -43,6 +46,8 @@
     public Configuration createConfiguration(ConfigurationBuilder<BuiltConfiguration> builder) {
         String nodeId = config.getNodeId();
         File logDir = new File(config.getLogDir());
+        File ncLog = new File(logDir, "nc-" + nodeId + ".log");
+        LOGGER.warn("logs are being redirected to: {}", ncLog::getAbsolutePath);
         builder.setStatusLevel(Level.WARN);
         builder.setConfigurationName("RollingBuilder");
         // create a rolling file appender
@@ -51,11 +56,11 @@
         ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
                 .addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?"))
                 .addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "50M"));
-        AppenderComponentBuilder defaultRoll = builder.newAppender("default", "RollingFile")
-                .addAttribute("fileName", new File(logDir, "nc-" + nodeId + ".log").getAbsolutePath())
-                .addAttribute("filePattern",
-                        new File(logDir, "nc-" + nodeId + "-%d{MM-dd-yy-ss}.log.gz").getAbsolutePath())
-                .add(defaultLayout).addComponent(triggeringPolicy);
+        AppenderComponentBuilder defaultRoll =
+                builder.newAppender("default", "RollingFile").addAttribute("fileName", ncLog.getAbsolutePath())
+                        .addAttribute("filePattern",
+                                new File(logDir, "nc-" + nodeId + "-%d{MM-dd-yy-ss}.log.gz").getAbsolutePath())
+                        .add(defaultLayout).addComponent(triggeringPolicy);
         builder.add(defaultRoll);
 
         // create the new logger
@@ -74,9 +79,9 @@
         LayoutComponentBuilder traceLayout = builder.newLayout("PatternLayout").addAttribute("pattern", "%m,%n")
                 .addAttribute("header", "[").addAttribute("footer", "]");
         AppenderComponentBuilder traceRoll = builder.newAppender("trace", "RollingFile")
-                .addAttribute("fileName", logDir + "trace-" + nodeId + ".log")
-                .addAttribute("filePattern", logDir + "trace-" + nodeId + "-%d{MM-dd-yy-ss}.log.gz").add(traceLayout)
-                .addComponent(triggeringPolicy);
+                .addAttribute("fileName", new File(logDir, "trace-" + nodeId + ".log"))
+                .addAttribute("filePattern", new File(logDir, "trace-" + nodeId + "-%d{MM-dd-yy-ss}.log.gz"))
+                .add(traceLayout).addComponent(triggeringPolicy);
         builder.add(traceRoll);
         builder.add(builder.newLogger("org.apache.hyracks.util.trace.Tracer.Traces", Level.forName("TRACER", 570))
                 .add(builder.newAppenderRef("trace")).addAttribute("additivity", false));
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
index 94a6440..152f6c7 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
@@ -50,8 +50,6 @@
 import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.logging.LogLevel;
 import io.netty.handler.logging.LoggingHandler;
-import io.netty.util.internal.logging.InternalLoggerFactory;
-import io.netty.util.internal.logging.Log4J2LoggerFactory;
 
 public class HttpServer {
     // Constants
@@ -84,10 +82,6 @@
     private Throwable cause;
     private HttpServerConfig config;
 
-    static {
-        InternalLoggerFactory.setDefaultFactory(Log4J2LoggerFactory.INSTANCE);
-    }
-
     public HttpServer(EventLoopGroup bossGroup, EventLoopGroup workerGroup, int port, HttpServerConfig config) {
         this(bossGroup, workerGroup, new InetSocketAddress(port), config, null);
     }
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/WebManager.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/WebManager.java
index 55741e4..b8347ae 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/WebManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/WebManager.java
@@ -24,12 +24,18 @@
 
 import io.netty.channel.EventLoopGroup;
 import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.util.internal.logging.Log4J2LoggerFactory;
 
 public class WebManager {
     private final List<HttpServer> servers;
     private final EventLoopGroup bosses;
     private final EventLoopGroup workers;
 
+    static {
+        // bootstrap netty with log4j2 logging
+        io.netty.util.internal.logging.InternalLoggerFactory.setDefaultFactory(Log4J2LoggerFactory.INSTANCE);
+    }
+
     /**
      * Create a web manager with number of bosses = 1
      * and number of workers = MultithreadEventLoopGroup.DEFAULT_EVENT_LOOP_THREADS
diff --git a/hyracks-fullstack/hyracks/hyracks-server/pom.xml b/hyracks-fullstack/hyracks/hyracks-server/pom.xml
index 7e566d6..6c5639f 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-server/pom.xml
@@ -185,9 +185,5 @@
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-api</artifactId>
     </dependency>
-    <dependency>
-      <groupId>commons-io</groupId>
-      <artifactId>commons-io</artifactId>
-    </dependency>
   </dependencies>
 </project>
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/NCServiceIT.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/NCServiceIT.java
index 5cbc5b4..cd0359f 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/NCServiceIT.java
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/NCServiceIT.java
@@ -21,11 +21,8 @@
 import java.io.File;
 import java.io.IOException;
 import java.net.InetAddress;
-import java.nio.file.Files;
-import java.nio.file.StandardOpenOption;
 import java.util.Iterator;
 
-import org.apache.commons.io.FileUtils;
 import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
 import org.apache.http.client.HttpClient;
@@ -58,13 +55,9 @@
     @BeforeClass
     public static void setUp() throws Exception {
         cluster = new HyracksVirtualCluster(new File(APP_HOME), null);
-        File tempConf = new File(TARGET_DIR, "cc.conf");
-        FileUtils.copyFile(new File(RESOURCE_DIR, "cc.conf"), tempConf);
-        Files.write(tempConf.toPath(), ("log.dir: " + LOG_DIR).getBytes(), StandardOpenOption.APPEND);
-        File log4jPath = new File(FileUtil.joinPath("..", "..", "src", "test", "resources", "log4j2-hyracks-test.xml"));
 
-        cluster.addNCService(new File(RESOURCE_DIR, "nc-red.conf"), new File(LOG_DIR, "nc-red.log"), log4jPath);
-        cluster.addNCService(new File(RESOURCE_DIR, "nc-blue.conf"), new File(LOG_DIR, "nc-blue.log"), log4jPath);
+        cluster.addNCService(new File(RESOURCE_DIR, "nc-red.conf"), null);
+        cluster.addNCService(new File(RESOURCE_DIR, "nc-blue.conf"), null);
 
         try {
             Thread.sleep(2000);
@@ -72,7 +65,7 @@
         }
 
         // Start CC
-        cluster.start(tempConf, new File(LOG_DIR, "cc.log"), log4jPath);
+        cluster.start(new File(RESOURCE_DIR, "cc.conf"), null);
 
         try {
             Thread.sleep(10000);
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksCCProcess.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksCCProcess.java
index a79d033..0b529fb 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksCCProcess.java
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksCCProcess.java
@@ -25,18 +25,14 @@
 
 public class HyracksCCProcess extends HyracksServerProcess {
 
-    public HyracksCCProcess(File configFile, File logFile, File appHome, File workingDir) {
+    HyracksCCProcess(File configFile, File logFile, File appHome, File workingDir) {
+        super(" cc");
         this.configFile = configFile;
         this.logFile = logFile;
         this.appHome = appHome;
         this.workingDir = workingDir;
     }
 
-    public HyracksCCProcess(File configFile, File logFile, File appHome, File workingDir, File log4jPath) {
-        this(configFile, logFile, appHome, workingDir);
-        args.add("-Dlog4j.configurationFile=file://" + log4jPath.getAbsolutePath());
-    }
-
     @Override
     protected String getMainClassName() {
         return CCDriver.class.getName();
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksNCServiceProcess.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksNCServiceProcess.java
index d0e0244..958eb9a 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksNCServiceProcess.java
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksNCServiceProcess.java
@@ -20,23 +20,21 @@
 
 import java.io.File;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.hyracks.control.nc.service.NCService;
 
 public class HyracksNCServiceProcess extends HyracksServerProcess {
+    private static final AtomicInteger ncServiceCounter = new AtomicInteger();
 
-    public HyracksNCServiceProcess(File configFile, File logFile, File appHome, File workingDir) {
+    HyracksNCServiceProcess(File configFile, File logFile, File appHome, File workingDir) {
+        super("nc" + ncServiceCounter.incrementAndGet());
         this.configFile = configFile;
         this.logFile = logFile;
         this.appHome = appHome;
         this.workingDir = workingDir;
     }
 
-    public HyracksNCServiceProcess(File configFile, File logFile, File appHome, File workingDir, File log4jPath) {
-        this(configFile, logFile, appHome, workingDir);
-        args.add("-Dlog4j.configurationFile=file://" + log4jPath.getAbsolutePath());
-    }
-
     @Override
     protected String getMainClassName() {
         return NCService.class.getName();
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksServerProcess.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksServerProcess.java
index a8c363b..59ab4f7 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksServerProcess.java
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksServerProcess.java
@@ -18,9 +18,12 @@
  */
 package org.apache.hyracks.test.server.process;
 
+import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -32,14 +35,21 @@
 abstract class HyracksServerProcess {
     private static final Logger LOGGER = LogManager.getLogger();
 
+    protected final String processName;
     protected Process process;
+    protected Thread pipeThread;
     protected File configFile = null;
     protected File logFile = null;
     protected File appHome = null;
     protected File workingDir = null;
     protected List<String> args = new ArrayList<>();
 
+    protected HyracksServerProcess(String processName) {
+        this.processName = processName;
+    }
+
     public void start() throws IOException {
+
         String[] cmd = buildCommand();
         if (LOGGER.isInfoEnabled()) {
             LOGGER.info("Starting command: " + Arrays.toString(cmd));
@@ -47,34 +57,45 @@
 
         ProcessBuilder pb = new ProcessBuilder(cmd);
         pb.redirectErrorStream(true);
+        pb.directory(workingDir);
         if (logFile != null) {
-            if (LOGGER.isInfoEnabled()) {
-                LOGGER.info("Logging to: " + logFile.getCanonicalPath());
-            }
+            LOGGER.info("Logging to: " + logFile.getCanonicalPath());
             logFile.getParentFile().mkdirs();
             try (FileWriter writer = new FileWriter(logFile, true)) {
                 writer.write("---------------------\n");
             }
             pb.redirectOutput(ProcessBuilder.Redirect.appendTo(logFile));
+            process = pb.start();
         } else {
-            if (LOGGER.isInfoEnabled()) {
-                LOGGER.info("Logfile not set, subprocess will output to stdout");
-            }
+            pb.redirectOutput(ProcessBuilder.Redirect.PIPE);
+            process = pb.start();
+            pipeThread = new Thread(() -> {
+                try (BufferedReader reader =
+                        new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) {
+                    String line;
+                    while ((line = reader.readLine()) != null) {
+                        System.out.println(processName + ": " + line);
+                    }
+                } catch (IOException e) {
+                    LOGGER.debug("exception reading process pipe", e);
+                }
+            });
+            pipeThread.start();
         }
-        pb.directory(workingDir);
-        process = pb.start();
     }
 
     public void stop() {
         process.destroy();
         try {
             boolean success = process.waitFor(30, TimeUnit.SECONDS);
-            if (LOGGER.isWarnEnabled()) {
-                LOGGER.warn("Killing unresponsive NC Process");
-            }
             if (!success) {
+                LOGGER.warn("Killing unresponsive NC Process");
                 process.destroyForcibly();
             }
+            if (pipeThread != null) {
+                pipeThread.interrupt();
+                pipeThread.join();
+            }
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
         }
@@ -106,7 +127,7 @@
             cList.add(configFile.getAbsolutePath());
         }
         addCmdLineArgs(cList);
-        return cList.toArray(new String[cList.size()]);
+        return cList.toArray(new String[0]);
     }
 
     protected void addJvmArgs(List<String> cList) {
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksVirtualCluster.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksVirtualCluster.java
index 6c77628..062d429 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksVirtualCluster.java
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksVirtualCluster.java
@@ -65,22 +65,6 @@
     }
 
     /**
-     * Creates and starts an NCService.
-     *
-     * @param configFile
-     *            - full path to an ncservice.conf. May be null to accept all defaults.
-     * @throws IOException
-     *             - if there are errors starting the process.
-     */
-    public HyracksNCServiceProcess addNCService(File configFile, File logFile, File log4jConfig) throws IOException {
-        HyracksNCServiceProcess proc =
-                new HyracksNCServiceProcess(configFile, logFile, appHome, workingDir, log4jConfig);
-        proc.start();
-        ncProcs.add(proc);
-        return proc;
-    }
-
-    /**
      * Starts the CC, initializing the cluster. Expects that any NCs referenced
      * in the cluster configuration have already been started with addNCService().
      *
@@ -97,22 +81,6 @@
     }
 
     /**
-     * Starts the CC, initializing the cluster. Expects that any NCs referenced
-     * in the cluster configuration have already been started with addNCService().
-     *
-     * @param ccConfigFile
-     *            - full path to a cluster conf file. May be null to accept all
-     *            defaults, although this is seldom useful since there are no NCs.
-     * @throws IOException
-     *             - if there are errors starting the process.
-     */
-    public HyracksCCProcess start(File ccConfigFile, File logFile, File log4jConfig) throws IOException {
-        ccProc = new HyracksCCProcess(ccConfigFile, logFile, appHome, workingDir, log4jConfig);
-        ccProc.start();
-        return ccProc;
-    }
-
-    /**
      * Stops all processes in the cluster.
      * QQQ Someday this should probably do a graceful stop of NCs rather than
      * killing the NCService.
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf
index 9c80c7d..9b1a1cd 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf
@@ -30,3 +30,4 @@
 console.listen.port = 12345
 
 [common]
+log.dir=target/NCServiceIT
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-blue.conf b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-blue.conf
index 6eb38dd..baccd46 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-blue.conf
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-blue.conf
@@ -18,3 +18,4 @@
 [ncservice]
 address=127.0.0.1
 port=9091
+logdir=-
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-red.conf b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-red.conf
index 286bd32..7616b37 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-red.conf
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-red.conf
@@ -18,3 +18,4 @@
 [ncservice]
 address=127.0.0.1
 port=9090
+logdir=-
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java
index 46e77e3..5a8edbd 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java
@@ -21,7 +21,6 @@
 import java.lang.management.ManagementFactory;
 import java.lang.management.RuntimeMXBean;
 import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 
 import org.apache.logging.log4j.Level;
@@ -35,20 +34,11 @@
     private PidHelper() {
     }
 
-    public static long getPid() {
+    public static int getPid() {
         return getPid(ManagementFactory.getRuntimeMXBean());
     }
 
-    public static long getPid(RuntimeMXBean runtimeMXBean) {
-        // TODO: replace with direct invoke of getPid() once compatibility is at JDK 10 or higher
-        try {
-            Method getPidMethod = runtimeMXBean.getClass().getMethod("getPid");
-            return (Long) getPidMethod.invoke(runtimeMXBean);
-        } catch (NoSuchMethodException e) {
-            LOGGER.debug("ignoring exception trying to find getPid() (expected pre-JDK 10)", e);
-        } catch (IllegalAccessException | InvocationTargetException e) {
-            LOGGER.debug("ignoring exception trying to execute getPid()", e);
-        }
+    public static int getPid(RuntimeMXBean runtimeMXBean) {
         try {
             Field jvmField = runtimeMXBean.getClass().getDeclaredField("jvm");
             jvmField.setAccessible(true);
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java
index b744198..b5fe3d3 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java
@@ -28,14 +28,14 @@
     public final long cat;
     public final ITracer.Phase ph;
     public final long ts;
-    public final long pid;
+    public final int pid;
     public final long tid;
     public final ITracer.Scope scope;
     public final String args;
     public final TraceCategoryRegistry registry;
 
-    private Event(String name, long cat, ITracer.Phase ph, long ts, long pid, long tid, ITracer.Scope scope,
-            String args, TraceCategoryRegistry registry) {
+    private Event(String name, long cat, ITracer.Phase ph, long ts, int pid, long tid, ITracer.Scope scope, String args,
+            TraceCategoryRegistry registry) {
         this.name = name;
         this.cat = cat;
         this.ph = ph;
@@ -51,7 +51,7 @@
         return (System.nanoTime() - NANOTIME_DELTA_TO_EPOCH) / 1000;
     }
 
-    public static Event create(String name, long cat, ITracer.Phase ph, long pid, long tid, ITracer.Scope scope,
+    public static Event create(String name, long cat, ITracer.Phase ph, int pid, long tid, ITracer.Scope scope,
             String args, TraceCategoryRegistry registry) {
         return new Event(name, cat, ph, timestamp(), pid, tid, scope, args, registry);
     }
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java
index c1c38cf..9019fdf 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java
@@ -44,7 +44,7 @@
     private long categories;
     private final TraceCategoryRegistry registry;
 
-    private static final long pid = PidHelper.getPid();
+    private static final int pid = PidHelper.getPid();
 
     public Tracer(String name, long categories, TraceCategoryRegistry registry) {
         final String traceLoggerName = Tracer.class.getName() + ".Traces." + name;