ASTERIXDB-1482: Added NCServiceExecutionIT, HyracksVirtualCluster.
NCServiceExecutionIT runs all execution tests against a local cluster
managed by the NCService deployment framework.
HyracksVirtualCluster offers programmatic NCService deployment
control along with improved HyracksNCProcess/HyracksCCProcess.
Further fixes and improvements:
1. Fix handling of iodevices/storagedir (ASTERIXDB-1482)
2. Proper handling of [nc] default section in all cases
3. Ensure asterixnc, etc. scripts are executable
4. Consolidate Ini handling
5. Pruned some dead code, including VirtualClusterDriver
6. A bit of refactoring and extended commenting
Change-Id: If3eb450782a595cf85d04a2c2e9cc732564e65e6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/958
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <imaxon@apache.org>
diff --git a/hyracks-fullstack/hyracks/hyracks-server/pom.xml b/hyracks-fullstack/hyracks/hyracks-server/pom.xml
index 3bda1d3..52958f8 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-server/pom.xml
@@ -86,10 +86,6 @@
<mainClass>org.apache.hyracks.control.nc.service.NCService</mainClass>
<name>hyracksncservice</name>
</program>
- <program>
- <mainClass>org.apache.hyracks.server.drivers.VirtualClusterDriver</mainClass>
- <name>hyracks-virtual-cluster</name>
- </program>
</programs>
<repositoryLayout>flat</repositoryLayout>
<repositoryName>lib</repositoryName>
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/drivers/VirtualClusterDriver.java b/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/drivers/VirtualClusterDriver.java
deleted file mode 100644
index 41c14a7..0000000
--- a/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/drivers/VirtualClusterDriver.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.server.drivers;
-
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-
-import org.apache.hyracks.control.common.controllers.CCConfig;
-import org.apache.hyracks.control.common.controllers.NCConfig;
-import org.apache.hyracks.server.process.HyracksCCProcess;
-import org.apache.hyracks.server.process.HyracksNCProcess;
-
-public class VirtualClusterDriver {
- private static class Options {
- @Option(name = "-n", required = false, usage = "Number of node controllers (default: 2)")
- public int n = 2;
-
- @Option(name = "-cc-client-net-port", required = false, usage = "CC Port (default: 1098)")
- public int ccClientNetPort = 1098;
-
- @Option(name = "-cc-cluster-net-port", required = false, usage = "CC Port (default: 1099)")
- public int ccClusterNetPort = 1099;
-
- @Option(name = "-cc-http-port", required = false, usage = "CC Port (default: 16001)")
- public int ccHttpPort = 16001;
- }
-
- public static void main(String[] args) throws Exception {
- Options options = new Options();
- CmdLineParser cp = new CmdLineParser(options);
- try {
- cp.parseArgument(args);
- } catch (Exception e) {
- System.err.println(e.getMessage());
- cp.printUsage(System.err);
- return;
- }
-
- CCConfig ccConfig = new CCConfig();
- ccConfig.clusterNetIpAddress = "127.0.0.1";
- ccConfig.clusterNetPort = options.ccClusterNetPort;
- ccConfig.clientNetIpAddress = "127.0.0.1";
- ccConfig.clientNetPort = options.ccClientNetPort;
- ccConfig.httpPort = options.ccHttpPort;
- HyracksCCProcess ccp = new HyracksCCProcess(ccConfig);
- ccp.start();
-
- Thread.sleep(5000);
-
- HyracksNCProcess ncps[] = new HyracksNCProcess[options.n];
- for (int i = 0; i < options.n; ++i) {
- NCConfig ncConfig = new NCConfig();
- ncConfig.ccHost = "127.0.0.1";
- ncConfig.ccPort = options.ccClusterNetPort;
- ncConfig.clusterNetIPAddress = "127.0.0.1";
- ncConfig.nodeId = "nc" + i;
- ncConfig.dataIPAddress = "127.0.0.1";
- ncps[i] = new HyracksNCProcess(ncConfig);
- ncps[i].start();
- }
-
- while (true) {
- Thread.sleep(10000);
- }
- }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksCCProcess.java b/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksCCProcess.java
index d0d8d63..4a70120 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksCCProcess.java
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksCCProcess.java
@@ -18,25 +18,28 @@
*/
package org.apache.hyracks.server.process;
+import org.apache.hyracks.control.cc.CCDriver;
+
+import java.io.File;
import java.util.List;
-import org.apache.hyracks.control.cc.CCDriver;
-import org.apache.hyracks.control.common.controllers.CCConfig;
-
public class HyracksCCProcess extends HyracksServerProcess {
- private CCConfig config;
- public HyracksCCProcess(CCConfig config) {
- this.config = config;
- }
-
- @Override
- protected void addCmdLineArgs(List<String> cList) {
- config.toCommandLine(cList);
+ public HyracksCCProcess(File configFile, File logFile, File appHome, File workingDir) {
+ this.configFile = configFile;
+ this.logFile = logFile;
+ this.appHome = appHome;
+ this.workingDir = workingDir;
}
@Override
protected String getMainClassName() {
return CCDriver.class.getName();
}
+
+ @Override
+ protected void addJvmArgs(List<String> cList) {
+ // CC needs more than default memory
+ cList.add("-Xmx1024m");
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksNCProcess.java b/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksNCProcess.java
index c4517e6..8bc1694 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksNCProcess.java
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksNCProcess.java
@@ -18,25 +18,28 @@
*/
package org.apache.hyracks.server.process;
+import org.apache.hyracks.control.nc.service.NCService;
+
+import java.io.File;
import java.util.List;
-import org.apache.hyracks.control.common.controllers.NCConfig;
-import org.apache.hyracks.control.nc.NCDriver;
-
public class HyracksNCProcess extends HyracksServerProcess {
- private NCConfig config;
- public HyracksNCProcess(NCConfig config) {
- this.config = config;
- }
-
- @Override
- protected void addCmdLineArgs(List<String> cList) {
- config.toCommandLine(cList);
+ public HyracksNCProcess(File configFile, File logFile, File appHome, File workingDir) {
+ this.configFile = configFile;
+ this.logFile = logFile;
+ this.appHome = appHome;
+ this.workingDir = workingDir;
}
@Override
protected String getMainClassName() {
- return NCDriver.class.getName();
+ return NCService.class.getName();
+ }
+
+ @Override
+ protected void addJvmArgs(List<String> cList) {
+ // NCService needs little memory
+ cList.add("-Xmx128m");
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksServerProcess.java b/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksServerProcess.java
index 9dec0ec..13cb445 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksServerProcess.java
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksServerProcess.java
@@ -29,53 +29,69 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-public abstract class HyracksServerProcess {
+abstract class HyracksServerProcess {
private static final Logger LOGGER = Logger.getLogger(HyracksServerProcess.class.getName());
protected Process process;
+ protected File configFile = null;
+ protected File logFile = null;
+ protected File appHome = null;
+ protected File workingDir = null;
public void start() throws IOException {
String[] cmd = buildCommand();
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Starting command: " + Arrays.toString(cmd));
}
- process = Runtime.getRuntime().exec(cmd, null, null);
- dump(process.getInputStream());
- dump(process.getErrorStream());
+
+ ProcessBuilder pb = new ProcessBuilder(cmd);
+ pb.redirectErrorStream(true);
+ if (logFile != null) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Logging to: " + logFile.getCanonicalPath());
+ }
+ logFile.getParentFile().mkdirs();
+ logFile.delete();
+ pb.redirectOutput(ProcessBuilder.Redirect.appendTo(logFile));
+ } else {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Logfile not set, subprocess will output to stdout");
+ }
+ }
+ pb.directory(workingDir);
+ process = pb.start();
}
- private void dump(InputStream input) {
- final int streamBufferSize = 1000;
- final Reader in = new InputStreamReader(input);
- new Thread(new Runnable() {
- public void run() {
- try {
- char[] chars = new char[streamBufferSize];
- int c;
- while ((c = in.read(chars)) != -1) {
- if (c > 0) {
- System.out.print(String.valueOf(chars, 0, c));
- }
- }
- } catch (IOException e) {
- }
- }
- }).start();
+ public void stop() {
+ process.destroy();
+ try {
+ process.waitFor();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
}
private String[] buildCommand() {
List<String> cList = new ArrayList<String>();
cList.add(getJavaCommand());
- cList.add("-Dbasedir=" + System.getProperty("basedir"));
- cList.add("-Djava.rmi.server.hostname=127.0.0.1");
+ addJvmArgs(cList);
+ cList.add("-Dapp.home=" + appHome.getAbsolutePath());
cList.add("-classpath");
cList.add(getClasspath());
cList.add(getMainClassName());
+ if (configFile != null) {
+ cList.add("-config-file");
+ cList.add(configFile.getAbsolutePath());
+ }
addCmdLineArgs(cList);
return cList.toArray(new String[cList.size()]);
}
- protected abstract void addCmdLineArgs(List<String> cList);
+ protected void addJvmArgs(List<String> cList) {
+ }
+
+ protected void addCmdLineArgs(List<String> cList) {
+ }
protected abstract String getMainClassName();
@@ -83,7 +99,7 @@
return System.getProperty("java.class.path");
}
- protected final String getJavaCommand() {
+ private final String getJavaCommand() {
return System.getProperty("java.home") + File.separator + "bin" + File.separator + "java";
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksVirtualCluster.java b/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksVirtualCluster.java
new file mode 100644
index 0000000..f08bb43
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/main/java/org/apache/hyracks/server/process/HyracksVirtualCluster.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.server.process;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Starts a local hyracks-based cluster (NC and CC child processes).
+ */
+public class HyracksVirtualCluster {
+ private final File appHome;
+ private final File workingDir;
+ private List<HyracksNCProcess> ncProcs = new ArrayList<>(3);
+ private HyracksCCProcess ccProc = null;
+
+ /**
+ * Construct a Hyracks-based cluster.
+ * @param appHome - path to the installation root of the Hyracks application.
+ * At least bin/hyracksnc (or the equivalent NC script for
+ * the application) must exist in this directory.
+ * @param workingDir - directory to use as CWD for all child processes. May
+ * be null, in which case the CWD of the invoking process is used.
+ */
+ public HyracksVirtualCluster(File appHome, File workingDir) {
+ this.appHome = appHome;
+ this.workingDir = workingDir;
+ }
+
+ /**
+ * Creates and starts an NCService.
+ * @param configFile - full path to an ncservice.conf. May be null to accept all defaults.
+ * @throws IOException - if there are errors starting the process.
+ */
+ public void addNC(File configFile, File logFile) throws IOException {
+ HyracksNCProcess proc = new HyracksNCProcess(configFile, logFile, appHome, workingDir);
+ proc.start();
+ ncProcs.add(proc);
+ }
+
+ /**
+ * Starts the CC, initializing the cluster. Expects that any NCs referenced
+ * in the cluster configuration have already been started with addNC().
+ * @param ccConfigFile - full path to a cluster conf file. May be null to accept all
+ * defaults, although this is seldom useful since there are no NCs.
+ * @throws IOException - if there are errors starting the process.
+ */
+ public void start(File ccConfigFile, File logFile) throws IOException {
+ ccProc = new HyracksCCProcess(ccConfigFile, logFile, appHome, workingDir);
+ ccProc.start();
+ }
+
+ /**
+ * Stops all processes in the cluster.
+ * QQQ Someday this should probably do a graceful stop of NCs rather than
+ * killing the NCService.
+ */
+ public void stop() {
+ ccProc.stop();
+ for (HyracksNCProcess proc : ncProcs) {
+ proc.stop();
+ }
+ }
+}
+
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java
index bd99c8c..9a231a0 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java
@@ -23,6 +23,7 @@
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hyracks.server.process.HyracksVirtualCluster;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.AfterClass;
@@ -30,6 +31,7 @@
import org.junit.Test;
import java.io.File;
+import java.io.IOException;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.List;
@@ -38,23 +40,29 @@
public class NCServiceIT {
private static final String TARGET_DIR = StringUtils
- .join(new String[]{System.getProperty("basedir"), "target"}, File.separator);
+ .join(new String[] { System.getProperty("basedir"), "target" }, File.separator);
private static final String LOG_DIR = StringUtils
- .join(new String[]{TARGET_DIR, "surefire-reports"}, File.separator);
+ .join(new String[] { TARGET_DIR, "failsafe-reports" }, File.separator);
private static final String RESOURCE_DIR = StringUtils
- .join(new String[]{TARGET_DIR, "test-classes", "NCServiceIT"}, File.separator);
- private static final String APP_DIR = StringUtils
- .join(new String[]{TARGET_DIR, "appassembler", "bin"}, File.separator);
+ .join(new String[] { TARGET_DIR, "test-classes", "NCServiceIT" }, File.separator);
+ private static final String APP_HOME = StringUtils
+ .join(new String[] { TARGET_DIR, "appassembler" }, File.separator);
private static final Logger LOGGER = Logger.getLogger(NCServiceIT.class.getName());
- private static List<Process> procs = new ArrayList<>();
+
+ private static HyracksVirtualCluster cluster = null;
@BeforeClass
public static void setUp() throws Exception {
- // Start two NC Services - don't read their output as they don't terminate
- procs.add(invoke("nc-red.log", APP_DIR + File.separator + "hyracksncservice",
- "-config-file", RESOURCE_DIR + File.separator + "nc-red.conf"));
- procs.add(invoke("nc-blue.log", APP_DIR + File.separator + "hyracksncservice",
- "-config-file", RESOURCE_DIR + File.separator + "nc-blue.conf"));
+ cluster = new HyracksVirtualCluster(new File(APP_HOME), null);
+ cluster.addNC(
+ new File(RESOURCE_DIR, "nc-red.conf"),
+ new File(LOG_DIR, "nc-red.log")
+ );
+ cluster.addNC(
+ new File(RESOURCE_DIR, "nc-blue.conf"),
+ new File(LOG_DIR, "nc-blue.log")
+ );
+
try {
Thread.sleep(2000);
}
@@ -62,8 +70,11 @@
}
// Start CC
- procs.add(invoke("cc.log", APP_DIR + File.separator + "hyrackscc",
- "-config-file", RESOURCE_DIR + File.separator + "cc.conf"));
+ cluster.start(
+ new File(RESOURCE_DIR, "cc.conf"),
+ new File(LOG_DIR, "cc.log")
+ );
+
try {
Thread.sleep(10000);
}
@@ -72,11 +83,8 @@
}
@AfterClass
- public static void tearDown() throws Exception {
- for (Process p : procs) {
- p.destroy();
- p.waitFor();
- }
+ public static void tearDown() throws IOException {
+ cluster.stop();
}
private static String getHttp(String url) throws Exception {
@@ -97,18 +105,6 @@
}
}
- private static Process invoke(String logfile, String... args) throws Exception {
- ProcessBuilder pb = new ProcessBuilder(args);
- pb.redirectErrorStream(true);
- File logDir = new File(LOG_DIR);
- logDir.mkdirs();
- File log = new File(logDir, logfile);
- log.delete();
- pb.redirectOutput(ProcessBuilder.Redirect.appendTo(log));
- Process p = pb.start();
- return p;
- }
-
@Test
public void IsNodelistCorrect() throws Exception {
// Ping the nodelist HTTP API
@@ -138,5 +134,4 @@
tearDown();
}
}
-
}
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/logging.properties b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/logging.properties
index c888bb1..e9f84796 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/logging.properties
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/logging.properties
@@ -46,8 +46,8 @@
# Note that the ConsoleHandler also has a separate level
# setting to limit messages printed to the console.
-.level= WARNING
-# .level= INFO
+# .level= WARNING
+.level= INFO
# .level= FINE
# .level = FINEST