ASTERIXDB-1428: Config-management improvements for AsterixDB
1. Added ability for NC Service to log NC output to file
2. Improve config communication from CC to NC
3. Allow overriding NC command name from CC config (and removed said
option from NC Service config as that's the wrong place to manage it)
4. Improve finding NC command (don't depend on PATH)
5. Create asterixncservice.bat script for Windows
6. Fixed NCServiceIT accordingly
Change-Id: I0f2ad32d489ffc27adbb06aebcc1f22a9fcf784e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/887
Reviewed-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <imaxon@apache.org>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>
diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml
index f22049b..812fd59 100644
--- a/asterixdb/asterix-server/pom.xml
+++ b/asterixdb/asterix-server/pom.xml
@@ -105,6 +105,13 @@
<commandLineArgument>org.apache.asterix.hyracks.bootstrap.NCApplicationEntryPoint</commandLineArgument>
</commandLineArguments>
</daemon>
+ <daemon>
+ <id>asterixncservice</id>
+ <mainClass>org.apache.hyracks.control.nc.service.NCService</mainClass>
+ <platforms>
+ <platform>booter-windows</platform>
+ </platforms>
+ </daemon>
</daemons>
</configuration>
<executions>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TriggerNCWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TriggerNCWork.java
index 75d4dd2..ee79d38 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TriggerNCWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TriggerNCWork.java
@@ -104,14 +104,18 @@
*/
String serializeIni(Ini ccini) throws IOException {
Ini ini = new Ini();
+
+ // First copy the global [nc] section to a new section named for
+ // *this* NC, so that those values serve as defaults.
String ncsection = "nc/" + ncId;
+ copyIniSection(ccini, "nc", ini, ncsection);
+ // Now copy all sections to their same name in the derived config.
for (String section : ccini.keySet()) {
- if (section.equals(ncsection)) {
- copyIniSection(ccini, ncsection, ini, "localnc");
- ini.put("localnc", "id", ncId);
- }
copyIniSection(ccini, section, ini, section);
}
+ // Finally insert *this* NC's name into localnc section - this is a fixed
+ // entry point so that NCs can determine where all their config is.
+ ini.put("localnc", "id", ncId);
StringWriter iniString = new StringWriter();
ini.store(iniString);
if (LOGGER.isLoggable(Level.FINE)) {
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
index 4240e3a..b408083 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
@@ -124,33 +124,36 @@
// that logic really should be handled by the ini file sent from the CC
ccHost = IniUtils.getString(ini, "cc", "cluster.address", ccHost);
ccPort = IniUtils.getInt(ini, "cc", "cluster.port", ccPort);
+
+ // Get ID of *this* NC
nodeId = IniUtils.getString(ini, "localnc", "id", nodeId);
+ String nodeSection = "nc/" + nodeId;
// Network ports
+ ipAddress = IniUtils.getString(ini, nodeSection, "address", ipAddress);
- ipAddress = IniUtils.getString(ini, "localnc", "address", ipAddress);
+ clusterNetIPAddress = IniUtils.getString(ini, nodeSection, "cluster.address", clusterNetIPAddress);
+ clusterNetPort = IniUtils.getInt(ini, nodeSection, "cluster.port", clusterNetPort);
+ dataIPAddress = IniUtils.getString(ini, nodeSection, "data.address", dataIPAddress);
+ dataPort = IniUtils.getInt(ini, nodeSection, "data.port", dataPort);
+ resultIPAddress = IniUtils.getString(ini, nodeSection, "result.address", resultIPAddress);
+ resultPort = IniUtils.getInt(ini, nodeSection, "result.port", resultPort);
- clusterNetIPAddress = IniUtils.getString(ini, "localnc", "cluster.address", clusterNetIPAddress);
- clusterNetPort = IniUtils.getInt(ini, "localnc", "cluster.port", clusterNetPort);
- dataIPAddress = IniUtils.getString(ini, "localnc", "data.address", dataIPAddress);
- dataPort = IniUtils.getInt(ini, "localnc", "data.port", dataPort);
- resultIPAddress = IniUtils.getString(ini, "localnc", "result.address", resultIPAddress);
- resultPort = IniUtils.getInt(ini, "localnc", "result.port", resultPort);
+ clusterNetPublicIPAddress = IniUtils.getString(
+ ini, nodeSection, "public.cluster.address", clusterNetPublicIPAddress);
+ clusterNetPublicPort = IniUtils.getInt(ini, nodeSection, "public.cluster.port", clusterNetPublicPort);
+ dataPublicIPAddress = IniUtils.getString(ini, nodeSection, "public.data.address", dataPublicIPAddress);
+ dataPublicPort = IniUtils.getInt(ini, nodeSection, "public.data.port", dataPublicPort);
+ resultPublicIPAddress = IniUtils.getString(ini, nodeSection, "public.result.address", resultPublicIPAddress);
+ resultPublicPort = IniUtils.getInt(ini, nodeSection, "public.result.port", resultPublicPort);
- clusterNetPublicIPAddress = IniUtils.getString(ini, "localnc", "public.cluster.address", clusterNetPublicIPAddress);
- clusterNetPublicPort = IniUtils.getInt(ini, "localnc", "public.cluster.port", clusterNetPublicPort);
- dataPublicIPAddress = IniUtils.getString(ini, "localnc", "public.data.address", dataPublicIPAddress);
- dataPublicPort = IniUtils.getInt(ini, "localnc", "public.data.port", dataPublicPort);
- resultPublicIPAddress = IniUtils.getString(ini, "localnc", "public.result.address", resultPublicIPAddress);
- resultPublicPort = IniUtils.getInt(ini, "localnc", "public.result.port", resultPublicPort);
-
- retries = IniUtils.getInt(ini, "localnc", "retries", retries);
+ retries = IniUtils.getInt(ini, nodeSection, "retries", retries);
// Directories
- ioDevices = IniUtils.getString(ini, "localnc", "iodevices", ioDevices);
+ ioDevices = IniUtils.getString(ini, nodeSection, "iodevices", ioDevices);
// Hyracks client entrypoint
- appNCMainClass = IniUtils.getString(ini, "localnc", "app.class", appNCMainClass);
+ appNCMainClass = IniUtils.getString(ini, nodeSection, "app.class", appNCMainClass);
}
/*
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java
index df92d1a..ea220c1 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java
@@ -50,6 +50,16 @@
private static Ini ini = new Ini();
/**
+ * ID of *this* NC
+ */
+ private static String ncId = "";
+
+ /**
+ * The Ini section representing *this* NC
+ */
+ private static String nodeSection = null;
+
+ /**
* The NCServiceConfig
*/
private static NCServiceConfig config;
@@ -73,12 +83,22 @@
private static List<String> buildCommand() throws IOException {
List<String> cList = new ArrayList<String>();
+
+ // Find the command to run. For now, we allow overriding the name, but
+ // still assume it's located in the bin/ directory of the deployment.
+ // Even this is likely more configurability than we need.
+ String command = getStringINIOpt(ini, nodeSection, "command", "hyracksnc");
+ // app.home is specified by the Maven appassembler plugin. If it isn't set,
+ // fall back to user's home dir. Again this is likely more flexibility
+ // than we need.
+ String apphome = System.getProperty("app.home", System.getProperty("user.home"));
+ String path = apphome + File.separator + "bin" + File.separator;
if (SystemUtils.IS_OS_WINDOWS) {
- cList.add(config.command + ".bat");
+ cList.add(path + command + ".bat");
+ } else {
+ cList.add(path + command);
}
- else {
- cList.add(config.command);
- }
+
cList.add("-config-file");
// Store the Ini file from the CC locally so NCConfig can read it.
// QQQ should arrange to delete this when done
@@ -92,7 +112,7 @@
if (env.containsKey("JAVA_OPTS")) {
return;
}
- String jvmargs = getStringINIOpt(ini, "localnc", "jvm.args", "-Xmx1536m");
+ String jvmargs = getStringINIOpt(ini, nodeSection, "jvm.args", "-Xmx1536m");
env.put("JAVA_OPTS", jvmargs);
}
@@ -114,6 +134,20 @@
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Launching NCDriver process");
}
+
+ // Logfile
+ if (! "-".equals(config.logdir)) {
+ pb.redirectErrorStream(true);
+ File log = new File(config.logdir);
+ if (! log.mkdirs()) {
+ throw new IOException(config.logdir + ": cannot create");
+ }
+ File logfile = new File(config.logdir, "nc-" + ncId + ".log");
+ pb.redirectOutput(ProcessBuilder.Redirect.appendTo(logfile));
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Logging to " + logfile.getCanonicalPath());
+ }
+ }
proc = pb.start();
boolean waiting = true;
@@ -155,6 +189,8 @@
}
String iniString = ois.readUTF();
ini = new Ini(new StringReader(iniString));
+ ncId = getStringINIOpt(ini, "localnc", "id", "");
+ nodeSection = "nc/" + ncId;
return launchNCProcess();
} catch (Exception e) {
LOGGER.log(Level.SEVERE, "Error decoding connection from server", e);
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCServiceConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCServiceConfig.java
index bc982f3..91c9f6d 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCServiceConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCServiceConfig.java
@@ -22,8 +22,8 @@
import org.ini4j.Ini;
import org.kohsuke.args4j.Option;
+import java.io.File;
import java.io.IOException;
-import java.net.InetAddress;
/**
* Command-line arguments for NC Service.
@@ -47,9 +47,9 @@
usage = "Port to listen on for connections from CC (default: 9090)")
public int port = 9090;
- @Option(name = "-command", required = false,
- usage = "NC command to run (default: 'hyracksnc' on PATH)")
- public String command = "hyracksnc";
+ @Option(name = "-logdir", required = false,
+ usage = "Directory to log NC output ('-' for stdout of NC service; default: $app.home/logs)")
+ public String logdir = null;
private Ini ini = null;
@@ -61,6 +61,7 @@
ini = IniUtils.loadINIFile(configFile);
address = IniUtils.getString(ini, "ncservice", "address", address);
port = IniUtils.getInt(ini, "ncservice", "port", port);
+ logdir = IniUtils.getString(ini, "ncservice", "logdir", logdir);
}
/**
@@ -73,6 +74,8 @@
if (configFile != null) {
loadINIFile();
}
- // No defaults necessary beyond the static ones for this config
+ if (logdir == null) {
+ logdir = System.getProperty("app.home", System.getProperty("user.home")) + File.separator + "logs";
+ }
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java
index 7f431f6..0cc1031 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/server/test/NCServiceIT.java
@@ -37,24 +37,24 @@
public class NCServiceIT {
+ private static final String TARGET_DIR = StringUtils
+ .join(new String[]{System.getProperty("basedir"), "target"}, File.separator);
+ private static final String LOG_DIR = StringUtils
+ .join(new String[]{TARGET_DIR, "surefire-reports"}, File.separator);
private static final String RESOURCE_DIR = StringUtils
- .join(new String[]{System.getProperty("user.dir"), "src", "test", "resources", "NCServiceIT"},
- File.separator);
+ .join(new String[]{TARGET_DIR, "test-classes", "NCServiceIT"}, File.separator);
private static final String APP_DIR = StringUtils
- .join(new String[]{System.getProperty("user.dir"), "target", "appassembler", "bin"},
- File.separator);
+ .join(new String[]{TARGET_DIR, "appassembler", "bin"}, File.separator);
private static final Logger LOGGER = Logger.getLogger(NCServiceIT.class.getName());
private static List<Process> procs = new ArrayList<>();
@BeforeClass
public static void setUp() throws Exception {
// Start two NC Services - don't read their output as they don't terminate
- procs.add(invoke(APP_DIR + File.separator + "hyracksncservice",
- "-config-file", RESOURCE_DIR + File.separator + "nc-red.conf",
- "-command", APP_DIR + File.separator + "hyracksnc"));
- procs.add(invoke(APP_DIR + File.separator + "hyracksncservice",
- "-config-file", RESOURCE_DIR + File.separator + "nc-blue.conf",
- "-command", APP_DIR + File.separator + "hyracksnc"));
+ procs.add(invoke("nc-red.log", APP_DIR + File.separator + "hyracksncservice",
+ "-config-file", RESOURCE_DIR + File.separator + "nc-red.conf"));
+ procs.add(invoke("nc-blue.log", APP_DIR + File.separator + "hyracksncservice",
+ "-config-file", RESOURCE_DIR + File.separator + "nc-blue.conf"));
try {
Thread.sleep(2000);
}
@@ -62,7 +62,7 @@
}
// Start CC
- procs.add(invoke(APP_DIR + File.separator + "hyrackscc",
+ procs.add(invoke("cc.log", APP_DIR + File.separator + "hyrackscc",
"-config-file", RESOURCE_DIR + File.separator + "cc.conf"));
try {
Thread.sleep(10000);
@@ -97,9 +97,12 @@
}
}
- private static Process invoke(String... args) throws Exception {
+ private static Process invoke(String logfile, String... args) throws Exception {
ProcessBuilder pb = new ProcessBuilder(args);
pb.redirectErrorStream(true);
+ File log = new File(LOG_DIR, logfile);
+ log.delete();
+ pb.redirectOutput(ProcessBuilder.Redirect.appendTo(log));
Process p = pb.start();
return p;
}
@@ -112,7 +115,7 @@
JSONObject result = new JSONObject(response);
JSONArray nodes = result.getJSONArray("result");
int numNodes = nodes.length();
- Assert.assertEquals("Wrong number of nodes!", numNodes, 2);
+ Assert.assertEquals("Wrong number of nodes!", 2, numNodes);
for (int i = 0; i < nodes.length(); i++) {
JSONObject node = nodes.getJSONObject(i);
String id = node.getString("node-id");
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-blue.conf b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-blue.conf
index d070b59..1cd1666 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-blue.conf
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-blue.conf
@@ -1,3 +1,5 @@
[ncservice]
address=127.0.0.1
port=9091
+logdir=-
+
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-red.conf b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-red.conf
index 58a8f1d..74b49b0 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-red.conf
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-red.conf
@@ -1,4 +1,5 @@
[ncservice]
address=127.0.0.1
port=9090
+logdir=-