Overhaul of Hyracks configuration management.
Includes Asterix changes to make use of new conf management as a
Hyracks application.
Change-Id: Ie3027c8c839f25ea858790bd3340187f4b11f212
Reviewed-on: https://asterix-gerrit.ics.uci.edu/336
Tested-by: Chris Hillery <ceej@lambda.nu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <michael.blow@couchbase.com>
Reviewed-by: Ian Maxon <imaxon@apache.org>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/pom.xml
index 912e447..d885b38 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/pom.xml
@@ -44,5 +44,10 @@
<type>jar</type>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.ini4j</groupId>
+ <artifactId>ini4j</artifactId>
+ <version>0.5.4</version>
+ </dependency>
</dependencies>
</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/application/ApplicationContext.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/application/ApplicationContext.java
index c4e9ea2..23d287c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/application/ApplicationContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/application/ApplicationContext.java
@@ -22,6 +22,7 @@
import java.io.Serializable;
import java.util.concurrent.ThreadFactory;
+import org.apache.hyracks.api.application.IApplicationConfig;
import org.apache.hyracks.api.application.IApplicationContext;
import org.apache.hyracks.api.job.IJobSerializerDeserializerContainer;
import org.apache.hyracks.api.job.JobSerializerDeserializerContainer;
@@ -32,6 +33,7 @@
protected ServerContext serverCtx;
protected Serializable distributedState;
protected IMessageBroker messageBroker;
+ protected final IApplicationConfig appConfig;
protected IJobSerializerDeserializerContainer jobSerDeContainer = new JobSerializerDeserializerContainer();
protected ThreadFactory threadFactory = new ThreadFactory() {
public Thread newThread(Runnable r) {
@@ -41,8 +43,9 @@
}
};
- public ApplicationContext(ServerContext serverCtx) throws IOException {
+ public ApplicationContext(ServerContext serverCtx, IApplicationConfig appConfig) {
this.serverCtx = serverCtx;
+ this.appConfig = appConfig;
}
@Override
@@ -74,4 +77,9 @@
public void setThreadFactory(ThreadFactory threadFactory) {
this.threadFactory = threadFactory;
}
-}
+
+ @Override
+ public IApplicationConfig getAppConfig() {
+ return appConfig;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/application/IniApplicationConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/application/IniApplicationConfig.java
new file mode 100644
index 0000000..3a8a2de
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/application/IniApplicationConfig.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.common.application;
+
+import org.apache.hyracks.api.application.IApplicationConfig;
+import org.ini4j.Ini;
+
+import java.util.Set;
+
+/**
+ * An implementation of IApplicationConfig which is backed by Ini4j.
+ */
+public class IniApplicationConfig implements IApplicationConfig {
+ private final Ini ini;
+
+ public IniApplicationConfig(Ini ini) {
+ if (ini != null) {
+ this.ini = ini;
+ } else {
+ this.ini = new Ini();
+ }
+ }
+
+ private <T> T getIniValue(String section, String key, T default_value, Class<T> clazz) {
+ T value = ini.get(section, key, clazz);
+ return (value != null) ? value : default_value;
+ }
+
+ @Override
+ public String getString(String section, String key) {
+ return getIniValue(section, key, null, String.class);
+ }
+
+ @Override
+ public String getString(String section, String key, String defaultValue) {
+ return getIniValue(section, key, defaultValue, String.class);
+ }
+
+ @Override
+ public int getInt(String section, String key) {
+ return getIniValue(section, key, 0, Integer.class);
+ }
+
+ @Override
+ public int getInt(String section, String key, int defaultValue) {
+ return getIniValue(section, key, defaultValue, Integer.class);
+ }
+
+ @Override
+ public long getLong(String section, String key) {
+ return getIniValue(section, key, (long) 0, Long.class);
+ }
+
+ @Override
+ public long getLong(String section, String key, long defaultValue) {
+ return getIniValue(section, key, defaultValue, Long.class);
+ }
+
+ @Override
+ public Set<String> getSections() {
+ return ini.keySet();
+ }
+
+ @Override
+ public Set<String> getKeys(String section) {
+ return ini.get(section).keySet();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java
index 4e7c394..a04d750 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java
@@ -19,20 +19,31 @@
package org.apache.hyracks.control.common.controllers;
import java.io.File;
+import java.io.IOException;
+import java.net.InetAddress;
import java.util.List;
+import org.apache.hyracks.api.application.IApplicationConfig;
+import org.apache.hyracks.control.common.application.IniApplicationConfig;
+import org.ini4j.Ini;
import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.Option;
import org.kohsuke.args4j.spi.StopOptionHandler;
public class CCConfig {
- @Option(name = "-client-net-ip-address", usage = "Sets the IP Address to listen for connections from clients", required = true)
+ @Option(name = "-address", usage = "IP Address for CC (default: localhost)", required = false)
+ public String ipAddress = InetAddress.getLoopbackAddress().getHostAddress();
+
+ @Option(name = "-client-net-ip-address", usage = "Sets the IP Address to listen for connections from clients (default: same as -address)", required = false)
public String clientNetIpAddress;
@Option(name = "-client-net-port", usage = "Sets the port to listen for connections from clients (default 1098)")
public int clientNetPort = 1098;
- @Option(name = "-cluster-net-ip-address", usage = "Sets the IP Address to listen for connections from", required = true)
+ // QQQ Note that clusterNetIpAddress is *not directly used* yet. Both
+ // the cluster listener and the web server listen on "all interfaces".
+ // This IP address is only used to instruct the NC on which IP to call in.
+ @Option(name = "-cluster-net-ip-address", usage = "Sets the IP Address to listen for connections from NCs (default: same as -address)", required = false)
public String clusterNetIpAddress;
@Option(name = "-cluster-net-port", usage = "Sets the port to listen for connections from node controllers (default 1099)")
@@ -71,10 +82,77 @@
@Option(name = "-app-cc-main-class", required = false, usage = "Application CC Main Class")
public String appCCMainClass = null;
+ @Option(name = "-config-file", usage = "Specify path to master configuration file (default: none)", required = false)
+ public String configFile = null;
+
@Argument
@Option(name = "--", handler = StopOptionHandler.class)
public List<String> appArgs;
+ private Ini ini = null;
+
+ private void loadINIFile() throws IOException {
+ // This method simply maps from the ini parameters to the CCConfig's fields.
+ // It does not apply defaults or any logic.
+ ini = IniUtils.loadINIFile(configFile);
+
+ ipAddress = IniUtils.getString(ini, "cc", "address", ipAddress);
+ clientNetIpAddress = IniUtils.getString(ini, "cc", "client.address", clientNetIpAddress);
+ clientNetPort = IniUtils.getInt(ini, "cc", "client.port", clientNetPort);
+ clusterNetIpAddress = IniUtils.getString(ini, "cc", "cluster.address", clusterNetIpAddress);
+ clusterNetPort = IniUtils.getInt(ini, "cc", "cluster.port", clusterNetPort);
+ httpPort = IniUtils.getInt(ini, "cc", "http.port", httpPort);
+ heartbeatPeriod = IniUtils.getInt(ini, "cc", "heartbeat.period", heartbeatPeriod);
+ maxHeartbeatLapsePeriods = IniUtils.getInt(ini, "cc", "heartbeat.maxlapse", maxHeartbeatLapsePeriods);
+ profileDumpPeriod = IniUtils.getInt(ini, "cc", "profiledump.period", profileDumpPeriod);
+ defaultMaxJobAttempts = IniUtils.getInt(ini, "cc", "job.defaultattempts", defaultMaxJobAttempts);
+ jobHistorySize = IniUtils.getInt(ini, "cc", "job.historysize", jobHistorySize);
+ resultTTL = IniUtils.getLong(ini, "cc", "results.ttl", resultTTL);
+ resultSweepThreshold = IniUtils.getLong(ini, "cc", "results.sweepthreshold", resultSweepThreshold);
+ ccRoot = IniUtils.getString(ini, "cc", "rootfolder", ccRoot);
+ // QQQ clusterTopologyDefinition is a "File"; should support verifying that the file
+ // exists, as @Option likely does
+ appCCMainClass = IniUtils.getString(ini, "cc", "app.class", appCCMainClass);
+ }
+
+ /**
+ * Once all @Option fields have been loaded from command-line or otherwise
+ * specified programmatically, call this method to:
+ * 1. Load options from a config file (as specified by -config-file)
+ * 2. Set default values for certain derived values, such as setting
+ * clusterNetIpAddress to ipAddress
+ */
+ public void loadConfigAndApplyDefaults() throws IOException {
+ if (configFile != null) {
+ loadINIFile();
+ // QQQ This way of passing overridden/defaulted values back into
+ // the ini feels clunky, and it's clearly incomplete
+ ini.add("cc", "cluster.address", clusterNetIpAddress);
+ ini.add("cc", "client.address", clientNetIpAddress);
+ }
+
+ // "address" is the default for all IP addresses
+ if (clusterNetIpAddress == null) clusterNetIpAddress = ipAddress;
+ if (clientNetIpAddress == null) clientNetIpAddress = ipAddress;
+ }
+
+ /**
+ * Returns the global config Ini file. Note: this will be null
+ * if -config-file wasn't specified.
+ */
+ public Ini getIni() {
+ return ini;
+ }
+
+ /**
+ * @return An IApplicationConfig representing this NCConfig.
+ * Note: Currently this only includes the values from the configuration
+ * file, not anything specified on the command-line. QQQ
+ */
+ public IApplicationConfig getAppConfig() {
+ return new IniApplicationConfig(ini);
+ }
+
public void toCommandLine(List<String> cList) {
cList.add("-client-net-ip-address");
cList.add(clientNetIpAddress);
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/IniUtils.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/IniUtils.java
new file mode 100644
index 0000000..9a5c9a0
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/IniUtils.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.common.controllers;
+
+import org.ini4j.Ini;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * Some utility functions for reading Ini4j objects with default values.
+ */
+public class IniUtils {
+ public static String getString(Ini ini, String section, String key, String defaultValue) {
+ String value = ini.get(section, key, String.class);
+ return (value != null) ? value : defaultValue;
+ }
+
+ public static int getInt(Ini ini, String section, String key, int defaultValue) {
+ Integer value = ini.get(section, key, Integer.class);
+ return (value != null) ? value : defaultValue;
+ }
+
+ public static long getLong(Ini ini, String section, String key, long defaultValue) {
+ Long value = ini.get(section, key, Long.class);
+ return (value != null) ? value : defaultValue;
+ }
+
+ public static Ini loadINIFile(String configFile) throws IOException {
+ Ini ini = new Ini();
+ File conffile = new File(configFile);
+ if (!conffile.exists()) {
+ throw new FileNotFoundException(configFile);
+ }
+ ini.load(conffile);
+ return ini;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
index 547ffe7..4240e3a 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
@@ -18,24 +18,32 @@
*/
package org.apache.hyracks.control.common.controllers;
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
-
+import org.apache.hyracks.api.application.IApplicationConfig;
+import org.apache.hyracks.control.common.application.IniApplicationConfig;
+import org.ini4j.Ini;
import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.Option;
import org.kohsuke.args4j.spi.StopOptionHandler;
-public class NCConfig implements Serializable {
- private static final long serialVersionUID = 1L;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.InetAddress;
+import java.util.List;
+import java.util.Map;
- @Option(name = "-cc-host", usage = "Cluster Controller host name", required = true)
- public String ccHost;
+public class NCConfig implements Serializable {
+ private static final long serialVersionUID = 2L;
+
+ @Option(name = "-cc-host", usage = "Cluster Controller host name (required unless specified in config file)", required = false)
+ public String ccHost = null;
@Option(name = "-cc-port", usage = "Cluster Controller port (default: 1099)", required = false)
public int ccPort = 1099;
- @Option(name = "-cluster-net-ip-address", usage = "IP Address to bind cluster listener", required = true)
+ @Option(name = "-address", usage = "IP Address for NC (default: localhost)", required = false)
+ public String ipAddress = InetAddress.getLoopbackAddress().getHostAddress();
+
+ @Option(name = "-cluster-net-ip-address", usage = "IP Address to bind cluster listener (default: same as -address)", required = false)
public String clusterNetIPAddress;
@Option(name = "-cluster-net-port", usage = "IP port to bind cluster listener (default: random port)", required = false)
@@ -47,10 +55,10 @@
@Option(name = "-cluster-net-public-port", usage = "Public IP port to announce cluster listener (default: same as -cluster-net-port; must set -cluser-net-public-ip-address also)", required = false)
public int clusterNetPublicPort = 0;
- @Option(name = "-node-id", usage = "Logical name of node controller unique within the cluster", required = true)
- public String nodeId;
+ @Option(name = "-node-id", usage = "Logical name of node controller unique within the cluster (required unless specified in config file)", required = false)
+ public String nodeId = null;
- @Option(name = "-data-ip-address", usage = "IP Address to bind data listener", required = true)
+ @Option(name = "-data-ip-address", usage = "IP Address to bind data listener (default: same as -address)", required = false)
public String dataIPAddress;
@Option(name = "-data-port", usage = "IP port to bind data listener (default: random port)", required = false)
@@ -62,7 +70,7 @@
@Option(name = "-data-public-port", usage = "Public IP port to announce data listener (default: same as -data-port; must set -data-public-ip-address also)", required = false)
public int dataPublicPort = 0;
- @Option(name = "-result-ip-address", usage = "IP Address to bind dataset result distribution listener", required = true)
+ @Option(name = "-result-ip-address", usage = "IP Address to bind dataset result distribution listener (default: same as -address)", required = false)
public String resultIPAddress;
@Option(name = "-result-port", usage = "IP port to bind dataset result distribution listener (default: random port)", required = false)
@@ -74,6 +82,9 @@
@Option(name = "-result-public-port", usage = "Public IP port to announce dataset result distribution listener (default: same as -result-port; must set -result-public-ip-address also)", required = false)
public int resultPublicPort = 0;
+ @Option(name = "-retries", usage ="Number of attempts to contact CC before giving up (default = 5)")
+ public int retries = 5;
+
@Option(name = "-iodevices", usage = "Comma separated list of IO Device mount points (default: One device in default temp folder)", required = false)
public String ioDevices = System.getProperty("java.io.tmpdir");
@@ -98,10 +109,85 @@
@Option(name = "-app-nc-main-class", usage = "Application NC Main Class")
public String appNCMainClass;
+ @Option(name = "-config-file", usage = "Specify path to local configuration file (default: no local config)", required = false)
+ public String configFile = null;
+
@Argument
@Option(name = "--", handler = StopOptionHandler.class)
public List<String> appArgs;
+ private transient Ini ini = null;
+
+ private void loadINIFile() throws IOException {
+ ini = IniUtils.loadINIFile(configFile);
+ // QQQ This should default to cc/address if cluster.address not set, but
+ // that logic really should be handled by the ini file sent from the CC
+ ccHost = IniUtils.getString(ini, "cc", "cluster.address", ccHost);
+ ccPort = IniUtils.getInt(ini, "cc", "cluster.port", ccPort);
+ nodeId = IniUtils.getString(ini, "localnc", "id", nodeId);
+
+ // Network ports
+
+ ipAddress = IniUtils.getString(ini, "localnc", "address", ipAddress);
+
+ clusterNetIPAddress = IniUtils.getString(ini, "localnc", "cluster.address", clusterNetIPAddress);
+ clusterNetPort = IniUtils.getInt(ini, "localnc", "cluster.port", clusterNetPort);
+ dataIPAddress = IniUtils.getString(ini, "localnc", "data.address", dataIPAddress);
+ dataPort = IniUtils.getInt(ini, "localnc", "data.port", dataPort);
+ resultIPAddress = IniUtils.getString(ini, "localnc", "result.address", resultIPAddress);
+ resultPort = IniUtils.getInt(ini, "localnc", "result.port", resultPort);
+
+ clusterNetPublicIPAddress = IniUtils.getString(ini, "localnc", "public.cluster.address", clusterNetPublicIPAddress);
+ clusterNetPublicPort = IniUtils.getInt(ini, "localnc", "public.cluster.port", clusterNetPublicPort);
+ dataPublicIPAddress = IniUtils.getString(ini, "localnc", "public.data.address", dataPublicIPAddress);
+ dataPublicPort = IniUtils.getInt(ini, "localnc", "public.data.port", dataPublicPort);
+ resultPublicIPAddress = IniUtils.getString(ini, "localnc", "public.result.address", resultPublicIPAddress);
+ resultPublicPort = IniUtils.getInt(ini, "localnc", "public.result.port", resultPublicPort);
+
+ retries = IniUtils.getInt(ini, "localnc", "retries", retries);
+
+ // Directories
+ ioDevices = IniUtils.getString(ini, "localnc", "iodevices", ioDevices);
+
+ // Hyracks client entrypoint
+ appNCMainClass = IniUtils.getString(ini, "localnc", "app.class", appNCMainClass);
+ }
+
+ /*
+ * Once all @Option fields have been loaded from command-line or otherwise
+ * specified programmatically, call this method to:
+ * 1. Load options from a config file (as specified by -config-file)
+ * 2. Set default values for certain derived values, such as setting
+ * clusterNetIpAddress to ipAddress
+ */
+ public void loadConfigAndApplyDefaults() throws IOException {
+ if (configFile != null) {
+ loadINIFile();
+ }
+
+ // "address" is the default for all IP addresses
+ if (clusterNetIPAddress == null) clusterNetIPAddress = ipAddress;
+ if (dataIPAddress == null) dataIPAddress = ipAddress;
+ if (resultIPAddress == null) resultIPAddress = ipAddress;
+
+ // All "public" options default to their "non-public" versions
+ if (clusterNetPublicIPAddress == null) clusterNetPublicIPAddress = clusterNetIPAddress;
+ if (clusterNetPublicPort == 0) clusterNetPublicPort = clusterNetPort;
+ if (dataPublicIPAddress == null) dataPublicIPAddress = dataIPAddress;
+ if (dataPublicPort == 0) dataPublicPort = dataPort;
+ if (resultPublicIPAddress == null) resultPublicIPAddress = resultIPAddress;
+ if (resultPublicPort == 0) resultPublicPort = resultPort;
+ }
+
+ /**
+ * @return An IApplicationConfig representing this NCConfig.
+ * Note: Currently this only includes the values from the configuration
+ * file, not anything specified on the command-line. QQQ
+ */
+ public IApplicationConfig getAppConfig() {
+ return new IniApplicationConfig(ini);
+ }
+
public void toCommandLine(List<String> cList) {
cList.add("-cc-host");
cList.add(ccHost);
@@ -133,6 +219,8 @@
cList.add(resultPublicIPAddress);
cList.add("-result-public-port");
cList.add(String.valueOf(resultPublicPort));
+ cList.add("-retries");
+ cList.add(String.valueOf(retries));
cList.add("-iodevices");
cList.add(ioDevices);
cList.add("-net-thread-count");
@@ -176,6 +264,7 @@
configuration.put("result-port", String.valueOf(resultPort));
configuration.put("result-public-ip-address", resultPublicIPAddress);
configuration.put("result-public-port", String.valueOf(resultPublicPort));
+ configuration.put("retries", String.valueOf(retries));
configuration.put("iodevices", ioDevices);
configuration.put("net-thread-count", String.valueOf(nNetThreads));
configuration.put("net-buffer-count", String.valueOf(nNetBuffers));