Automatically set parameters.
The parameters include:
-- storage.buffercache.size
-- storage.memorycomponent.numpages
-- storage.memorycomponent.globalbudget
-- -Xmx for NCDriver
Change-Id: I02c233b86b10a998482b8d44123d45b53ebf14dc
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1417
Reviewed-by: Till Westmann <tillw@apache.org>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
index 8a96882..b7103dd 100644
--- a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
+++ b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
@@ -77,8 +77,7 @@
<name>storage.buffercache.size</name>
<value>48MB</value>
<description>The size of memory allocated to the disk buffer cache.
- The value should be a multiple of the buffer cache page size
- (Default = "512MB")
+ The value should be a multiple of the buffer cache page size.
</description>
</property>
<property>
@@ -88,7 +87,14 @@
This budget is shared by all the memory components of the primary
index and all its secondary indexes across all I/O devices on a node.
Note: in-memory components usually has fill factor of 75% since
- the pages are 75% full and the remaining 25% is un-utilized. (Default = 256)
+ the pages are 75% full and the remaining 25% is un-utilized.
+ </description>
+ </property>
+ <property>
+ <name>storage.memorycomponent.globalbudget</name>
+ <value>512MB</value>
+ <description>The size of memory allocated to the memory components.
+ The value should be a multiple of the memory component page size.
</description>
</property>
<property>
diff --git a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml
index 2989fa9..c5b8470 100644
--- a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml
+++ b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml
@@ -81,8 +81,7 @@
<name>storage.buffercache.size</name>
<value>48MB</value>
<description>The size of memory allocated to the disk buffer cache.
- The value should be a multiple of the buffer cache page size
- (Default = "512MB")
+ The value should be a multiple of the buffer cache page size.
</description>
</property>
<property>
@@ -92,7 +91,14 @@
This budget is shared by all the memory components of the primary
index and all its secondary indexes across all I/O devices on a node.
Note: in-memory components usually has fill factor of 75% since
- the pages are 75% full and the remaining 25% is un-utilized. (Default = 256)
+ the pages are 75% full and the remaining 25% is un-utilized.
+ </description>
+ </property>
+ <property>
+ <name>storage.memorycomponent.globalbudget</name>
+ <value>512MB</value>
+ <description>The size of memory allocated to the memory components.
+ The value should be a multiple of the memory component page size.
</description>
</property>
<property>
diff --git a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml
index 85881f9..73a8ea8 100644
--- a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml
+++ b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml
@@ -81,8 +81,7 @@
<name>storage.buffercache.size</name>
<value>48MB</value>
<description>The size of memory allocated to the disk buffer cache.
- The value should be a multiple of the buffer cache page size
- (Default = "512MB")
+ The value should be a multiple of the buffer cache page size.
</description>
</property>
<property>
@@ -92,7 +91,14 @@
This budget is shared by all the memory components of the primary
index and all its secondary indexes across all I/O devices on a node.
Note: in-memory components usually has fill factor of 75% since
- the pages are 75% full and the remaining 25% is un-utilized. (Default = 256)
+ the pages are 75% full and the remaining 25% is un-utilized.
+ </description>
+ </property>
+ <property>
+ <name>storage.memorycomponent.globalbudget</name>
+ <value>512MB</value>
+ <description>The size of memory allocated to the memory components.
+ The value should be a multiple of the memory component page size.
</description>
</property>
<property>
diff --git a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration4.xml b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration4.xml
new file mode 100644
index 0000000..f7bb7fc
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration4.xml
@@ -0,0 +1,98 @@
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<asterixConfiguration xmlns="asterixconf">
+ <metadataNode>asterix_nc1</metadataNode>
+ <store>
+ <ncId>asterix_nc1</ncId>
+ <storeDirs>iodevice0,iodevice1</storeDirs>
+ </store>
+ <store>
+ <ncId>asterix_nc2</ncId>
+ <storeDirs>iodevice0,iodevice1</storeDirs>
+ </store>
+ <transactionLogDir>
+ <ncId>asterix_nc1</ncId>
+ <txnLogDirPath>target/txnLogDir/asterix_nc1</txnLogDirPath>
+ </transactionLogDir>
+ <transactionLogDir>
+ <ncId>asterix_nc2</ncId>
+ <txnLogDirPath>target/txnLogDir/asterix_nc2</txnLogDirPath>
+ </transactionLogDir>
+
+ <property>
+ <name>max.wait.active.cluster</name>
+ <value>60</value>
+ <description>Maximum wait (in seconds) for a cluster to be ACTIVE (all
+ nodes are available)
+ before a submitted query/statement can be
+ executed. (Default = 60 seconds)
+ </description>
+ </property>
+
+ <property>
+ <name>log.level</name>
+ <value>INFO</value>
+ <description>Log level for running tests/build</description>
+ </property>
+ <property>
+ <name>compiler.framesize</name>
+ <value>32KB</value>
+ </property>
+ <property>
+ <name>compiler.sortmemory</name>
+ <value>320KB</value>
+ </property>
+ <property>
+ <name>compiler.groupmemory</name>
+ <value>160KB</value>
+ </property>
+ <property>
+ <name>compiler.joinmemory</name>
+ <value>256KB</value>
+ </property>
+ <property>
+ <name>compiler.parallelism</name>
+ <value>-1</value>
+ </property>
+ <property>
+ <name>storage.buffercache.pagesize</name>
+ <value>32KB</value>
+ <description>The page size in bytes for pages in the buffer cache.
+ (Default = "128KB")
+ </description>
+ </property>
+ <property>
+ <name>plot.activate</name>
+ <value>false</value>
+ <description>Enabling plot of Algebricks plan to tmp folder. (Default = false)
+ </description>
+ </property>
+ <property>
+ <name>messaging.frame.size</name>
+ <value>4096</value>
+ <description>The frame size to be used for NC to NC messaging. (Default = 4kb)
+ </description>
+ </property>
+ <property>
+ <name>messaging.frame.count</name>
+ <value>512</value>
+ <description>Number of reusable frames for NC to NC messaging. (Default = 512)
+ </description>
+ </property>
+</asterixConfiguration>
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java
new file mode 100644
index 0000000..3defc40
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.runtime;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.Collection;
+
+import org.apache.asterix.common.config.StorageProperties;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.json.JSONObject;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import org.mortbay.util.ajax.JSON;
+
+/**
+ * Runs the cluster state runtime tests with the storage parallelism.
+ */
+@RunWith(Parameterized.class)
+public class ClusterStateDefaultParameterTest {
+ protected static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration4.xml";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ LangExecutionUtil.tearDown();
+ }
+
+ @Parameters(name = "ClusterStateExecutionTest {index}: {0}")
+ public static Collection<Object[]> tests() throws Exception {
+ return LangExecutionUtil.tests("only_cluster_state.xml", "cluster_state.xml");
+ }
+
+ protected TestCaseContext tcCtx;
+
+ public ClusterStateDefaultParameterTest(TestCaseContext tcCtx) {
+ this.tcCtx = tcCtx;
+ }
+
+ @Test
+ public void test() throws Exception {
+ StringBuilder result = new StringBuilder();
+ URL url = new URL("http://localhost:19002/admin/cluster");
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.setRequestMethod("GET");
+ BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream()));
+ String line;
+ while ((line = rd.readLine()) != null) {
+ result.append(line);
+ }
+ rd.close();
+ long maxHeap = Runtime.getRuntime().maxMemory();
+ int matchCount = 0;
+ String[] rows = result.toString().split(",");
+ for (String row : rows) {
+ if (row.contains("storage.buffercache.size")) {
+ Assert.assertTrue(getValue(row) == maxHeap / 4);
+ matchCount++;
+ }
+ if (row.contains("storage.memorycomponent.globalbudget")) {
+ Assert.assertTrue(getValue(row) == maxHeap / 4);
+ matchCount++;
+ }
+ if (row.contains("storage.memorycomponent.numpages")) {
+ Assert.assertTrue(getValue(row) == maxHeap / (131072 * 64));
+ matchCount++;
+ }
+ }
+ Assert.assertTrue(matchCount == 3);
+ }
+
+ // Parses a long value parameter.
+ private long getValue(String row) {
+ String valueStr = row.split(":")[1].trim();
+ return Long.parseLong(valueStr);
+ }
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index b86d961..fbaf0a7 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -22,7 +22,6 @@
import org.apache.hyracks.util.StorageUtil;
import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
-import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE;
public class StorageProperties extends AbstractProperties {
@@ -30,7 +29,6 @@
private static final int STORAGE_BUFFERCACHE_PAGESIZE_DEFAULT = StorageUtil.getSizeInBytes(128, KILOBYTE);
private static final String STORAGE_BUFFERCACHE_SIZE_KEY = "storage.buffercache.size";
- private static final long STORAGE_BUFFERCACHE_SIZE_DEFAULT = StorageUtil.getSizeInBytes(512, MEGABYTE);
private static final String STORAGE_BUFFERCACHE_MAXOPENFILES_KEY = "storage.buffercache.maxopenfiles";
private static final int STORAGE_BUFFERCACHE_MAXOPENFILES_DEFAULT = Integer.MAX_VALUE;
@@ -39,7 +37,6 @@
private static final int STORAGE_MEMORYCOMPONENT_PAGESIZE_DEFAULT = StorageUtil.getSizeInBytes(128, KILOBYTE);
private static final String STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY = "storage.memorycomponent.numpages";
- private static final int STORAGE_MEMORYCOMPONENT_NUMPAGES_DEFAULT = 256; // ... so 32MB components
private static final String STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY =
"storage.metadata.memorycomponent.numpages";
@@ -49,14 +46,28 @@
private static final int STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_DEFAULT = 2; // 2 components
private static final String STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY = "storage.memorycomponent.globalbudget";
- private static final long STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT = StorageUtil.getSizeInBytes(512, MEGABYTE);
private static final String STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE_KEY =
"storage.lsm.bloomfilter.falsepositiverate";
private static final double STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE_DEFAULT = 0.01;
+ private final long storageBufferCacheSizeDefault;
+ private final int storageMemoryComponentNumPages;
+ private final long storageMemorycomponentGlobalbudgetDefault;
+
public StorageProperties(PropertiesAccessor accessor) {
super(accessor);
+
+ // Gets the -Xmx value for the JVM.
+ long maxHeapSize = Runtime.getRuntime().maxMemory();
+ // By default, uses 1/4 of the maximum heap size for read cache, i.e., disk buffer cache.
+ storageBufferCacheSizeDefault = maxHeapSize / 4;
+ // By default, uses 1/4 of the maximum heap size for the write buffer, i.e., globalbudget for memory components.
+ storageMemorycomponentGlobalbudgetDefault = maxHeapSize / 4;
+ // By default, uses 1/16 of the storageMemorycomponentGlobalbudgetDefault for the write buffer budget
+ // for a dataset, including data and indexes.
+ storageMemoryComponentNumPages = (int) storageMemorycomponentGlobalbudgetDefault
+ / (16 * getMemoryComponentPageSize());
}
@PropertyKey(STORAGE_BUFFERCACHE_PAGESIZE_KEY)
@@ -67,7 +78,7 @@
@PropertyKey(STORAGE_BUFFERCACHE_SIZE_KEY)
public long getBufferCacheSize() {
- return accessor.getProperty(STORAGE_BUFFERCACHE_SIZE_KEY, STORAGE_BUFFERCACHE_SIZE_DEFAULT,
+ return accessor.getProperty(STORAGE_BUFFERCACHE_SIZE_KEY, storageBufferCacheSizeDefault,
PropertyInterpreters.getLongBytePropertyInterpreter());
}
@@ -89,7 +100,7 @@
@PropertyKey(STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY)
public int getMemoryComponentNumPages() {
- return accessor.getProperty(STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY, STORAGE_MEMORYCOMPONENT_NUMPAGES_DEFAULT,
+ return accessor.getProperty(STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY, storageMemoryComponentNumPages,
PropertyInterpreters.getIntegerPropertyInterpreter());
}
@@ -108,8 +119,8 @@
@PropertyKey(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY)
public long getMemoryComponentGlobalBudget() {
- return accessor.getProperty(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY,
- STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT, PropertyInterpreters.getLongBytePropertyInterpreter());
+ return accessor.getProperty(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY, storageMemorycomponentGlobalbudgetDefault,
+ PropertyInterpreters.getLongBytePropertyInterpreter());
}
@PropertyKey(STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE_KEY)
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml
index dfa1807..2f966a3 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/pom.xml
@@ -65,6 +65,17 @@
<artifactId>commons-lang3</artifactId>
<version>3.5</version>
</dependency>
+ <dependency>
+ <groupId>com.e-movimento.tinytools</groupId>
+ <artifactId>privilegedaccessor</artifactId>
+ <version>1.2.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java
index 8ca61ad..5a03d3c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/main/java/org/apache/hyracks/control/nc/service/NCService.java
@@ -23,6 +23,8 @@
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.StringReader;
+import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
import java.net.InetAddress;
import java.net.ServerSocket;
import java.net.Socket;
@@ -72,6 +74,11 @@
*/
private static Process proc = null;
+ /**
+ * The management bean for obtaining settings of the underlying operating system and hardware.
+ */
+ private static OperatingSystemMXBean osMXBean;
+
private static List<String> buildCommand() throws IOException {
List<String> cList = new ArrayList<>();
@@ -114,7 +121,8 @@
LOGGER.info("Using JAVA_OPTS from environment");
} else {
LOGGER.info("Using default JAVA_OPTS");
- jvmargs = "-Xmx1536m";
+ long ramSize = ((com.sun.management.OperatingSystemMXBean) osMXBean).getTotalPhysicalMemorySize();
+ jvmargs = "-Xmx" + (int) Math.ceil(0.6 * ramSize / (1024 * 1024)) + "m";
}
}
env.put("JAVA_OPTS", jvmargs);
@@ -243,6 +251,9 @@
}
config.loadConfigAndApplyDefaults();
+ // Initializes the oxMXBean.
+ osMXBean = ManagementFactory.getOperatingSystemMXBean();
+
// For now we implement a trivial listener which just
// accepts an IP/port combination from the CC. This could
// be made more advanced in several ways depending on whether
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/test/java/org/apache/hyracks/control/nc/service/NCServiceTest.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/test/java/org/apache/hyracks/control/nc/service/NCServiceTest.java
new file mode 100644
index 0000000..00d7ca4
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-nc-service/src/test/java/org/apache/hyracks/control/nc/service/NCServiceTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.nc.service;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import junit.extensions.PA;
+
+public class NCServiceTest {
+
+ @Test
+ public void testJvmArgs() {
+ Map<String, String> configuration = new HashMap<>();
+ OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean();
+ PA.setValue(NCService.class, "nodeSection", "nc/nc1");
+ PA.setValue(NCService.class, "osMXBean", osMXBean);
+ PA.invokeMethod(NCService.class, "configEnvironment(java.util.Map)", configuration);
+ String javaOpts = configuration.get("JAVA_OPTS");
+ String prefix = javaOpts.substring(4);
+ String sizeStr = prefix.substring(0, prefix.length() - 1);
+ int size = Integer.parseInt(sizeStr);
+ long ramSize = ((com.sun.management.OperatingSystemMXBean) osMXBean).getTotalPhysicalMemorySize();
+ int base = 1024 * 1024 * 5;
+ Assert.assertTrue(size == ramSize * 3 / base + ((ramSize * 3) % base == 0 ? 0 : 1));
+ }
+}