[ASTERIXDB-3386][CONF] Add runtime memory overhead config
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
- Add a new compiler configuration that allows specifying
a percentage of memory to be added to the job's required
memory to account for runtime memory overhead and default
it to 5%.
Change-Id: I33a1911ff6b0df64b273ba10a7626dd10cc4cca2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18246
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 3da25fe..16f922f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -342,6 +342,7 @@
getJobLocations(spec, nodeJobTracker, computationLocations);
final IClusterCapacity jobRequiredCapacity =
ResourceUtils.getRequiredCapacity(plan, jobLocations, physOptConf);
+ addRuntimeMemoryOverhead(jobRequiredCapacity, compilerProperties);
spec.setRequiredClusterCapacity(jobRequiredCapacity);
}
}
@@ -600,4 +601,14 @@
return new AlgebricksAbsolutePartitionConstraint(Arrays.stream(clusterLocations.getLocations())
.filter(jobParticipatingNodes::contains).toArray(String[]::new));
}
+
+ private static void addRuntimeMemoryOverhead(IClusterCapacity jobRequiredCapacity,
+ CompilerProperties compilerProperties) {
+ int runtimeMemoryOverheadPercentage = compilerProperties.getRuntimeMemoryOverheadPercentage();
+ if (runtimeMemoryOverheadPercentage > 0) {
+ double multiplier = 1 + runtimeMemoryOverheadPercentage / 100.0;
+ long aggregatedMemoryByteSize = jobRequiredCapacity.getAggregatedMemoryByteSize();
+ jobRequiredCapacity.setAggregatedMemoryByteSize((long) (aggregatedMemoryByteSize * multiplier));
+ }
+ }
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 1bb6ef9..6888acc 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -32,6 +32,7 @@
"compiler\.min\.memory\.allocation" : true,
"compiler\.parallelism" : 0,
"compiler.queryplanshape" : "zigzag",
+ "compiler.runtime.memory.overhead" : 5,
"compiler\.sort\.parallel" : false,
"compiler\.sort\.samples" : 100,
"compiler\.sortmemory" : 327680,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index c2883ae..31daf99 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -32,6 +32,7 @@
"compiler\.min\.memory\.allocation" : true,
"compiler\.parallelism" : -1,
"compiler.queryplanshape" : "zigzag",
+ "compiler.runtime.memory.overhead" : 5,
"compiler\.sort\.parallel" : true,
"compiler\.sort\.samples" : 100,
"compiler\.sortmemory" : 327680,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 8c54bb8..a47b3be 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -32,6 +32,7 @@
"compiler\.min\.memory\.allocation" : true,
"compiler\.parallelism" : 3,
"compiler.queryplanshape" : "zigzag",
+ "compiler.runtime.memory.overhead" : 5,
"compiler\.sort\.parallel" : true,
"compiler\.sort\.samples" : 100,
"compiler\.sortmemory" : 327680,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex
index e2d166b..13bbece 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex
@@ -1 +1 @@
-/memory\D+229376/
\ No newline at end of file
+/memory\D+240844/
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex
index e2d166b..13bbece 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex
@@ -1 +1 @@
-/memory\D+229376/
\ No newline at end of file
+/memory\D+240844/
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex
index e2d166b..13bbece 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex
@@ -1 +1 @@
-/memory\D+229376/
\ No newline at end of file
+/memory\D+240844/
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex
index e2d166b..13bbece 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex
@@ -1 +1 @@
-/memory\D+229376/
\ No newline at end of file
+/memory\D+240844/
\ No newline at end of file
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 85b80a6..edd25bf 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -22,6 +22,7 @@
import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER;
import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER_BYTE_UNIT;
import static org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT;
+import static org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER;
import static org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER;
import static org.apache.hyracks.control.common.config.OptionTypes.STRING;
import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
@@ -124,7 +125,11 @@
AlgebricksConfig.COLUMN_FILTER_DEFAULT,
"Enable/disable the use of column min/max filters"),
//TODO(DB): remove after
- COMPILER_ENABLE_DB_RESOLUTION(BOOLEAN, true, "Enable/disable the resolution of namespaces to database");
+ COMPILER_ENABLE_DB_RESOLUTION(BOOLEAN, true, "Enable/disable the resolution of namespaces to database"),
+ COMPILER_RUNTIME_MEMORY_OVERHEAD(
+ NONNEGATIVE_INTEGER,
+ 5,
+ "A percentage of the job's required memory to be added to account for runtime memory overhead");
private final IOptionType type;
private final Object defaultValue;
@@ -207,8 +212,6 @@
public static final String COMPILER_COLUMN_FILTER_KEY = Option.COMPILER_COLUMN_FILTER.ini();
- public static final String COMPILER_ENABLE_DB_RESOLUTION_KEY = Option.COMPILER_ENABLE_DB_RESOLUTION.ini();
-
public static final int COMPILER_PARALLELISM_AS_STORAGE = 0;
public CompilerProperties(PropertiesAccessor accessor) {
@@ -312,7 +315,7 @@
return accessor.getBoolean(Option.COMPILER_COLUMN_FILTER);
}
- public boolean isDbResolutionEnabled() {
- return accessor.getBoolean(Option.COMPILER_ENABLE_DB_RESOLUTION);
+ public int getRuntimeMemoryOverheadPercentage() {
+ return accessor.getInt(Option.COMPILER_RUNTIME_MEMORY_OVERHEAD);
}
}