[NO ISSUE][COMP] Add compiler property to skip assiging low budget
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Add a compiler property to allow users to instruct the compiler
not to assign low budget for queries such metadata queries.
Change-Id: Ibba711d76dbdc73c1072c999c05c43dbce30debc
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9848
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Reviewed-by: Dmitry Lychagin <dmitry.lychagin@couchbase.com>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java
index abe4b71..8de1fb5 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java
@@ -52,6 +52,9 @@
}
private boolean forceMinMemoryBudget(AsterixOptimizationContext context) {
+ if (!context.getPhysicalOptimizationConfig().getMinMemoryAllocation()) {
+ return false;
+ }
Int2ObjectMap<Set<DataSource>> dataSourceMap = context.getDataSourceMap();
if (dataSourceMap.isEmpty()) {
return false;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index baf083d..9088db6 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -135,7 +135,8 @@
CompilerProperties.COMPILER_SORT_PARALLEL_KEY, CompilerProperties.COMPILER_SORT_SAMPLES_KEY,
CompilerProperties.COMPILER_INDEXONLY_KEY, CompilerProperties.COMPILER_INTERNAL_SANITYCHECK_KEY,
CompilerProperties.COMPILER_EXTERNAL_FIELD_PUSHDOWN_KEY, CompilerProperties.COMPILER_SUBPLAN_MERGE_KEY,
- CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
+ CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY,
+ CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
FuzzyUtils.SIM_FUNCTION_PROP_NAME, FuzzyUtils.SIM_THRESHOLD_PROP_NAME,
StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME,
FeedActivityDetails.COLLECT_LOCATIONS, SqlppQueryRewriter.INLINE_WITH_OPTION,
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index e202ddf..44038cc 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -42,6 +42,8 @@
import java.net.URISyntaxException;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.text.MessageFormat;
import java.util.ArrayDeque;
import java.util.ArrayList;
@@ -161,6 +163,7 @@
private static final Pattern HANDLE_VARIABLE_PATTERN = Pattern.compile("handlevariable=(\\w+)");
private static final Pattern RESULT_VARIABLE_PATTERN = Pattern.compile("resultvariable=(\\w+)");
private static final Pattern COMPARE_UNORDERED_ARRAY_PATTERN = Pattern.compile("compareunorderedarray=(\\w+)");
+ private static final Pattern BODY_REF_PATTERN = Pattern.compile("bodyref=(.*)", Pattern.MULTILINE);
private static final Pattern MACRO_PARAM_PATTERN =
Pattern.compile("macro (?<name>[\\w-$]+)=(?<value>.*)", Pattern.MULTILINE);
@@ -1329,12 +1332,16 @@
final String trimmedPathAndQuery = stripAllComments(statement).trim();
final String variablesReplaced = replaceVarRef(trimmedPathAndQuery, variableCtx);
final List<Parameter> params = extractParameters(statement);
- final Optional<String> body = extractBody(statement);
+ Optional<String> body = extractBody(statement);
final Predicate<Integer> statusCodePredicate = extractStatusCodePredicate(statement);
final boolean extracResult = isExtracResult(statement);
final boolean extractStatus = isExtractStatus(statement);
final String mimeReqType = extractHttpRequestType(statement);
+ final String saveResponseVar = getResultVariable(statement);
ContentType contentType = mimeReqType != null ? ContentType.create(mimeReqType, UTF_8) : TEXT_PLAIN_UTF8;
+ if (!body.isPresent()) {
+ body = getBodyFromReference(statement, variableCtx);
+ }
InputStream resultStream;
if ("http".equals(extension)) {
resultStream = executeHttp(reqType, variablesReplaced, fmt, params, statusCodePredicate, body, contentType);
@@ -1356,6 +1363,8 @@
} else {
throw new Exception("no handle for test " + testFile.toString());
}
+ } else if (saveResponseVar != null) {
+ variableCtx.put(saveResponseVar, IOUtils.toString(resultStream, UTF_8));
} else {
if (expectedResultFile == null) {
if (testFile.getName().startsWith(DIAGNOSE)) {
@@ -1689,6 +1698,27 @@
return resultVariableMatcher.find() ? resultVariableMatcher.group(1) : null;
}
+ protected static Optional<String> getBodyFromReference(String statement, Map<String, Object> varMap)
+ throws IOException {
+ Optional<String> bodyRef = findPattern(statement, BODY_REF_PATTERN);
+ if (!bodyRef.isPresent()) {
+ return Optional.empty();
+ }
+ String bodyReference = bodyRef.get();
+ String body = (String) varMap.get(bodyReference);
+ if (body != null) {
+ return Optional.of(body);
+ }
+ try (Stream<String> stream = Files.lines(Paths.get(bodyReference), UTF_8)) {
+ return Optional.of(stream.collect(Collectors.joining()));
+ }
+ }
+
+ protected static Optional<String> findPattern(String statement, Pattern pattern) {
+ final Matcher matcher = pattern.matcher(statement);
+ return matcher.find() ? Optional.of(matcher.group(1)) : Optional.empty();
+ }
+
protected static boolean getCompareUnorderedArray(String statement) {
final Matcher matcher = COMPARE_UNORDERED_ARRAY_PATTERN.matcher(statement);
return matcher.find() && Boolean.parseBoolean(matcher.group(1));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.2.plans.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.2.plans.sqlpp
new file mode 100644
index 0000000..0b6fdbf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.2.plans.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/* Test that a metadata-only query gets a normal memory budget instead of the compiler-allocated memory */
+
+-- param job:string=true
+
+SET `compiler.parallelism` "1";
+SET `compiler.min.memory.allocation` "false";
+SET `compiler.sortmemory` "320KB";
+SET `compiler.groupmemory` "160KB";
+SET `compiler.joinmemory` "256KB";
+SET `compiler.textsearchmemory` "160KB";
+
+from Metadata.`Dataset` ds
+join Metadata.`Index` idx
+on ds.DataverseName = idx.DataverseName and ds.DatasetName = idx.DatasetName
+where contains(ds.DataverseName, "Metadata")
+ and ds.DatasetName in ["Dataverse", "Dataset", "Index"]
+ and idx.IsPrimary
+group by ds.DataverseName
+select value count(*);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 378fc83..b6a49a6 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -14,6 +14,7 @@
"compiler\.indexonly" : true,
"compiler\.internal\.sanitycheck" : true,
"compiler\.joinmemory" : 262144,
+ "compiler\.min\.memory\.allocation" : true,
"compiler\.parallelism" : 0,
"compiler\.sort\.parallel" : false,
"compiler\.sort\.samples" : 100,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 5ff1b7a..dedb40f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -14,6 +14,7 @@
"compiler\.indexonly" : true,
"compiler\.internal\.sanitycheck" : false,
"compiler\.joinmemory" : 262144,
+ "compiler\.min\.memory\.allocation" : true,
"compiler\.parallelism" : -1,
"compiler\.sort\.parallel" : true,
"compiler\.sort\.samples" : 100,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 2e76d04..985d3bd 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -14,6 +14,7 @@
"compiler\.indexonly" : true,
"compiler\.internal\.sanitycheck" : false,
"compiler\.joinmemory" : 262144,
+ "compiler\.min\.memory\.allocation" : true,
"compiler\.parallelism" : 3,
"compiler\.sort\.parallel" : true,
"compiler\.sort\.samples" : 100,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.2.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.2.regex
new file mode 100644
index 0000000..6af6770
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.2.regex
@@ -0,0 +1 @@
+/memory\D+1146880/
\ No newline at end of file
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 9340110..39142e5 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -91,7 +91,11 @@
COMPILER_SUBPLAN_NESTEDPUSHDOWN(
BOOLEAN,
AlgebricksConfig.SUBPLAN_NESTEDPUSHDOWN_DEFAULT,
- "When merging subplans into groupby/suplan allow nesting of subplans");
+ "When merging subplans into groupby/suplan allow nesting of subplans"),
+ COMPILER_MIN_MEMORY_ALLOCATION(
+ BOOLEAN,
+ AlgebricksConfig.MIN_MEMORY_ALLOCATION_DEFAULT,
+ "Enable/disable allocating minimum budget for certain queries");
private final IOptionType type;
private final Object defaultValue;
@@ -150,6 +154,8 @@
public static final String COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY = Option.COMPILER_SUBPLAN_NESTEDPUSHDOWN.ini();
+ public static final String COMPILER_MIN_MEMORY_ALLOCATION_KEY = Option.COMPILER_MIN_MEMORY_ALLOCATION.ini();
+
public static final int COMPILER_PARALLELISM_AS_STORAGE = 0;
public CompilerProperties(PropertiesAccessor accessor) {
@@ -211,4 +217,8 @@
public boolean getSubplanNestedPushdown() {
return accessor.getBoolean(Option.COMPILER_SUBPLAN_NESTEDPUSHDOWN);
}
+
+ public boolean getMinMemoryAllocation() {
+ return accessor.getBoolean(Option.COMPILER_MIN_MEMORY_ALLOCATION);
+ }
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
index 5079b25..8832054 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
@@ -70,6 +70,8 @@
compilerProperties.getSubplanMerge());
boolean subplanNestedPushdown = getBoolean(querySpecificConfig,
CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY, compilerProperties.getSubplanNestedPushdown());
+ boolean minMemoryAllocation = getBoolean(querySpecificConfig,
+ CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, compilerProperties.getMinMemoryAllocation());
PhysicalOptimizationConfig physOptConf = new PhysicalOptimizationConfig();
physOptConf.setFrameSize(frameSize);
@@ -85,6 +87,7 @@
physOptConf.setExternalFieldPushdown(externalFieldPushdown);
physOptConf.setSubplanMerge(subplanMerge);
physOptConf.setSubplanNestedPushdown(subplanNestedPushdown);
+ physOptConf.setMinMemoryAllocation(minMemoryAllocation);
return physOptConf;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
index 2ed40fd..c99117b 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
@@ -32,4 +32,5 @@
public static final boolean EXTERNAL_FIELD_PUSHDOWN_DEFAULT = false;
public static final boolean SUBPLAN_MERGE_DEFAULT = true;
public static final boolean SUBPLAN_NESTEDPUSHDOWN_DEFAULT = true;
+ public static final boolean MIN_MEMORY_ALLOCATION_DEFAULT = true;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
index bfa4298..92f579c 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
@@ -44,8 +44,9 @@
private static final String EXTERNAL_FIELD_PUSHDOWN = "EXTERNAL_FIELD_PUSHDOWN";
private static final String SUBPLAN_MERGE = "SUBPLAN_MERGE";
private static final String SUBPLAN_NESTEDPUSHDOWN = "SUBPLAN_NESTEDPUSHDOWN";
+ private static final String MIN_MEMORY_ALLOCATION = "MIN_MEMORY_ALLOCATION";
- private Properties properties = new Properties();
+ private final Properties properties = new Properties();
public PhysicalOptimizationConfig() {
int frameSize = 32768;
@@ -217,6 +218,14 @@
setBoolean(SUBPLAN_NESTEDPUSHDOWN, value);
}
+ public boolean getMinMemoryAllocation() {
+ return getBoolean(MIN_MEMORY_ALLOCATION, AlgebricksConfig.MIN_MEMORY_ALLOCATION_DEFAULT);
+ }
+
+ public void setMinMemoryAllocation(boolean value) {
+ setBoolean(MIN_MEMORY_ALLOCATION, value);
+ }
+
private void setInt(String property, int value) {
properties.setProperty(property, Integer.toString(value));
}