[ASTERIXDB-3293][COMP] Do not require job capacity for metadata queries
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
Metadata queries can sometimes fail to run because
they are forced to use the minimum budget for the operators.
This patch is to lift this minimum budget and let the operators
be assigned the default budget like regular queries.
In addition, do not assign a job capacity for metadata queries.
A better solution for metadata queries should consider
a combination of solutions (e.g. calculate job capacity precisely
given that metadata queries are run in a single partition, don't
make the default budget of operators for regular and metadata
queries the same, make minimum budget configurable, ... etc)
Change-Id: I2a6721e2b6182aa0e12a1b0173709bc139fc77e0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17894
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Tested-by: Michael Blow <mblow@apache.org>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AsterixOptimizationContext.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AsterixOptimizationContext.java
index 7b2dd7a..05e7f7b 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AsterixOptimizationContext.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AsterixOptimizationContext.java
@@ -21,8 +21,10 @@
import java.util.HashSet;
import java.util.Set;
+import java.util.function.Predicate;
import org.apache.asterix.metadata.declared.DataSource;
+import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.core.algebra.expressions.IConflictingTypeResolver;
import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
@@ -73,4 +75,26 @@
public Int2ObjectMap<Set<DataSource>> getDataSourceMap() {
return dataSourceMap;
}
+
+ @Override
+ public boolean skipJobCapacityAssignment() {
+ if (dataSourceMap.isEmpty()) {
+ return false;
+ }
+ for (Int2ObjectMap.Entry<Set<DataSource>> me : dataSourceMap.int2ObjectEntrySet()) {
+ int dataSourceType = me.getIntKey();
+ if (dataSourceType != DataSource.Type.INTERNAL_DATASET) {
+ return false;
+ }
+ Predicate<DataSource> dataSourceTest = AsterixOptimizationContext::isMetadata;
+ if (!me.getValue().stream().allMatch(dataSourceTest)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static boolean isMetadata(DataSource ds) {
+ return MetadataConstants.METADATA_DATAVERSE_NAME.equals(ds.getId().getDataverseName());
+ }
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java
index e9de353..9467555 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java
@@ -63,9 +63,6 @@
int dataSourceType = me.getIntKey();
Predicate<DataSource> dataSourceTest;
switch (dataSourceType) {
- case DataSource.Type.INTERNAL_DATASET:
- dataSourceTest = SetAsterixMemoryRequirementsRule::isMinMemoryBudgetDataset;
- break;
case DataSource.Type.FUNCTION:
dataSourceTest = SetAsterixMemoryRequirementsRule::isMinMemoryBudgetFunction;
break;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index be21673..99678da 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -323,16 +323,18 @@
JobSpecification spec = compiler.createJob(ccAppContext, jobEventListenerFactory);
if (isQuery) {
- if (requestParameters == null || !requestParameters.isSkipAdmissionPolicy()) {
- // Sets a required capacity, only for read-only queries.
- // DDLs and DMLs are considered not that frequent.
- // limit the computation locations to the locations that will be used in the query
- final INodeJobTracker nodeJobTracker = ccAppContext.getNodeJobTracker();
- final AlgebricksAbsolutePartitionConstraint jobLocations =
- getJobLocations(spec, nodeJobTracker, computationLocations);
- final IClusterCapacity jobRequiredCapacity =
- ResourceUtils.getRequiredCapacity(plan, jobLocations, physOptConf);
- spec.setRequiredClusterCapacity(jobRequiredCapacity);
+ if (!compiler.skipJobCapacityAssignment()) {
+ if (requestParameters == null || !requestParameters.isSkipAdmissionPolicy()) {
+ // Sets a required capacity, only for read-only queries.
+ // DDLs and DMLs are considered not that frequent.
+ // limit the computation locations to the locations that will be used in the query
+ final INodeJobTracker nodeJobTracker = ccAppContext.getNodeJobTracker();
+ final AlgebricksAbsolutePartitionConstraint jobLocations =
+ getJobLocations(spec, nodeJobTracker, computationLocations);
+ final IClusterCapacity jobRequiredCapacity =
+ ResourceUtils.getRequiredCapacity(plan, jobLocations, physOptConf);
+ spec.setRequiredClusterCapacity(jobRequiredCapacity);
+ }
}
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.1.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.1.regex
index 7bc4c42..959bc0d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.1.regex
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.1.regex
@@ -1 +1 @@
-/memory\D+917504/
+/memory\D+0/
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.2.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.2.regex
index acf8405..959bc0d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.2.regex
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.2.regex
@@ -1 +1 @@
-/memory\D+1146880/
+/memory\D+0/
diff --git a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
index e35a539..85910aa 100644
--- a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
+++ b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
@@ -170,5 +170,10 @@
PlanCompiler pc = factory.createPlanCompiler(oc, appContext, writerFactory);
return pc.compilePlan(plan, jobEventListenerFactory);
}
+
+ @Override
+ public boolean skipJobCapacityAssignment() {
+ return oc.skipJobCapacityAssignment();
+ }
}
}
diff --git a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompiler.java b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompiler.java
index 7dac3eb..0f5798e 100644
--- a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompiler.java
+++ b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompiler.java
@@ -27,4 +27,6 @@
public JobSpecification createJob(Object appContext, IJobletEventListenerFactory jobEventListenerFactory)
throws AlgebricksException;
+
+ boolean skipJobCapacityAssignment();
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IOptimizationContext.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IOptimizationContext.java
index 69ec210..10b0451 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IOptimizationContext.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IOptimizationContext.java
@@ -100,4 +100,8 @@
Object getCompilerFactory();
IOptimizationContextFactory getOptimizationContextFactory();
+
+ default boolean skipJobCapacityAssignment() {
+ return false;
+ }
}