[NO ISSUE][COMP] Reduce memory requirements for monitoring functions
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Queries that only invoke monitoring/metadata datasource functions
should run with minimal memory requirements
- The functions are: active_requests(), completed_requests(),
dataset_resources(), storage_components(), jobs(), and ping()
Change-Id: If7de53a5b476c8ce0d3fe485bf526a5d425e4c37
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3548
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dmitry.lychagin@couchbase.com>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java
index a850cf4..91dfe7f 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java
@@ -20,12 +20,15 @@
package org.apache.asterix.optimizer.rules;
import java.util.Set;
+import java.util.function.Predicate;
import org.apache.asterix.metadata.declared.DataSource;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.utils.MetadataConstants;
+import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.optimizer.base.AsterixOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
import org.apache.hyracks.algebricks.rewriter.rules.SetMemoryRequirementsRule;
@@ -34,7 +37,9 @@
/**
* This rule extends {@link SetMemoryRequirementsRule} and modifies its functionality as follows:
* <ul>
- * <li>It skips memory requirements configuration if the query operates only on metadata datasets.
+ * <li>It skips memory requirements configuration if the query operates only on metadata datasets and/or
+ * datasource functions annotated with
+ * {@link BuiltinFunctions.DataSourceFunctionProperty#MIN_MEMORY_BUDGET MIN_MEMORY_BUDGET} property.
* In this case operators will retain their default (minimal) memory requirements.
* </li>
* </ul>
@@ -49,11 +54,36 @@
private boolean forceMinMemoryBudget(AsterixOptimizationContext context) {
Int2ObjectMap<Set<DataSourceId>> dataSourceMap = context.getDataSourceMap();
- if (dataSourceMap.size() == 1) {
- Set<DataSourceId> dataSources = dataSourceMap.get(DataSource.Type.INTERNAL_DATASET);
- return dataSources != null && dataSources.stream()
- .allMatch(dsId -> MetadataConstants.METADATA_DATAVERSE_NAME.equals(dsId.getDataverseName()));
+ if (dataSourceMap.isEmpty()) {
+ return false;
}
- return false;
+ for (Int2ObjectMap.Entry<Set<DataSourceId>> me : dataSourceMap.int2ObjectEntrySet()) {
+ int dataSourceType = me.getIntKey();
+ Predicate<DataSourceId> dataSourceTest;
+ switch (dataSourceType) {
+ case DataSource.Type.INTERNAL_DATASET:
+ dataSourceTest = SetAsterixMemoryRequirementsRule::isMinMemoryBudgetDataset;
+ break;
+ case DataSource.Type.FUNCTION:
+ dataSourceTest = SetAsterixMemoryRequirementsRule::isMinMemoryBudgetFunction;
+ break;
+ default:
+ return false;
+ }
+ if (!me.getValue().stream().allMatch(dataSourceTest)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static boolean isMinMemoryBudgetDataset(DataSourceId dsId) {
+ return MetadataConstants.METADATA_DATAVERSE_NAME.equals(dsId.getDataverseName());
+ }
+
+ private static boolean isMinMemoryBudgetFunction(DataSourceId dsId) {
+ return BuiltinFunctions.builtinFunctionHasProperty(
+ new FunctionIdentifier(dsId.getDataverseName(), dsId.getDatasourceName()),
+ BuiltinFunctions.DataSourceFunctionProperty.MIN_MEMORY_BUDGET);
}
}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java
index eb5416b..2afd11c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java
@@ -29,8 +29,8 @@
public class ActiveRequestsDatasource extends FunctionDataSource {
- private static final DataSourceId ACTIVE_REQUESTS_DATASOURCE_ID = new DataSourceId(
- ActiveRequestsRewriter.ACTIVE_REQUESTS.getNamespace(), ActiveRequestsRewriter.ACTIVE_REQUESTS.getName());
+ private static final DataSourceId ACTIVE_REQUESTS_DATASOURCE_ID =
+ createDataSourceId(ActiveRequestsRewriter.ACTIVE_REQUESTS);
public ActiveRequestsDatasource(INodeDomain domain) throws AlgebricksException {
super(ACTIVE_REQUESTS_DATASOURCE_ID, domain);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java
index 4c3672b..1fe103e 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java
@@ -30,8 +30,7 @@
public class CompletedRequestsDatasource extends FunctionDataSource {
private static final DataSourceId COMPLETED_REQUESTS_DATASOURCE_ID =
- new DataSourceId(ActiveRequestsRewriter.ACTIVE_REQUESTS.getNamespace(),
- CompletedRequestsRewriter.COMPLETED_REQUESTS.getName());
+ createDataSourceId(CompletedRequestsRewriter.COMPLETED_REQUESTS);
public CompletedRequestsDatasource(INodeDomain domain) throws AlgebricksException {
super(COMPLETED_REQUESTS_DATASOURCE_ID, domain);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java
index e6b025a..ffd04ab 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java
@@ -27,11 +27,14 @@
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
public class DatasetResourcesDatasource extends FunctionDataSource {
+
+ private static final DataSourceId DATASET_RESOURCES_DATASOURCE_ID =
+ createDataSourceId(DatasetResourcesRewriter.DATASET_RESOURCES);
+
private final int datasetId;
public DatasetResourcesDatasource(INodeDomain domain, int datasetId) throws AlgebricksException {
- super(new DataSourceId(DatasetResourcesRewriter.DATASET_RESOURCES.getNamespace(),
- DatasetResourcesRewriter.DATASET_RESOURCES.getName()), domain);
+ super(DATASET_RESOURCES_DATASOURCE_ID, domain);
this.datasetId = datasetId;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
index 5981787..04e3eea 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
@@ -31,14 +31,15 @@
public class DumpIndexDatasource extends FunctionDataSource {
+ public static final DataSourceId DUMP_INDEX_DATASOURCE_ID = createDataSourceId(DumpIndexRewriter.DUMP_INDEX);
+
private final IndexDataflowHelperFactory indexDataflowHelperFactory;
private final RecordDescriptor recDesc;
private final IBinaryComparatorFactory[] comparatorFactories;
public DumpIndexDatasource(INodeDomain domain, IndexDataflowHelperFactory indexDataflowHelperFactory,
RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories) throws AlgebricksException {
- super(new DataSourceId(DumpIndexRewriter.DUMP_INDEX.getNamespace(), DumpIndexRewriter.DUMP_INDEX.getName()),
- domain);
+ super(DUMP_INDEX_DATASOURCE_ID, domain);
this.indexDataflowHelperFactory = indexDataflowHelperFactory;
this.recDesc = recDesc;
this.comparatorFactories = comparatorFactories;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java
index b92dc55..835a97d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java
@@ -28,8 +28,8 @@
public class JobSummariesDatasource extends FunctionDataSource {
- private static final DataSourceId JOB_SUMMARIES_DATASOURCE_ID = new DataSourceId(
- JobSummariesRewriter.JOBSUMMARIES.getNamespace(), JobSummariesRewriter.JOBSUMMARIES.getName());
+ private static final DataSourceId JOB_SUMMARIES_DATASOURCE_ID =
+ createDataSourceId(JobSummariesRewriter.JOBSUMMARIES);
public JobSummariesDatasource(INodeDomain domain) throws AlgebricksException {
super(JOB_SUMMARIES_DATASOURCE_ID, domain);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java
index 480f3c4..67d5e53 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java
@@ -28,8 +28,7 @@
public class PingDatasource extends FunctionDataSource {
- private static final DataSourceId PING_DATASOURCE_ID =
- new DataSourceId(PingRewriter.PING.getNamespace(), PingRewriter.PING.getName());
+ private static final DataSourceId PING_DATASOURCE_ID = createDataSourceId(PingRewriter.PING);
public PingDatasource(INodeDomain domain) throws AlgebricksException {
super(PING_DATASOURCE_ID, domain);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java
index 7245b88..b8a4167 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java
@@ -27,11 +27,14 @@
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
public class StorageComponentsDatasource extends FunctionDataSource {
+
+ public static final DataSourceId STORAGE_COMPONENTS_DATASOURCE_ID =
+ createDataSourceId(StorageComponentsRewriter.STORAGE_COMPONENTS);
+
private final int datasetId;
public StorageComponentsDatasource(INodeDomain domain, int datasetId) throws AlgebricksException {
- super(new DataSourceId(StorageComponentsRewriter.STORAGE_COMPONENTS.getNamespace(),
- StorageComponentsRewriter.STORAGE_COMPONENTS.getName()), domain);
+ super(STORAGE_COMPONENTS_DATASOURCE_ID, domain);
this.datasetId = datasetId;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
index 352fea8..c382fb4 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
@@ -54,8 +54,7 @@
* @return A DataSourceId that's based on the function details and its parameters
*/
private static DataSourceId createDataSourceId(double scalingFactor) {
- return new DataSourceId(TPCDSAllTablesDataGeneratorRewriter.TPCDS_ALL_TABLES_DATA_GENERATOR.getNamespace(),
- TPCDSAllTablesDataGeneratorRewriter.TPCDS_ALL_TABLES_DATA_GENERATOR.getName(),
+ return createDataSourceId(TPCDSAllTablesDataGeneratorRewriter.TPCDS_ALL_TABLES_DATA_GENERATOR,
Double.toString(scalingFactor));
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
index 968e289..df3903f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
@@ -58,8 +58,7 @@
* @return A DataSourceId that's based on the function details and its parameters
*/
private static DataSourceId createDataSourceId(String tableName, double scalingFactor) {
- return new DataSourceId(TPCDSSingleTableDataGeneratorRewriter.TPCDS_SINGLE_TABLE_DATA_GENERATOR.getNamespace(),
- TPCDSSingleTableDataGeneratorRewriter.TPCDS_SINGLE_TABLE_DATA_GENERATOR.getName(), tableName,
+ return createDataSourceId(TPCDSSingleTableDataGeneratorRewriter.TPCDS_SINGLE_TABLE_DATA_GENERATOR, tableName,
Double.toString(scalingFactor));
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java
index 26726f6..d6dc67c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java
@@ -48,18 +48,19 @@
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE, true);
BuiltinFunctions.addUnnestFun(DatasetResourcesRewriter.DATASET_RESOURCES, false);
BuiltinFunctions.addDatasourceFunction(DatasetResourcesRewriter.DATASET_RESOURCES,
- DatasetResourcesRewriter.INSTANCE);
+ DatasetResourcesRewriter.INSTANCE, BuiltinFunctions.DataSourceFunctionProperty.MIN_MEMORY_BUDGET);
// Dataset components function
BuiltinFunctions.addPrivateFunction(StorageComponentsRewriter.STORAGE_COMPONENTS,
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE, true);
BuiltinFunctions.addUnnestFun(StorageComponentsRewriter.STORAGE_COMPONENTS, false);
BuiltinFunctions.addDatasourceFunction(StorageComponentsRewriter.STORAGE_COMPONENTS,
- StorageComponentsRewriter.INSTANCE);
+ StorageComponentsRewriter.INSTANCE, BuiltinFunctions.DataSourceFunctionProperty.MIN_MEMORY_BUDGET);
// Ping function
BuiltinFunctions.addPrivateFunction(PingRewriter.PING,
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE, true);
BuiltinFunctions.addUnnestFun(PingRewriter.PING, true);
- BuiltinFunctions.addDatasourceFunction(PingRewriter.PING, PingRewriter.INSTANCE);
+ BuiltinFunctions.addDatasourceFunction(PingRewriter.PING, PingRewriter.INSTANCE,
+ BuiltinFunctions.DataSourceFunctionProperty.MIN_MEMORY_BUDGET);
// TPC-DS data generation function
BuiltinFunctions.addPrivateFunction(TPCDSSingleTableDataGeneratorRewriter.TPCDS_SINGLE_TABLE_DATA_GENERATOR,
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE, true);
@@ -75,18 +76,20 @@
BuiltinFunctions.addFunction(ActiveRequestsRewriter.ACTIVE_REQUESTS,
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE, true);
BuiltinFunctions.addUnnestFun(ActiveRequestsRewriter.ACTIVE_REQUESTS, true);
- BuiltinFunctions.addDatasourceFunction(ActiveRequestsRewriter.ACTIVE_REQUESTS, ActiveRequestsRewriter.INSTANCE);
+ BuiltinFunctions.addDatasourceFunction(ActiveRequestsRewriter.ACTIVE_REQUESTS, ActiveRequestsRewriter.INSTANCE,
+ BuiltinFunctions.DataSourceFunctionProperty.MIN_MEMORY_BUDGET);
// job-summaries function
BuiltinFunctions.addPrivateFunction(JobSummariesRewriter.JOBSUMMARIES,
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE, true);
BuiltinFunctions.addUnnestFun(JobSummariesRewriter.JOBSUMMARIES, true);
- BuiltinFunctions.addDatasourceFunction(JobSummariesRewriter.JOBSUMMARIES, JobSummariesRewriter.INSTANCE);
+ BuiltinFunctions.addDatasourceFunction(JobSummariesRewriter.JOBSUMMARIES, JobSummariesRewriter.INSTANCE,
+ BuiltinFunctions.DataSourceFunctionProperty.MIN_MEMORY_BUDGET);
// completed requests function
BuiltinFunctions.addFunction(CompletedRequestsRewriter.COMPLETED_REQUESTS,
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE, true);
BuiltinFunctions.addUnnestFun(CompletedRequestsRewriter.COMPLETED_REQUESTS, true);
BuiltinFunctions.addDatasourceFunction(CompletedRequestsRewriter.COMPLETED_REQUESTS,
- CompletedRequestsRewriter.INSTANCE);
+ CompletedRequestsRewriter.INSTANCE, BuiltinFunctions.DataSourceFunctionProperty.MIN_MEMORY_BUDGET);
// Dump index function
BuiltinFunctions.addPrivateFunction(DumpIndexRewriter.DUMP_INDEX,
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE, true);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/active_requests/active_requests.4.plans.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/active_requests/active_requests.4.plans.sqlpp
new file mode 100644
index 0000000..fe4447f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/active_requests/active_requests.4.plans.sqlpp
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test that a query that only uses active_requests()
+ * gets assigned a minimal memory budget
+ */
+
+-- param job:string=true
+
+SET `compiler.parallelism` "1";
+
+from active_requests() t
+select value t
+order by t
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.2.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.2.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.plans.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.plans.sqlpp
new file mode 100644
index 0000000..0e75d22
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.plans.sqlpp
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test that a query that only uses completed_requests()
+ * gets assigned a minimal memory budget
+ */
+
+-- param job:string=true
+
+SET `compiler.parallelism` "1";
+
+from completed_requests() t
+select value t
+order by t
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.6.plans.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.6.plans.sqlpp
new file mode 100644
index 0000000..65e7f5b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.6.plans.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test that a query that only uses dataset_resources()
+ * gets assigned a minimal memory budget
+ */
+
+-- param job:string=true
+
+SET `compiler.parallelism` "1";
+SET `import-private-functions` `true`;
+
+from dataset_resources('test','LineItem') t
+select value t
+order by t
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.7.plans.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.7.plans.sqlpp
new file mode 100644
index 0000000..c1aebd1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.7.plans.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test that a query that only uses storage_components()
+ * gets assigned a minimal memory budget
+ */
+
+-- param job:string=true
+
+SET `compiler.parallelism` "1";
+SET `import-private-functions` `true`;
+
+from storage_components('test','LineItem') t
+select value t
+order by t
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/jobs/jobs.2.plans.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/jobs/jobs.2.plans.sqlpp
new file mode 100644
index 0000000..9143a86
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/jobs/jobs.2.plans.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test that a query that only uses jobs()
+ * gets assigned a minimal memory budget
+ */
+
+-- param job:string=true
+
+SET `compiler.parallelism` "1";
+SET `import-private-functions` "true";
+
+from jobs() t
+select value t
+order by t
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.1.plans.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.1.plans.sqlpp
index 6862b8f..c2e086f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.1.plans.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.1.plans.sqlpp
@@ -17,6 +17,8 @@
* under the License.
*/
+/* Test that a metadata-only query gets assigned a minimal memory budget */
+
-- param job:string=true
SET `compiler.parallelism` "1";
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/ping/ping.2.plans.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/ping/ping.2.plans.sqlpp
new file mode 100644
index 0000000..6928d06
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/ping/ping.2.plans.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test that a query that only uses ping()
+ * get assigned a minimal memory budget
+ */
+
+-- param job:string=true
+
+SET `compiler.parallelism` "1";
+SET `import-private-functions` "true";
+
+from ping() t
+select value t
+order by t
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/active_requests/active_requests.4.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/active_requests/active_requests.4.regex
new file mode 100644
index 0000000..e2d166b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/active_requests/active_requests.4.regex
@@ -0,0 +1 @@
+/memory\D+229376/
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/completed_requests/completed_requests.3.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/completed_requests/completed_requests.3.regex
new file mode 100644
index 0000000..e2d166b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/completed_requests/completed_requests.3.regex
@@ -0,0 +1 @@
+/memory\D+229376/
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex
new file mode 100644
index 0000000..e2d166b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex
@@ -0,0 +1 @@
+/memory\D+229376/
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex
new file mode 100644
index 0000000..e2d166b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex
@@ -0,0 +1 @@
+/memory\D+229376/
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex
new file mode 100644
index 0000000..e2d166b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex
@@ -0,0 +1 @@
+/memory\D+229376/
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex
new file mode 100644
index 0000000..e2d166b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex
@@ -0,0 +1 @@
+/memory\D+229376/
\ No newline at end of file
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
index 1635507..8e79683 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
@@ -37,6 +37,7 @@
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
@@ -96,4 +97,15 @@
Set<String> ncs = new HashSet<>(Arrays.asList(allPartitions));
return new AlgebricksAbsolutePartitionConstraint(ncs.toArray(new String[ncs.size()]));
}
+
+ protected static DataSourceId createDataSourceId(FunctionIdentifier fid, String... parameters) {
+ int paramCount = parameters != null ? parameters.length : 0;
+ String[] components = new String[paramCount + 2];
+ components[0] = fid.getNamespace();
+ components[1] = fid.getName();
+ if (paramCount > 0) {
+ System.arraycopy(parameters, 0, components, 2, paramCount);
+ }
+ return new DataSourceId(components);
+ }
}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
index e95dfd0..e96d455 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
@@ -2984,8 +2984,16 @@
addUnnestFun(SUBSET_COLLECTION, false);
}
- public static void addDatasourceFunction(FunctionIdentifier fi, IFunctionToDataSourceRewriter transformer) {
- datasourceFunctions.put(getAsterixFunctionInfo(fi), transformer);
+ public enum DataSourceFunctionProperty implements BuiltinFunctionProperty {
+ /** Force minimum memory budget if a query only uses this function */
+ MIN_MEMORY_BUDGET
+ }
+
+ public static void addDatasourceFunction(FunctionIdentifier fi, IFunctionToDataSourceRewriter transformer,
+ DataSourceFunctionProperty... properties) {
+ IFunctionInfo finfo = getAsterixFunctionInfo(fi);
+ datasourceFunctions.put(finfo, transformer);
+ registerFunctionProperties(finfo, DataSourceFunctionProperty.class, properties);
}
public static IFunctionToDataSourceRewriter getDatasourceTransformer(FunctionIdentifier fi) {