ASTERIXDB-1760: support per-query customizations for compilation parameters.
Change-Id: I6e18f3f7706e574553a02e15a39daddda3c413b2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1458
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 4140f9a..90a8599 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -35,7 +35,9 @@
import org.apache.asterix.app.result.ResultUtil;
import org.apache.asterix.common.config.CompilerProperties;
import org.apache.asterix.common.config.ExternalProperties;
+import org.apache.asterix.common.config.IPropertyInterpreter;
import org.apache.asterix.common.config.OptimizationConfUtil;
+import org.apache.asterix.common.config.PropertyInterpreters;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
@@ -59,9 +61,9 @@
import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
import org.apache.asterix.runtime.util.AppContextInfo;
import org.apache.asterix.transaction.management.service.transaction.JobIdFactory;
+import org.apache.asterix.translator.SessionConfig;
import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
import org.apache.asterix.translator.IStatementExecutor.Stats;
-import org.apache.asterix.translator.SessionConfig;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -208,9 +210,13 @@
CompilerProperties compilerProperties = AppContextInfo.INSTANCE.getCompilerProperties();
int frameSize = compilerProperties.getFrameSize();
- int sortFrameLimit = (int) (compilerProperties.getSortMemorySize() / frameSize);
- int groupFrameLimit = (int) (compilerProperties.getGroupMemorySize() / frameSize);
- int joinFrameLimit = (int) (compilerProperties.getJoinMemorySize() / frameSize);
+ Map<String, String> querySpecificConfig = metadataProvider.getConfig();
+ int sortFrameLimit = getFrameLimit(querySpecificConfig.get(CompilerProperties.COMPILER_SORTMEMORY_KEY),
+ compilerProperties.getSortMemorySize(), frameSize);
+ int groupFrameLimit = getFrameLimit(querySpecificConfig.get(CompilerProperties.COMPILER_GROUPMEMORY_KEY),
+ compilerProperties.getGroupMemorySize(), frameSize);
+ int joinFrameLimit = getFrameLimit(querySpecificConfig.get(CompilerProperties.COMPILER_JOINMEMORY_KEY),
+ compilerProperties.getJoinMemorySize(), frameSize);
OptimizationConfUtil.getPhysicalOptimizationConfig().setFrameSize(frameSize);
OptimizationConfUtil.getPhysicalOptimizationConfig().setMaxFramesExternalSort(sortFrameLimit);
OptimizationConfUtil.getPhysicalOptimizationConfig().setMaxFramesExternalGroupBy(groupFrameLimit);
@@ -230,7 +236,8 @@
builder.setMissableTypeComputer(MissableTypeComputer.INSTANCE);
builder.setConflictingTypeResolver(ConflictingTypeResolver.INSTANCE);
- int parallelism = compilerProperties.getParallelism();
+ int parallelism = getParallelism(querySpecificConfig.get(CompilerProperties.COMPILER_PARALLELISM_KEY),
+ compilerProperties.getParallelism());
builder.setClusterLocations(parallelism == CompilerProperties.COMPILER_PARALLELISM_AS_STORAGE
? metadataProvider.getClusterLocations() : getComputationLocations(clusterInfoCollector, parallelism));
@@ -404,4 +411,19 @@
throw new AlgebricksException(e);
}
}
+
+ // Gets the frame limit.
+ private int getFrameLimit(String parameter, long memBudgetInConfiguration, int frameSize) {
+ IPropertyInterpreter<Long> longBytePropertyInterpreter = PropertyInterpreters.getLongBytePropertyInterpreter();
+ long memBudget = parameter == null ? memBudgetInConfiguration
+ : longBytePropertyInterpreter.interpret(parameter);
+ return (int) (memBudget / frameSize);
+ }
+
+ // Gets the parallelism parameter.
+ private int getParallelism(String parameter, int parallelismInConfiguration) {
+ IPropertyInterpreter<Integer> integerIPropertyInterpreter = PropertyInterpreters
+ .getIntegerPropertyInterpreter();
+ return parameter == null ? parallelismInConfiguration : integerIPropertyInterpreter.interpret(parameter);
+ }
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_load_20M/big_object_load_20M.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_load_20M/big_object_load_20M.2.update.aql
index f384e96..39842e4 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_load_20M/big_object_load_20M.2.update.aql
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_load_20M/big_object_load_20M.2.update.aql
@@ -26,8 +26,8 @@
use dataverse testdv2;
-set hash_merge "true"
+set "compiler.sortmemory" "64MB"
load dataset testds
using localfs
-(("path"="asterix_nc1://target/data/big-object/big_object_20M.adm"),("format"="adm")) pre-sorted;
+(("path"="asterix_nc1://target/data/big-object/big_object_20M.adm"),("format"="adm"));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.1.ddl.sqlpp
new file mode 100644
index 0000000..e81fa19
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.1.ddl.sqlpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : big_object_bulkload.aql
+ * Description : bulkload insert of large objects
+ * Expected Result : Success
+ * Date : 20th April 2016
+ */
+
+drop dataverse testdv2 if exists;
+create dataverse testdv2;
+use testdv2;
+
+create type testtype as closed {
+ id: int64,
+ name: string,
+ hobbies: {{string}}
+}
+
+create dataset testds(testtype) primary key id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.2.update.sqlpp
new file mode 100644
index 0000000..09f3a69
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.2.update.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ *
+ * Big object (20 MB) loading test
+ * Expected result: success
+ *
+ */
+
+use testdv2;
+
+set `compiler.sortmemory` "64MB"
+
+load dataset testds
+using localfs
+(("path"="asterix_nc1://target/data/big-object/big_object_20M.adm"),("format"="adm"));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.3.query.sqlpp
new file mode 100644
index 0000000..4730ac8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.3.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use testdv2;
+
+select value d
+from testds d
+where d.id=1;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.1.ddl.sqlpp
new file mode 100644
index 0000000..92698ab
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.1.ddl.sqlpp
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse tpch if exists;
+create dataverse tpch;
+
+use tpch;
+
+
+create type tpch.LineItemType as
+ closed {
+ l_orderkey : bigint,
+ l_partkey : bigint,
+ l_suppkey : bigint,
+ l_linenumber : bigint,
+ l_quantity : double,
+ l_extendedprice : double,
+ l_discount : double,
+ l_tax : double,
+ l_returnflag : string,
+ l_linestatus : string,
+ l_shipdate : string,
+ l_commitdate : string,
+ l_receiptdate : string,
+ l_shipinstruct : string,
+ l_shipmode : string,
+ l_comment : string
+}
+
+create dataset LineItem(LineItemType) primary key l_orderkey,l_linenumber;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.2.update.sqlpp
new file mode 100644
index 0000000..5fe734c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.2.update.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use tpch;
+
+
+load dataset LineItem using localfs ((`path`=`asterix_nc1://data/tpch0.001/lineitem.tbl`),(`format`=`delimited-text`),(`delimiter`=`|`)) pre-sorted;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.3.query.sqlpp
new file mode 100644
index 0000000..a4876a2d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpch-sql-sugar/q01_pricing_summary_report_parallelism/q01_pricing_summary_report_parallelism.3.query.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE tpch;
+
+SET `compiler.parallelism` "5"
+
+SELECT l_returnflag,
+ l_linestatus,
+ sum(l_quantity) AS sum_qty,
+ sum(l_extendedprice) AS sum_base_price,
+ sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price,
+ sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge,
+ avg(l_quantity) AS ave_qty,
+ avg(l_extendedprice) AS ave_price,
+ avg(l_discount) AS ave_disc,
+ count(1) AS count_order
+FROM LineItem
+WHERE l_shipdate <= '1998-09-02'
+/* +hash */
+GROUP BY l_returnflag, l_linestatus
+ORDER BY l_returnflag, l_linestatus
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 0aaf8c3..9bd2126 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -6822,6 +6822,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="tpch-sql-sugar">
+ <compilation-unit name="q01_pricing_summary_report_parallelism">
+ <output-dir compare="Text">q01_pricing_summary_report_nt</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpch-sql-sugar">
<compilation-unit name="q20_potential_part_promotion">
<output-dir compare="Text">q20_potential_part_promotion</output-dir>
</compilation-unit>
@@ -7824,6 +7829,11 @@
<output-dir compare="Text">big_object_join</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="big-object">
+ <compilation-unit name="big_object_load_20M">
+ <output-dir compare="Text">big_object_load_20M</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="external-indexing">
<test-case FilePath="external-indexing">
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 4e18bde..3710e54b0 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -25,19 +25,19 @@
public class CompilerProperties extends AbstractProperties {
- private static final String COMPILER_SORTMEMORY_KEY = "compiler.sortmemory";
+ public static final String COMPILER_SORTMEMORY_KEY = "compiler.sortmemory";
private static final long COMPILER_SORTMEMORY_DEFAULT = StorageUtil.getSizeInBytes(32, MEGABYTE);
- private static final String COMPILER_GROUPMEMORY_KEY = "compiler.groupmemory";
+ public static final String COMPILER_GROUPMEMORY_KEY = "compiler.groupmemory";
private static final long COMPILER_GROUPMEMORY_DEFAULT = StorageUtil.getSizeInBytes(32, MEGABYTE);
- private static final String COMPILER_JOINMEMORY_KEY = "compiler.joinmemory";
+ public static final String COMPILER_JOINMEMORY_KEY = "compiler.joinmemory";
private static final long COMPILER_JOINMEMORY_DEFAULT = StorageUtil.getSizeInBytes(32, MEGABYTE);
private static final String COMPILER_FRAMESIZE_KEY = "compiler.framesize";
private static final int COMPILER_FRAMESIZE_DEFAULT = StorageUtil.getSizeInBytes(32, KILOBYTE);
- private static final String COMPILER_PARALLELISM_KEY = "compiler.parallelism";
+ public static final String COMPILER_PARALLELISM_KEY = "compiler.parallelism";
public static final int COMPILER_PARALLELISM_AS_STORAGE = 0;
private static final String COMPILER_PREGELIX_HOME = "compiler.pregelix.home";