[ASTERIXDB-2015][IDX] Introduce Primary Index Optimization Rule
- user model changes: no
- storage format changes: no
- interface changes: no
details:
This is the optimization rule that will optimize aggregation queries
when only PKs are involved. The rule will use the primary index and
replace the dataset scan or unnest-map operator.
Change-Id: I3bbb2b5e1f25e61928d73b866e91c592ce0bf954
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2111
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dmitry.lychagin@couchbase.com>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
index a8d9ec0..5146993 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
@@ -79,6 +79,7 @@
import org.apache.asterix.optimizer.rules.UnnestToDataScanRule;
import org.apache.asterix.optimizer.rules.am.IntroduceJoinAccessMethodRule;
import org.apache.asterix.optimizer.rules.am.IntroduceLSMComponentFilterRule;
+import org.apache.asterix.optimizer.rules.am.IntroducePrimaryIndexForAggregationRule;
import org.apache.asterix.optimizer.rules.am.IntroduceSelectAccessMethodRule;
import org.apache.asterix.optimizer.rules.subplan.AsterixMoveFreeVariableOperatorOutOfSubplanRule;
import org.apache.asterix.optimizer.rules.subplan.InlineSubplanInputForNestedTupleSourceRule;
@@ -284,6 +285,7 @@
accessMethod.add(new IntroduceSelectAccessMethodRule());
accessMethod.add(new IntroduceJoinAccessMethodRule());
accessMethod.add(new IntroduceLSMComponentFilterRule());
+ accessMethod.add(new IntroducePrimaryIndexForAggregationRule());
accessMethod.add(new IntroduceSecondaryIndexInsertDeleteRule());
accessMethod.add(new RemoveUnusedOneToOneEquiJoinRule());
accessMethod.add(new PushSimilarityFunctionsBelowJoin());
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroducePrimaryIndexForAggregationRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroducePrimaryIndexForAggregationRule.java
new file mode 100644
index 0000000..7633f4c
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroducePrimaryIndexForAggregationRule.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.optimizer.rules.am;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.asterix.common.config.DatasetConfig;
+import org.apache.asterix.metadata.declared.DatasetDataSource;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.utils.ConstantExpressionUtil;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Pattern to match in the plan:
+ * ...
+ * ^
+ * |
+ * aggregate operator (local)
+ * ^
+ * |
+ * (assign operator)?
+ * ^
+ * |
+ * datasource scan operator OR unnest map operator using the dataset (when WHERE exists on PK)
+ * ^
+ * |
+ * ...
+ *
+ *
+ * The plan is transformed into:
+ * ...
+ * ^
+ * |
+ * aggregate operator (local)
+ * ^
+ * |
+ * (assign operator)?
+ * ^
+ * |
+ * unnest map operator over the primary index
+ * ^
+ * |
+ * ...
+ * This rule optimizes aggregation queries involving only PKs. It uses the primary index, if present.
+ * The primary index is a BTree index that only stores PKs. Therefore, if an aggregation query can be answered by
+ * only the PKs, this rule will be fired to use the primary index instead of doing a scan/range search over the dataset.
+ */
+public class IntroducePrimaryIndexForAggregationRule implements IAlgebraicRewriteRule {
+ private final List<Mutable<ILogicalOperator>> parents;
+
+ public IntroducePrimaryIndexForAggregationRule() {
+ parents = new ArrayList<>();
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ parents.add(opRef);
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ // remove yourself
+ parents.remove(parents.size() - 1);
+ // already fired this rule on this operator?
+ if (context.checkIfInDontApplySet(this, opRef.getValue())) {
+ return false;
+ }
+ /* only interested in local aggregate operator */
+ if (opRef.getValue().getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+ return false;
+ }
+ AggregateOperator localAggregateOperator = (AggregateOperator) opRef.getValue();
+ if (localAggregateOperator.isGlobal()) {
+ return false;
+ }
+ context.addToDontApplySet(this, opRef.getValue());
+ // find the data scan or unnest map
+ Pair<Mutable<ILogicalOperator>,Mutable<ILogicalOperator>> scanAndAssignOpRef =
+ findScanAndAssignOperator(localAggregateOperator,context.getMetadataProvider());
+ if (scanAndAssignOpRef == null) {
+ return false;
+ }
+ // find its primary index and replace datascan
+ boolean transformed =
+ replaceDatascan(localAggregateOperator,scanAndAssignOpRef, context);
+ if (transformed) {
+ OperatorPropertiesUtil.typeOpRec(opRef, context);
+ }
+ return transformed;
+ }
+
+ private Pair<Mutable<ILogicalOperator>,Mutable<ILogicalOperator>> findScanAndAssignOperator(
+ ILogicalOperator localAggregateOperator, IMetadataProvider metadataProvider) throws AlgebricksException {
+ Mutable<ILogicalOperator> scanOpRef = localAggregateOperator.getInputs().get(0);
+ Mutable<ILogicalOperator> assignOpRef = null;
+ // assign operator may or may not exist
+ if (scanOpRef.getValue().getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+ AssignOperator assignOperator = (AssignOperator) scanOpRef.getValue();
+ assignOpRef = new MutableObject<>(assignOperator);
+ scanOpRef = scanOpRef.getValue().getInputs().get(0);
+ }
+ // next operator must be datascan or unnest map using the dataset
+ if (scanOpRef.getValue().getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN &&
+ scanOpRef.getValue().getOperatorTag() != LogicalOperatorTag.UNNEST_MAP) {
+ return null;
+ }
+ if (scanOpRef.getValue().getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) {
+ // for unnest_map, check the index used is the primary index
+ UnnestMapOperator unnestMapOperator = (UnnestMapOperator) scanOpRef.getValue();
+ ILogicalExpression logicalExpression = unnestMapOperator.getExpressionRef().getValue();
+ if (logicalExpression.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return null;
+ }
+ AbstractFunctionCallExpression functionCallExpression = (AbstractFunctionCallExpression)logicalExpression;
+ if (functionCallExpression.getFunctionIdentifier() != BuiltinFunctions.INDEX_SEARCH) {
+ return null;
+ }
+ String indexName = ConstantExpressionUtil.getStringArgument(functionCallExpression,0);
+ String dataverseName = ConstantExpressionUtil.getStringArgument(functionCallExpression,2);
+ String datasetName = ConstantExpressionUtil.getStringArgument(functionCallExpression,3);
+ Index index = ((MetadataProvider)metadataProvider).getIndex(dataverseName, datasetName, indexName);
+ if (!index.isPrimaryIndex()) {
+ return null;
+ }
+ }
+ return Pair.of(scanOpRef,assignOpRef);
+ }
+
+ private boolean replaceDatascan(AggregateOperator localAggregateOperator,
+ Pair<Mutable<ILogicalOperator>,Mutable<ILogicalOperator>> scanAndAssignOpRef, IOptimizationContext context)
+ throws AlgebricksException {
+ /* find the primary index */
+ Mutable<ILogicalOperator> scanOperatorRef = scanAndAssignOpRef.getLeft();
+ Mutable<ILogicalOperator> assignOperatorRef = scanAndAssignOpRef.getRight();
+ AbstractScanOperator scanOperator = (AbstractScanOperator) scanOperatorRef.getValue();
+ BTreeJobGenParams originalBTreeParameters = new BTreeJobGenParams();
+ Pair<Dataset,Index> datasetAndIndex = findDatasetAndSecondaryPrimaryIndex(scanOperator,originalBTreeParameters,
+ context);
+ if (datasetAndIndex == null) {
+ return false;
+ }
+ Dataset dataset = datasetAndIndex.getLeft();
+ Index primaryIndex = datasetAndIndex.getRight();
+ /////// replace the operator. prepare the parameters of the BTree of the new unnestmap operator ///////
+ if (dataset.getDatasetType() == DatasetConfig.DatasetType.INTERNAL) {
+ /////// check usage of variables produced by scan operator in parents ///////
+ Set<LogicalVariable> variablesProducedByScanOp = getVariablesProducedByScanOp(scanOperator,
+ dataset.getPrimaryKeys().size(), scanOperator.getVariables().size());
+ boolean variablesAreUsed = scanOperatorVariablesAreUsed(localAggregateOperator, assignOperatorRef,
+ variablesProducedByScanOp);
+ if (variablesAreUsed) {
+ return false;
+ }
+ /////// initialize the secondary primary BTree parameters ///////
+ boolean retainInput;
+ BTreeJobGenParams newBTreeParameters;
+ if (scanOperator.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
+ retainInput = AccessMethodUtils.retainInputs(scanOperator.getVariables(), scanOperator, parents);
+ newBTreeParameters = new BTreeJobGenParams(primaryIndex.getIndexName(), DatasetConfig.IndexType.BTREE,
+ dataset.getDataverseName(), dataset.getDatasetName(), retainInput,
+ scanOperator.getInputs().get(0).getValue().getExecutionMode() ==
+ AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
+ List<LogicalVariable> empty = new ArrayList<>();
+ newBTreeParameters.setLowKeyInclusive(true);
+ newBTreeParameters.setHighKeyInclusive(true);
+ newBTreeParameters.setIsEqCondition(false);
+ newBTreeParameters.setLowKeyVarList(empty, 0, 0);
+ newBTreeParameters.setHighKeyVarList(empty, 0, 0);
+ } else {
+ retainInput = originalBTreeParameters.getRetainInput();
+ newBTreeParameters = new BTreeJobGenParams(primaryIndex.getIndexName(), DatasetConfig.IndexType.BTREE,
+ dataset.getDataverseName(), dataset.getDatasetName(), retainInput,
+ originalBTreeParameters.getRequiresBroadcast());
+ newBTreeParameters.setLowKeyInclusive(originalBTreeParameters.isLowKeyInclusive());
+ newBTreeParameters.setHighKeyInclusive(originalBTreeParameters.isHighKeyInclusive());
+ newBTreeParameters.setIsEqCondition(originalBTreeParameters.isEqCondition());
+ newBTreeParameters.setLowKeyVarList(originalBTreeParameters.getLowKeyVarList(), 0,
+ originalBTreeParameters.getLowKeyVarList().size());
+ newBTreeParameters.setHighKeyVarList(originalBTreeParameters.getHighKeyVarList(), 0,
+ originalBTreeParameters.getHighKeyVarList().size());
+ }
+ ARecordType recordType = (ARecordType) ((MetadataProvider)context.getMetadataProvider()).findType(dataset);
+ ARecordType metaRecordType =
+ (ARecordType) ((MetadataProvider)context.getMetadataProvider()).findMetaType(dataset);
+ // create the operator that will replace the dataset scan/search
+ AbstractUnnestMapOperator primaryIndexUnnestOperator =
+ (AbstractUnnestMapOperator) AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
+ metaRecordType, primaryIndex, scanOperator.getInputs().get(0).getValue(),
+ newBTreeParameters, context, true, retainInput, false);
+
+ // re-use the PK variables of the original scan operator
+ primaryIndexUnnestOperator.getVariables().clear();
+ for (int i = 0; i < dataset.getPrimaryKeys().size(); i++) {
+ primaryIndexUnnestOperator.getVariables().add(scanOperator.getVariables().get(i));
+ }
+ // now replace
+ scanOperatorRef.setValue(primaryIndexUnnestOperator);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Returns null if there is no primary index defined on the dataset
+ * @param scanOperator Scan or unnest-map operator
+ * @param originalBTreeParameters The BTree parameters if the operator is unnest-map
+ * @param context Needed to get the metadata provider and ask for the index
+ * @return The dataset and its primary index
+ * @throws AlgebricksException when there is a problem getting the dataset or its indexes from the metadata
+ */
+ private Pair<Dataset,Index> findDatasetAndSecondaryPrimaryIndex(AbstractScanOperator scanOperator,
+ BTreeJobGenParams originalBTreeParameters, IOptimizationContext context) throws AlgebricksException {
+ // #1. get the dataset
+ Dataset dataset;
+ // case 1: dataset scan
+ if (scanOperator.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
+ dataset = ((DatasetDataSource)((DataSourceScanOperator)scanOperator).getDataSource()).getDataset();
+ } else {
+ // case 2: dataset range search
+ AbstractFunctionCallExpression primaryIndexFunctionCall =
+ (AbstractFunctionCallExpression) ((UnnestMapOperator)scanOperator).getExpressionRef().getValue();
+ originalBTreeParameters.readFromFuncArgs(primaryIndexFunctionCall.getArguments());
+ if (originalBTreeParameters.isEqCondition()) {
+ return null;
+ }
+ dataset = ((MetadataProvider)context.getMetadataProvider()).findDataset(
+ originalBTreeParameters.getDataverseName(), originalBTreeParameters.getDatasetName());
+ }
+ // #2. get all indexes and look for the primary one
+ List<Index> indexes = ((MetadataProvider)context.getMetadataProvider()).getDatasetIndexes(
+ dataset.getDataverseName(), dataset.getDatasetName());
+ for (Index index : indexes) {
+ if (index.getKeyFieldNames().isEmpty()) {
+ return Pair.of(dataset,index);
+ }
+ }
+ return null;
+ }
+
+ private Set<LogicalVariable> getVariablesProducedByScanOp(AbstractScanOperator scanOperator, int startPosition,
+ int endPosition) {
+ Set<LogicalVariable> variableSet = new HashSet<>();
+ // starting after PK, collect the produced variables
+ for (int i = startPosition; i < endPosition; i++) {
+ variableSet.add(scanOperator.getVariables().get(i));
+ }
+ return variableSet;
+ }
+
+ private boolean scanOperatorVariablesAreUsed(AggregateOperator localAggregateOperator,
+ Mutable<ILogicalOperator> assignOperatorRef, Set<LogicalVariable> variablesProducedByScanOp)
+ throws AlgebricksException {
+ // collect variables used by parents operators
+ Set<LogicalVariable> variablesUsedByParents = new HashSet<>();
+ for (Mutable<ILogicalOperator> parent : parents) {
+ VariableUtilities.getUsedVariables(parent.getValue(), variablesUsedByParents);
+ }
+ // collect variables used by local aggregate operator
+ VariableUtilities.getUsedVariables(localAggregateOperator, variablesUsedByParents);
+ // collect variables used by assign operator, if exists
+ if (assignOperatorRef != null) {
+ VariableUtilities.getUsedVariables(assignOperatorRef.getValue(), variablesUsedByParents);
+ }
+ // checking...
+ for (LogicalVariable producedVariable : variablesProducedByScanOp) {
+ if (variablesUsedByParents.contains(producedVariable)) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/btree-index/btree-secondary-64.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/btree-index/btree-secondary-64.sqlpp
new file mode 100644
index 0000000..723a551
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/btree-index/btree-secondary-64.sqlpp
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : BTree Index verification test
+ * : This test is intended to verify that the secondary primary BTree index is used for aggregations
+ * : in the optimized query plan.
+ * Expected Result : Success
+ * Date : 31st Oct 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+write output to asterix_nc1:"rttest/btree-index_btree-secondary-64.adm";
+create type test.TestType as
+{
+ id : integer,
+ fname : string,
+ lname : string
+};
+
+create dataset testdst(TestType) primary key id;
+
+create primary index on testdst;
+
+select count(*) AS count from testdst;
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/btree-index/btree-secondary-65.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/btree-index/btree-secondary-65.sqlpp
new file mode 100644
index 0000000..2de8f5b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/btree-index/btree-secondary-65.sqlpp
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : BTree Index verification test
+ * : This test is intended to verify that the secondary primary BTree index is used for aggregations
+ * : in the optimized query plan.
+ * Expected Result : Success
+ * Date : 31st Oct 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+write output to asterix_nc1:"rttest/btree-index_btree-secondary-65.adm";
+create type test.TestType as
+{
+ id : integer,
+ fname : string,
+ lname : string
+};
+
+create dataset testdst(TestType) primary key id;
+
+create primary index on testdst;
+
+select count(*) AS count from testdst t where t.id > 3;
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/btree-index/btree-secondary-66.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/btree-index/btree-secondary-66.sqlpp
new file mode 100644
index 0000000..800beef
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/btree-index/btree-secondary-66.sqlpp
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : BTree Index verification test
+ * : This test is intended to verify that the secondary primary BTree index is used for aggregations
+ * : in the optimized query plan.
+ * Expected Result : Success
+ * Date : 31st Oct 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+write output to asterix_nc1:"rttest/btree-index_btree-secondary-66.adm";
+create type test.TestType as
+{
+ id : integer,
+ fname : string,
+ lname : string
+};
+
+create dataset testdst(TestType) primary key id;
+
+create primary index on testdst;
+
+select MAX(t.id) as maximum from testdst t;
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/btree-index/btree-secondary-67.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/btree-index/btree-secondary-67.sqlpp
new file mode 100644
index 0000000..89e6697
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries_sqlpp/btree-index/btree-secondary-67.sqlpp
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : BTree Index verification test
+ * : This test is intended to verify that the secondary primary BTree index is used for aggregations
+ * : in the optimized query plan.
+ * Expected Result : Success
+ * Date : 31st Oct 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+write output to asterix_nc1:"rttest/btree-index_btree-secondary-67.adm";
+create type test.TestType as
+{
+ id : integer,
+ fname : string,
+ lname : string
+};
+
+create dataset testdst(TestType) primary key id;
+
+create primary index on testdst;
+
+select count(t.id) as count from testdst t;
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-secondary-64.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-secondary-64.plan
new file mode 100644
index 0000000..d0280bc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-secondary-64.plan
@@ -0,0 +1,13 @@
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ -- STREAM_PROJECT |UNPARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- AGGREGATE |UNPARTITIONED|
+ -- RANDOM_MERGE_EXCHANGE |PARTITIONED|
+ -- AGGREGATE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-secondary-65.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-secondary-65.plan
new file mode 100644
index 0000000..011a15f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-secondary-65.plan
@@ -0,0 +1,14 @@
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ -- STREAM_PROJECT |UNPARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- AGGREGATE |UNPARTITIONED|
+ -- RANDOM_MERGE_EXCHANGE |PARTITIONED|
+ -- AGGREGATE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-secondary-66.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-secondary-66.plan
new file mode 100644
index 0000000..697008a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-secondary-66.plan
@@ -0,0 +1,11 @@
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ -- STREAM_PROJECT |UNPARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- AGGREGATE |UNPARTITIONED|
+ -- RANDOM_MERGE_EXCHANGE |PARTITIONED|
+ -- AGGREGATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-secondary-67.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-secondary-67.plan
new file mode 100644
index 0000000..697008a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-secondary-67.plan
@@ -0,0 +1,11 @@
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ -- STREAM_PROJECT |UNPARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- AGGREGATE |UNPARTITIONED|
+ -- RANDOM_MERGE_EXCHANGE |PARTITIONED|
+ -- AGGREGATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_parser_sqlpp/btree-index/btree-secondary-64.ast b/asterixdb/asterix-app/src/test/resources/optimizerts/results_parser_sqlpp/btree-index/btree-secondary-64.ast
new file mode 100644
index 0000000..ed47244
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_parser_sqlpp/btree-index/btree-secondary-64.ast
@@ -0,0 +1,30 @@
+DataverseUse test
+WriteOutputTo asterix_nc1:rttest/btree-index_btree-secondary-64.adm
+TypeDecl TestType [
+ open RecordType {
+ id : integer,
+ fname : string,
+ lname : string
+ }
+]
+DatasetDecl testdst(TestType) partitioned by [[id]]
+Query:
+SELECT [
+FunctionCall asterix.sql-count@1[
+ (
+ SELECT ELEMENT [
+ LiteralExpr [LONG] [1]
+ ]
+ FROM [ Variable [ Name=#1 ]
+ AS Variable [ Name=#2 ]
+ ]
+ )
+]
+count
+]
+FROM [ FunctionCall Metadata.dataset@1[
+ LiteralExpr [STRING] [testdst]
+ ]
+ AS Variable [ Name=$testdst ]
+]
+Group All
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_parser_sqlpp/btree-index/btree-secondary-65.ast b/asterixdb/asterix-app/src/test/resources/optimizerts/results_parser_sqlpp/btree-index/btree-secondary-65.ast
new file mode 100644
index 0000000..a2eabf4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_parser_sqlpp/btree-index/btree-secondary-65.ast
@@ -0,0 +1,39 @@
+DataverseUse test
+WriteOutputTo asterix_nc1:rttest/btree-index_btree-secondary-65.adm
+TypeDecl TestType [
+ open RecordType {
+ id : integer,
+ fname : string,
+ lname : string
+ }
+]
+DatasetDecl testdst(TestType) partitioned by [[id]]
+Query:
+SELECT [
+FunctionCall asterix.sql-count@1[
+ (
+ SELECT ELEMENT [
+ LiteralExpr [LONG] [1]
+ ]
+ FROM [ Variable [ Name=#1 ]
+ AS Variable [ Name=#2 ]
+ ]
+ )
+]
+count
+]
+FROM [ FunctionCall Metadata.dataset@1[
+ LiteralExpr [STRING] [testdst]
+ ]
+ AS Variable [ Name=$t ]
+]
+Where
+ OperatorExpr [
+ FieldAccessor [
+ Variable [ Name=$t ]
+ Field=id
+ ]
+ >
+ LiteralExpr [LONG] [3]
+ ]
+Group All
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_parser_sqlpp/btree-index/btree-secondary-66.ast b/asterixdb/asterix-app/src/test/resources/optimizerts/results_parser_sqlpp/btree-index/btree-secondary-66.ast
new file mode 100644
index 0000000..cb26007
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_parser_sqlpp/btree-index/btree-secondary-66.ast
@@ -0,0 +1,36 @@
+DataverseUse test
+WriteOutputTo asterix_nc1:rttest/btree-index_btree-secondary-66.adm
+TypeDecl TestType [
+ open RecordType {
+ id : integer,
+ fname : string,
+ lname : string
+ }
+]
+DatasetDecl testdst(TestType) partitioned by [[id]]
+Query:
+SELECT [
+FunctionCall asterix.sql-max@1[
+ (
+ SELECT ELEMENT [
+ FieldAccessor [
+ FieldAccessor [
+ Variable [ Name=#2 ]
+ Field=t
+ ]
+ Field=id
+ ]
+ ]
+ FROM [ Variable [ Name=#1 ]
+ AS Variable [ Name=#2 ]
+ ]
+ )
+]
+maximum
+]
+FROM [ FunctionCall Metadata.dataset@1[
+ LiteralExpr [STRING] [testdst]
+ ]
+ AS Variable [ Name=$t ]
+]
+Group All
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results_parser_sqlpp/btree-index/btree-secondary-67.ast b/asterixdb/asterix-app/src/test/resources/optimizerts/results_parser_sqlpp/btree-index/btree-secondary-67.ast
new file mode 100644
index 0000000..63a4d0b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results_parser_sqlpp/btree-index/btree-secondary-67.ast
@@ -0,0 +1,36 @@
+DataverseUse test
+WriteOutputTo asterix_nc1:rttest/btree-index_btree-secondary-67.adm
+TypeDecl TestType [
+ open RecordType {
+ id : integer,
+ fname : string,
+ lname : string
+ }
+]
+DatasetDecl testdst(TestType) partitioned by [[id]]
+Query:
+SELECT [
+FunctionCall asterix.sql-count@1[
+ (
+ SELECT ELEMENT [
+ FieldAccessor [
+ FieldAccessor [
+ Variable [ Name=#2 ]
+ Field=t
+ ]
+ Field=id
+ ]
+ ]
+ FROM [ Variable [ Name=#1 ]
+ AS Variable [ Name=#2 ]
+ ]
+ )
+]
+count
+]
+FROM [ FunctionCall Metadata.dataset@1[
+ LiteralExpr [STRING] [testdst]
+ ]
+ AS Variable [ Name=$t ]
+]
+Group All
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-01/btree-sec-primary-index-01.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-01/btree-sec-primary-index-01.1.ddl.sqlpp
new file mode 100644
index 0000000..53c5a21
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-01/btree-sec-primary-index-01.1.ddl.sqlpp
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing using the secondary primary index for aggregation
+ * Expected Result : Success
+ * Date : Oct 31 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.Emp as
+ closed {
+ id : bigint,
+ fname : string,
+ lname : string,
+ age : bigint,
+ dept : string
+};
+
+create dataset employee(Emp) primary key id;
+
+create primary index sec_primary_idx on employee ;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-01/btree-sec-primary-index-01.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-01/btree-sec-primary-index-01.2.update.sqlpp
new file mode 100644
index 0000000..a2b8188
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-01/btree-sec-primary-index-01.2.update.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing using the secondary primary index for aggregation
+ * Expected Result : Success
+ * Date : Oct 31 2017
+ */
+
+use test;
+
+
+load dataset employee using localfs ((`path`=`asterix_nc1://data/names.adm`),(`format`=`delimited-text`),(`delimiter`=`|`));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-01/btree-sec-primary-index-01.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-01/btree-sec-primary-index-01.3.query.sqlpp
new file mode 100644
index 0000000..5d47c41
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-01/btree-sec-primary-index-01.3.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing using the secondary primary index for aggregation
+ * Expected Result : Success
+ * Date : Oct 31 2017
+ */
+
+use test;
+
+select count(*) from employee;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-02/btree-sec-primary-index-02.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-02/btree-sec-primary-index-02.1.ddl.sqlpp
new file mode 100644
index 0000000..53c5a21
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-02/btree-sec-primary-index-02.1.ddl.sqlpp
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing using the secondary primary index for aggregation
+ * Expected Result : Success
+ * Date : Oct 31 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.Emp as
+ closed {
+ id : bigint,
+ fname : string,
+ lname : string,
+ age : bigint,
+ dept : string
+};
+
+create dataset employee(Emp) primary key id;
+
+create primary index sec_primary_idx on employee ;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-02/btree-sec-primary-index-02.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-02/btree-sec-primary-index-02.2.update.sqlpp
new file mode 100644
index 0000000..a2b8188
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-02/btree-sec-primary-index-02.2.update.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing using the secondary primary index for aggregation
+ * Expected Result : Success
+ * Date : Oct 31 2017
+ */
+
+use test;
+
+
+load dataset employee using localfs ((`path`=`asterix_nc1://data/names.adm`),(`format`=`delimited-text`),(`delimiter`=`|`));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-02/btree-sec-primary-index-02.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-02/btree-sec-primary-index-02.3.query.sqlpp
new file mode 100644
index 0000000..f4c22c7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-02/btree-sec-primary-index-02.3.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing using the secondary primary index for aggregation
+ * Expected Result : Success
+ * Date : Oct 31 2017
+ */
+
+use test;
+
+select count(*) from employee e where e.id > 500;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-03/btree-sec-primary-index-03.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-03/btree-sec-primary-index-03.1.ddl.sqlpp
new file mode 100644
index 0000000..53c5a21
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-03/btree-sec-primary-index-03.1.ddl.sqlpp
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing using the secondary primary index for aggregation
+ * Expected Result : Success
+ * Date : Oct 31 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.Emp as
+ closed {
+ id : bigint,
+ fname : string,
+ lname : string,
+ age : bigint,
+ dept : string
+};
+
+create dataset employee(Emp) primary key id;
+
+create primary index sec_primary_idx on employee ;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-03/btree-sec-primary-index-03.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-03/btree-sec-primary-index-03.2.update.sqlpp
new file mode 100644
index 0000000..a2b8188
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-03/btree-sec-primary-index-03.2.update.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing using the secondary primary index for aggregation
+ * Expected Result : Success
+ * Date : Oct 31 2017
+ */
+
+use test;
+
+
+load dataset employee using localfs ((`path`=`asterix_nc1://data/names.adm`),(`format`=`delimited-text`),(`delimiter`=`|`));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-03/btree-sec-primary-index-03.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-03/btree-sec-primary-index-03.3.query.sqlpp
new file mode 100644
index 0000000..8c55f86
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-03/btree-sec-primary-index-03.3.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing using the secondary primary index for aggregation
+ * Expected Result : Success
+ * Date : Oct 31 2017
+ */
+
+use test;
+
+select MAX(e.id) from employee e;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-04/btree-sec-primary-index-04.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-04/btree-sec-primary-index-04.1.ddl.sqlpp
new file mode 100644
index 0000000..53c5a21
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-04/btree-sec-primary-index-04.1.ddl.sqlpp
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing using the secondary primary index for aggregation
+ * Expected Result : Success
+ * Date : Oct 31 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.Emp as
+ closed {
+ id : bigint,
+ fname : string,
+ lname : string,
+ age : bigint,
+ dept : string
+};
+
+create dataset employee(Emp) primary key id;
+
+create primary index sec_primary_idx on employee ;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-04/btree-sec-primary-index-04.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-04/btree-sec-primary-index-04.2.update.sqlpp
new file mode 100644
index 0000000..a2b8188
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-04/btree-sec-primary-index-04.2.update.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing using the secondary primary index for aggregation
+ * Expected Result : Success
+ * Date : Oct 31 2017
+ */
+
+use test;
+
+
+load dataset employee using localfs ((`path`=`asterix_nc1://data/names.adm`),(`format`=`delimited-text`),(`delimiter`=`|`));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-04/btree-sec-primary-index-04.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-04/btree-sec-primary-index-04.3.query.sqlpp
new file mode 100644
index 0000000..b86d634
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/btree-sec-primary-index-04/btree-sec-primary-index-04.3.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing using the secondary primary index for aggregation
+ * Expected Result : Success
+ * Date : Oct 31 2017
+ */
+
+use test;
+
+select count(e.id) from employee e;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/btree-sec-primary-index-01/btree-sec-primary-index-01.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/btree-sec-primary-index-01/btree-sec-primary-index-01.3.adm
new file mode 100644
index 0000000..044b3f5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/btree-sec-primary-index-01/btree-sec-primary-index-01.3.adm
@@ -0,0 +1 @@
+{ "$1": 120 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/btree-sec-primary-index-02/btree-sec-primary-index-02.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/btree-sec-primary-index-02/btree-sec-primary-index-02.3.adm
new file mode 100644
index 0000000..12d64a1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/btree-sec-primary-index-02/btree-sec-primary-index-02.3.adm
@@ -0,0 +1 @@
+{ "$1": 88 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/btree-sec-primary-index-03/btree-sec-primary-index-03.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/btree-sec-primary-index-03/btree-sec-primary-index-03.3.adm
new file mode 100644
index 0000000..ac25b3f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/btree-sec-primary-index-03/btree-sec-primary-index-03.3.adm
@@ -0,0 +1 @@
+{ "$1": 9941 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/btree-sec-primary-index-04/btree-sec-primary-index-04.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/btree-sec-primary-index-04/btree-sec-primary-index-04.3.adm
new file mode 100644
index 0000000..044b3f5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/btree-sec-primary-index-04/btree-sec-primary-index-04.3.adm
@@ -0,0 +1 @@
+{ "$1": 120 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 0d7ef36..a55d435 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -3040,6 +3040,26 @@
</compilation-unit>
</test-case>
<test-case FilePath="index-selection">
+ <compilation-unit name="btree-sec-primary-index-01">
+ <output-dir compare="Text">btree-sec-primary-index-01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sec-primary-index-02">
+ <output-dir compare="Text">btree-sec-primary-index-02</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sec-primary-index-03">
+ <output-dir compare="Text">btree-sec-primary-index-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="btree-sec-primary-index-04">
+ <output-dir compare="Text">btree-sec-primary-index-04</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
<compilation-unit name="btree-index-composite-key-mixed-intervals">
<output-dir compare="Text">btree-index-composite-key-mixed-intervals</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index 8e1c34d..3fec73b 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -562,6 +562,11 @@
storageComponentProvider.getTransactionSubsystemProvider(), ResourceType.LSM_BTREE)
: new PrimaryIndexInstantSearchOperationCallbackFactory(jobId, getDatasetId(), primaryKeyFields,
storageComponentProvider.getTransactionSubsystemProvider(), ResourceType.LSM_BTREE);
+ } else if (index.getKeyFieldNames().isEmpty()) {
+ // this is the case where the index is secondary primary index and locking is required
+ // since the secondary primary index replaces the dataset index (which locks)
+ return new PrimaryIndexInstantSearchOperationCallbackFactory(jobId, getDatasetId(), primaryKeyFields,
+ storageComponentProvider.getTransactionSubsystemProvider(), ResourceType.LSM_BTREE);
}
return new SecondaryIndexSearchOperationCallbackFactory();
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatBuilder.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatBuilder.java
index 9c452bf..10e3432 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatBuilder.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatBuilder.java
@@ -223,6 +223,7 @@
if (value == null) {
newValue = "";
}
+ newValue = newValue.replace("\n", "\\n");
return new StringValue("\"" + newValue.replace("\"","\'").trim() + "\"");
}
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatGenerator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatGenerator.java
index 392bf44..1ea2d19 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatGenerator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/DotFormatGenerator.java
@@ -178,11 +178,13 @@
rightOperator = entry.getValue().getRight().getLeft();
source = leftOperator.getClass().getName().substring(
leftOperator.getClass().getName().lastIndexOf(".") + 1);
- sourceNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(leftOperator.toString()),
+ sourceNode = graphBuilder.createNode(
+ DotFormatBuilder.StringValue.of(leftOperator.getOperatorId().toString()),
DotFormatBuilder.StringValue.of(leftOperator.toString() + "-" + source));
destination = rightOperator.getClass().getName().substring(
rightOperator.getClass().getName().lastIndexOf(".") + 1);
- destinationNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(rightOperator.toString()),
+ destinationNode = graphBuilder.createNode(
+ DotFormatBuilder.StringValue.of(rightOperator.getOperatorId().toString()),
DotFormatBuilder.StringValue.of(rightOperator.toString() + "-" + destination));
graphBuilder.createEdge(sourceNode, destinationNode).setLabel(DotFormatBuilder.StringValue.of(edgeLabel));
}