Intersect the secondary indexes before primary search
Change-Id: Ie167918fb23e39c8728840e4a90c1b85bf1bde85
Reviewed-on: https://asterix-gerrit.ics.uci.edu/578
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Jianfeng Jia <jianfeng.jia@gmail.com>
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
index 9a82309..238eaa3 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
@@ -42,6 +42,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
@@ -234,6 +235,11 @@
}
@Override
+ public Void visitIntersectOperator(IntersectOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
public Void visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
return null;
}
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
index 7a36db8..1531b8a 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
@@ -141,14 +141,21 @@
* Simply picks the first index that it finds. TODO: Improve this decision
* process by making it more systematic.
*/
- protected Pair<IAccessMethod, Index> chooseIndex(Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) {
+ protected Pair<IAccessMethod, Index> chooseBestIndex(Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) {
+ List<Pair<IAccessMethod, Index>> list = chooseAllIndex(analyzedAMs);
+ return list.isEmpty() ? null : list.get(0);
+ }
+
+ protected List<Pair<IAccessMethod, Index>> chooseAllIndex(
+ Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) {
+ List<Pair<IAccessMethod, Index>> result = new ArrayList<Pair<IAccessMethod, Index>>();
Iterator<Map.Entry<IAccessMethod, AccessMethodAnalysisContext>> amIt = analyzedAMs.entrySet().iterator();
while (amIt.hasNext()) {
Map.Entry<IAccessMethod, AccessMethodAnalysisContext> amEntry = amIt.next();
AccessMethodAnalysisContext analysisCtx = amEntry.getValue();
Iterator<Map.Entry<Index, List<Pair<Integer, Integer>>>> indexIt = analysisCtx.indexExprsAndVars.entrySet()
.iterator();
- if (indexIt.hasNext()) {
+ while (indexIt.hasNext()) {
Map.Entry<Index, List<Pair<Integer, Integer>>> indexEntry = indexIt.next();
// To avoid a case where the chosen access method and a chosen
// index type is different.
@@ -161,24 +168,24 @@
// LENGTH_PARTITIONED_NGRAM_INVIX]
IAccessMethod chosenAccessMethod = amEntry.getKey();
Index chosenIndex = indexEntry.getKey();
- boolean isKeywordOrNgramIndexChosen = false;
- if (chosenIndex.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX
+ boolean isKeywordOrNgramIndexChosen =
+ chosenIndex.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX
|| chosenIndex.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX
|| chosenIndex.getIndexType() == IndexType.SINGLE_PARTITION_WORD_INVIX
- || chosenIndex.getIndexType() == IndexType.SINGLE_PARTITION_NGRAM_INVIX)
- isKeywordOrNgramIndexChosen = true;
- if ((chosenAccessMethod == BTreeAccessMethod.INSTANCE && chosenIndex.getIndexType() != IndexType.BTREE)
+ || chosenIndex.getIndexType() == IndexType.SINGLE_PARTITION_NGRAM_INVIX;
+
+ if ((chosenAccessMethod == BTreeAccessMethod.INSTANCE && chosenIndex.getIndexType() == IndexType.BTREE)
|| (chosenAccessMethod == RTreeAccessMethod.INSTANCE
- && chosenIndex.getIndexType() != IndexType.RTREE)
- || (chosenAccessMethod == InvertedIndexAccessMethod.INSTANCE && !isKeywordOrNgramIndexChosen)) {
- continue;
+ && chosenIndex.getIndexType() == IndexType.RTREE)
+ || (chosenAccessMethod == InvertedIndexAccessMethod.INSTANCE && isKeywordOrNgramIndexChosen)) {
+ result.add(new Pair<IAccessMethod, Index>(chosenAccessMethod, chosenIndex));
}
- return new Pair<IAccessMethod, Index>(chosenAccessMethod, chosenIndex);
}
}
- return null;
+ return result;
}
+
/**
* Removes irrelevant access methods candidates, based on whether the
* expressions in the query match those in the index. For example, some
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
index dd6784e..370b90d 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
@@ -59,9 +59,11 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
/**
* Class for helping rewrite rules to choose and apply BTree indexes.
@@ -123,7 +125,7 @@
IOptimizationContext context) throws AlgebricksException {
SelectOperator select = (SelectOperator) selectRef.getValue();
Mutable<ILogicalExpression> conditionRef = select.getCondition();
- ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(selectRef, conditionRef, subTree, null,
+ ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(conditionRef, subTree, null,
chosenIndex, analysisCtx, false, false, false, context);
if (primaryIndexUnnestOp == null) {
return false;
@@ -184,7 +186,7 @@
newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0);
}
- ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(joinRef, conditionRef, indexSubTree,
+ ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(conditionRef, indexSubTree,
probeSubTree, chosenIndex, analysisCtx, true, isLeftOuterJoin, true, context);
if (primaryIndexUnnestOp == null) {
return false;
@@ -210,8 +212,8 @@
return true;
}
- private ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalOperator> topOpRef,
- Mutable<ILogicalExpression> conditionRef, OptimizableOperatorSubTree indexSubTree,
+ @Override
+ public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef, OptimizableOperatorSubTree indexSubTree,
OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context)
throws AlgebricksException {
@@ -470,7 +472,8 @@
// Assign operator that sets the constant secondary-index search-key fields if necessary.
AssignOperator assignConstantSearchKeys = new AssignOperator(assignKeyVarList, assignKeyExprList);
// Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input).
- assignConstantSearchKeys.getInputs().add(dataSourceOp.getInputs().get(0));
+ assignConstantSearchKeys.getInputs().add(new MutableObject<ILogicalOperator>(
+ OperatorManipulationUtil.deepCopyWithExcutionMode(dataSourceOp.getInputs().get(0).getValue())));
assignConstantSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode());
inputOp = assignConstantSearchKeys;
} else {
@@ -489,16 +492,11 @@
ExternalDataLookupOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(
dataSourceOp, dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex, retainInput,
retainNull);
- indexSubTree.dataSourceRef.setValue(externalDataAccessOp);
return externalDataAccessOp;
} else if (!isPrimaryIndex) {
primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType,
secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
- // Replace the datasource scan with the new plan rooted at
- // primaryIndexUnnestMap.
- indexSubTree.dataSourceRef.setValue(primaryIndexUnnestOp);
-
// Adds equivalence classes --- one equivalent class between a primary key
// variable and a record field-access expression.
EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(primaryIndexUnnestOp,
@@ -678,4 +676,15 @@
// No additional analysis required for BTrees.
return true;
}
+
+ @Override
+ public String getName() {
+ return "BTREE_ACCESS_METHOD";
+ }
+
+ @Override
+ public int compareTo(IAccessMethod o) {
+ return this.getName().compareTo(o.getName());
+ }
+
}
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java
index 6538ead..5691d57 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java
@@ -23,12 +23,14 @@
import org.apache.asterix.metadata.entities.Index;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import org.apache.hyracks.data.std.api.IDataOutputProvider;
/**
* Interface that an access method should implement to work with the rewrite
@@ -36,7 +38,7 @@
* methods for analyzing a select/join condition, and for rewriting the plan
* with a given index.
*/
-public interface IAccessMethod {
+public interface IAccessMethod extends Comparable<IAccessMethod>{
/**
* @return A list of function identifiers that are optimizable by this
@@ -83,6 +85,15 @@
OptimizableOperatorSubTree subTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
IOptimizationContext context) throws AlgebricksException;
+ public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef,
+ OptimizableOperatorSubTree indexSubTree,
+ OptimizableOperatorSubTree probeSubTree,
+ Index chosenIndex,
+ AccessMethodAnalysisContext analysisCtx,
+ boolean retainInput, boolean retainNull, boolean requiresBroadcast,
+ IOptimizationContext context)
+ throws AlgebricksException;
+
/**
* Applies the plan transformation to use chosenIndex to optimize a join query.
* In the case of a LeftOuterJoin, there may or may not be a needed groupby operation
@@ -100,4 +111,6 @@
*/
public boolean exprIsOptimizable(Index index, IOptimizableFuncExpr optFuncExpr) throws AlgebricksException;
+ public String getName();
+
}
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
index 13f7119..fce84d1 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
@@ -145,7 +145,7 @@
removeIndexCandidatesFromOuterBranch(analyzedAMs);
// Choose an index from the inner branch that will be used.
- Pair<IAccessMethod, Index> chosenIndex = chooseIndex(analyzedAMs);
+ Pair<IAccessMethod, Index> chosenIndex = chooseBestIndex(analyzedAMs);
if (chosenIndex == null) {
context.addToDontApplySet(this, join);
return false;
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
index 40e8712..405e4aa 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
@@ -18,13 +18,17 @@
*/
package org.apache.asterix.optimizer.rules.am;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
+import java.util.TreeMap;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.entities.Index;
import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -32,10 +36,14 @@
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
@@ -62,6 +70,9 @@
* 3. Check metadata to see if there are applicable indexes.
* 4. Choose an index to apply (for now only a single index will be chosen).
* 5. Rewrite plan using index (delegated to IAccessMethods).
+ * If there are multiple secondary index access path available, we will use the intersection operator to get the
+ * intersected primary key from all the secondary indexes. The detailed documentation is here
+ * https://cwiki.apache.org/confluence/display/ASTERIXDB/Intersect+multiple+secondary+index
*/
public class IntroduceSelectAccessMethodRule extends AbstractIntroduceAccessMethodRule {
@@ -94,7 +105,7 @@
}
// Analyze select condition.
- Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs = new HashMap<IAccessMethod, AccessMethodAnalysisContext>();
+ Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs = new TreeMap<IAccessMethod, AccessMethodAnalysisContext>();
if (!analyzeCondition(selectCond, subTree.assignsAndUnnests, analyzedAMs, context, typeEnvironment)) {
return false;
}
@@ -108,16 +119,15 @@
pruneIndexCandidates(analyzedAMs, context, typeEnvironment);
// Choose index to be applied.
- Pair<IAccessMethod, Index> chosenIndex = chooseIndex(analyzedAMs);
- if (chosenIndex == null) {
+ List<Pair<IAccessMethod, Index>> chosenIndexes = chooseAllIndex(analyzedAMs);
+ if (chosenIndexes == null || chosenIndexes.size() == 0) {
context.addToDontApplySet(this, select);
return false;
}
// Apply plan transformation using chosen index.
- AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(chosenIndex.first);
- boolean res = chosenIndex.first.applySelectPlanTransformation(selectRef, subTree, chosenIndex.second,
- analysisCtx, context);
+ boolean res = intersectAllSecondaryIndexes(chosenIndexes, analyzedAMs, context);
+
if (res) {
OperatorPropertiesUtil.typeOpRec(opRef, context);
}
@@ -125,6 +135,76 @@
return res;
}
+ private boolean intersectAllSecondaryIndexes(List<Pair<IAccessMethod, Index>> chosenIndexes,
+ Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs, IOptimizationContext context)
+ throws AlgebricksException {
+ Pair<IAccessMethod, Index> chosenIndex = null;
+ Optional<Pair<IAccessMethod, Index>> primaryIndex = chosenIndexes.stream()
+ .filter(pair -> pair.second.isPrimaryIndex()).findFirst();
+ if (chosenIndexes.size() == 1) {
+ chosenIndex = chosenIndexes.get(0);
+ } else if (primaryIndex.isPresent()) {
+ // one primary + secondary indexes, choose the primary index directly.
+ chosenIndex = primaryIndex.get();
+ }
+ if (chosenIndex != null) {
+ AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(chosenIndex.first);
+ return chosenIndex.first.applySelectPlanTransformation(selectRef, subTree, chosenIndex.second, analysisCtx,
+ context);
+ }
+
+ // Intersect all secondary indexes, and postpone the primary index search.
+ Mutable<ILogicalExpression> conditionRef = select.getCondition();
+
+ List<ILogicalOperator> subRoots = new ArrayList<>();
+ for (Pair<IAccessMethod, Index> pair : chosenIndexes) {
+ AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(pair.first);
+ subRoots.add(pair.first.createSecondaryToPrimaryPlan(conditionRef, subTree, null, pair.second, analysisCtx,
+ true, true, false, context));
+ }
+ ILogicalOperator primaryUnnest = connectAll2ndarySearchPlanWithIntersect(subRoots, context);
+
+ subTree.dataSourceRef.setValue(primaryUnnest);
+ return primaryUnnest != null;
+ }
+
+ private ILogicalOperator connectAll2ndarySearchPlanWithIntersect(List<ILogicalOperator> subRoots,
+ IOptimizationContext context) throws AlgebricksException {
+ ILogicalOperator lop = subRoots.get(0);
+ List<List<LogicalVariable>> inputVars = new ArrayList<>(subRoots.size());
+ for (int i = 0; i < subRoots.size(); i++) {
+ if (lop.getOperatorTag() != subRoots.get(i).getOperatorTag()) {
+ throw new AlgebricksException("The data source root should have the same operator type");
+ }
+ if (lop.getInputs().size() != 1) {
+ throw new AlgebricksException("The primary search has multiple input");
+ }
+
+ ILogicalOperator curRoot = subRoots.get(i);
+ OrderOperator order = (OrderOperator) curRoot.getInputs().get(0).getValue();
+ List<LogicalVariable> orderedColumn = new ArrayList<>(order.getOrderExpressions().size());
+ for (Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>> orderExpression : order
+ .getOrderExpressions()) {
+ if (orderExpression.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ throw new AlgebricksException("It should not happen, the order by expression is not variables");
+ }
+ VariableReferenceExpression orderedVar = (VariableReferenceExpression) orderExpression.second
+ .getValue();
+ orderedColumn.add(orderedVar.getVariableReference());
+ }
+ inputVars.add(orderedColumn);
+ }
+
+ List<LogicalVariable> outputVar = inputVars.get(0);
+ IntersectOperator intersect = new IntersectOperator(outputVar, inputVars);
+ for (ILogicalOperator secondarySearch : subRoots) {
+ intersect.getInputs().add(secondarySearch.getInputs().get(0));
+ }
+ context.computeAndSetTypeEnvironmentForOperator(intersect);
+ lop.getInputs().set(0, new MutableObject<>(intersect));
+ return lop;
+ }
+
protected boolean matchesOperatorPattern(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
// First check that the operator is a select and its condition is a function call.
AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
index d587649..d8e4c6a 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
@@ -66,6 +66,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
@@ -74,6 +75,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.LogicalOperatorDeepCopyWithNewVariablesVisitor;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifierFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.search.ConjunctiveEditDistanceSearchModifierFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.search.ConjunctiveListEditDistanceSearchModifierFactory;
@@ -359,10 +361,14 @@
return false;
}
- private ILogicalOperator createSecondaryToPrimaryPlan(OptimizableOperatorSubTree indexSubTree,
- OptimizableOperatorSubTree probeSubTree, Index chosenIndex, IOptimizableFuncExpr optFuncExpr,
+ @Override
+ public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef,
+ OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex,
+ AccessMethodAnalysisContext analysisCtx,
boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context)
throws AlgebricksException {
+
+ IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
Dataset dataset = indexSubTree.dataset;
ARecordType recordType = indexSubTree.recordType;
// we made sure indexSubTree has datasource scan
@@ -390,7 +396,8 @@
// Assign operator that sets the secondary-index search-key fields.
inputOp = new AssignOperator(keyVarList, keyExprList);
// Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input).
- inputOp.getInputs().add(dataSourceScan.getInputs().get(0));
+ inputOp.getInputs().add(new MutableObject<>(
+ OperatorManipulationUtil.deepCopyWithExcutionMode(dataSourceScan.getInputs().get(0).getValue())));
inputOp.setExecutionMode(dataSourceScan.getExecutionMode());
} else {
// We are optimizing a join. Add the input variable to the secondaryIndexFuncArgs.
@@ -429,8 +436,7 @@
public boolean applySelectPlanTransformation(Mutable<ILogicalOperator> selectRef,
OptimizableOperatorSubTree subTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
IOptimizationContext context) throws AlgebricksException {
- IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
- ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(subTree, null, chosenIndex, optFuncExpr, false,
+ ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null, subTree, null, chosenIndex, analysisCtx, false,
false, false, context);
// Replace the datasource scan with the new plan rooted at primaryIndexUnnestMap.
subTree.dataSourceRef.setValue(indexPlanRootOp);
@@ -511,8 +517,8 @@
probeSubTree.root = newProbeRootRef.getValue();
}
// Create regular indexed-nested loop join path.
- ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(indexSubTree, probeSubTree, chosenIndex,
- optFuncExpr, true, isLeftOuterJoin, true, context);
+ ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null, indexSubTree, probeSubTree, chosenIndex,
+ analysisCtx, true, isLeftOuterJoin, true, context);
indexSubTree.dataSourceRef.setValue(indexPlanRootOp);
// Change join into a select with the same condition.
@@ -1181,4 +1187,14 @@
}
context.computeAndSetTypeEnvironmentForOperator(op);
}
+
+ @Override
+ public String getName() {
+ return "INVERTED_INDEX_ACCESS_METHOD";
+ }
+
+ @Override
+ public int compareTo(IAccessMethod o) {
+ return this.getName().compareTo(o.getName());
+ }
}
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
index 461fb2a..c5bd8af 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
@@ -54,6 +54,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
/**
* Class for helping rewrite rules to choose and apply RTree indexes.
@@ -100,9 +101,8 @@
OptimizableOperatorSubTree subTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
IOptimizationContext context) throws AlgebricksException {
// TODO: We can probably do something smarter here based on selectivity or MBR area.
- IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
- ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(subTree, null, chosenIndex, optFuncExpr,
- analysisCtx, false, false, false, context);
+ ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(subTree, null, chosenIndex, analysisCtx,
+ false, false, false, context);
if (primaryIndexUnnestOp == null) {
return false;
}
@@ -112,6 +112,15 @@
}
@Override
+ public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef,
+ OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex,
+ AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast,
+ IOptimizationContext context) throws AlgebricksException {
+ return createSecondaryToPrimaryPlan(indexSubTree, probeSubTree, chosenIndex, analysisCtx, retainInput,
+ retainNull, requiresBroadcast, context);
+ }
+
+ @Override
public boolean applyJoinPlanTransformation(Mutable<ILogicalOperator> joinRef,
OptimizableOperatorSubTree leftSubTree, OptimizableOperatorSubTree rightSubTree, Index chosenIndex,
AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin,
@@ -140,9 +149,8 @@
}
// TODO: We can probably do something smarter here based on selectivity or MBR area.
- IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(indexSubTree, probeSubTree, chosenIndex,
- optFuncExpr, analysisCtx, true, isLeftOuterJoin, true, context);
+ analysisCtx, true, isLeftOuterJoin, true, context);
if (primaryIndexUnnestOp == null) {
return false;
}
@@ -165,9 +173,11 @@
}
private ILogicalOperator createSecondaryToPrimaryPlan(OptimizableOperatorSubTree indexSubTree,
- OptimizableOperatorSubTree probeSubTree, Index chosenIndex, IOptimizableFuncExpr optFuncExpr,
- AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast,
- IOptimizationContext context) throws AlgebricksException {
+ OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
+ boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context)
+ throws AlgebricksException {
+
+ IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
Dataset dataset = indexSubTree.dataset;
ARecordType recordType = indexSubTree.recordType;
@@ -221,7 +231,8 @@
if (probeSubTree == null) {
// We are optimizing a selection query.
// Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input).
- assignSearchKeys.getInputs().add(dataSourceOp.getInputs().get(0));
+ assignSearchKeys.getInputs().add(new MutableObject<ILogicalOperator>(
+ OperatorManipulationUtil.deepCopyWithExcutionMode(dataSourceOp.getInputs().get(0).getValue())));
assignSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode());
} else {
// We are optimizing a join, place the assign op top of the probe subtree.
@@ -254,4 +265,15 @@
// No additional analysis required.
return true;
}
+
+ @Override
+ public String getName() {
+ return "RTREE_ACCESS_METHOD";
+ }
+
+ @Override
+ public int compareTo(IAccessMethod o) {
+ return this.getName().compareTo(o.getName());
+ }
+
}
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
index 42905ee..0715d46 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
@@ -57,6 +57,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
@@ -528,6 +529,22 @@
}
@Override
+ public ILogicalOperator visitIntersectOperator(IntersectOperator op, Void arg) throws AlgebricksException {
+ visitMultiInputOperator(op);
+ List<LogicalVariable> outputVars = op.getOutputVars();
+ for (int i = 0; i < op.getNumInput(); i++) {
+ List<LogicalVariable> inputVars = op.getInputVariables(i);
+ if (inputVars.size() != outputVars.size()) {
+ throw new AlgebricksException("The cardinality of input and output are not equal for Intersection");
+ }
+ for (int j = 0; j < inputVars.size(); j++) {
+ updateInputToOutputVarMapping(inputVars.get(j), outputVars.get(j), false);
+ }
+ }
+ return op;
+ }
+
+ @Override
public ILogicalOperator visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
return visitSingleInputOperator(op);
}
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
index 191a2f9..5aaa59f 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
@@ -45,6 +45,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
@@ -307,6 +308,12 @@
}
@Override
+ public ILogicalOperator visitIntersectOperator(IntersectOperator op, Void arg) throws AlgebricksException {
+ throw new UnsupportedOperationException(
+ "Nested subplans with a intersect operator should have been disqualified for this rewriting!");
+ }
+
+ @Override
public ILogicalOperator visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
return visitSingleInputOperator(op);
}
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
index 1a2e8bb..c691365 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
@@ -31,6 +31,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
@@ -184,6 +185,11 @@
}
@Override
+ public Boolean visitIntersectOperator(IntersectOperator op, Void arg) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
public Boolean visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
return visitInputs(op);
}
diff --git a/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java b/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
index 538e7ca..c62f5d9 100644
--- a/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
+++ b/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
@@ -95,26 +95,34 @@
AsterixHyracksIntegrationUtil.deinit(true);
}
- private static void suiteBuild(File dir, Collection<Object[]> testArgs, String path) {
- for (File file : dir.listFiles()) {
- if (file.isDirectory() && !file.getName().startsWith(".")) {
- suiteBuild(file, testArgs, path + file.getName() + SEPARATOR);
+ private static void suiteBuildPerFile(File file, Collection<Object[]> testArgs, String path) {
+ if (file.isDirectory() && !file.getName().startsWith(".")) {
+ for (File innerfile : file.listFiles()) {
+ String subdir = innerfile.isDirectory() ? path + innerfile.getName() + SEPARATOR : path;
+ suiteBuildPerFile(innerfile, testArgs, subdir);
}
- if (file.isFile() && file.getName().endsWith(EXTENSION_QUERY)
- // && !ignore.contains(path + file.getName())
- ) {
- String resultFileName = AsterixTestHelper.extToResExt(file.getName(), EXTENSION_RESULT);
- File expectedFile = new File(PATH_EXPECTED + path + resultFileName);
- File actualFile = new File(PATH_ACTUAL + SEPARATOR + path.replace(SEPARATOR, "_") + resultFileName);
- testArgs.add(new Object[] { file, expectedFile, actualFile });
- }
+ }
+ if (file.isFile() && file.getName().endsWith(EXTENSION_QUERY)
+ // && !ignore.contains(path + file.getName())
+ ) {
+ String resultFileName = AsterixTestHelper.extToResExt(file.getName(), EXTENSION_RESULT);
+ File expectedFile = new File(PATH_EXPECTED + path + resultFileName);
+ File actualFile = new File(PATH_ACTUAL + SEPARATOR + path.replace(SEPARATOR, "_") + resultFileName);
+ testArgs.add(new Object[] { file, expectedFile, actualFile });
}
}
@Parameters(name = "OptimizerTest {index}: {0}")
public static Collection<Object[]> tests() {
Collection<Object[]> testArgs = new ArrayList<Object[]>();
- suiteBuild(new File(PATH_QUERIES), testArgs, "");
+ if (only.isEmpty()) {
+ suiteBuildPerFile(new File(PATH_QUERIES), testArgs, "");
+ } else {
+ for (String path : only) {
+ suiteBuildPerFile(new File(PATH_QUERIES + path), testArgs,
+ path.lastIndexOf(SEPARATOR) < 0 ? "" : path.substring(0, path.lastIndexOf(SEPARATOR) + 1));
+ }
+ }
return testArgs;
}
diff --git a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
index 4c79965..21fc240 100644
--- a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
+++ b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
@@ -52,8 +52,6 @@
protected static AsterixTransactionProperties txnProperties;
private final static TestExecutor testExecutor = new TestExecutor();
- protected static TestGroup FailedGroup;
-
@BeforeClass
public static void setUp() throws Exception {
try {
@@ -98,6 +96,6 @@
@Test
public void test() throws Exception {
- testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false, FailedGroup);
+ testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false, ExecutionTestUtil.FailedGroup);
}
}
diff --git a/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/btree-rtree-ngram-intersect.aql b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/btree-rtree-ngram-intersect.aql
new file mode 100644
index 0000000..5f57d07
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/btree-rtree-ngram-intersect.aql
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Tests three types of secondary indexes should trigger intersection rule
+ * Success : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ location: point,
+ message: string,
+ create_at: datetime,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id;
+
+create index ngram_index on dsTweet(message) type ngram(3);
+create index time_index on dsTweet(create_at) type btree;
+create index location_index on dsTweet(location) type rtree;
+
+write output to nc1:"rttest/btree-rtree-ngram-intersect.adm";
+
+let $region := create-rectangle(create-point(-128.43007812500002,20.298506037222175), create-point(-64.26992187500002,54.56902589732035))
+let $ts_start := datetime("2015-11-11T00:00:00Z")
+let $ts_end := datetime("2015-12-18T23:59:59Z")
+let $keyword := "hello"
+for $t in dataset dsTweet
+where $t.create_at >= $ts_start and $t.create_at < $ts_end
+ and spatial-intersect($t.location, $region)
+ and contains($t.message, $keyword)
+return $t
+
diff --git a/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-2nd-btree-intersect.aql b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-2nd-btree-intersect.aql
new file mode 100644
index 0000000..f0ecf57
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-2nd-btree-intersect.aql
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Tests two secondary Btree indexes should trigger intersection rule
+ * Success : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ location: point,
+ message: string,
+ create_at: datetime,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id;
+
+create index time_index on dsTweet(create_at) type btree;
+create index misc_index on dsTweet(misc) type btree;
+
+write output to nc1:"rttest/two-2nd-btree-intersect.adm";
+
+let $ts_start := datetime("2015-11-11T00:00:00Z")
+let $ts_end := datetime("2015-12-18T23:59:59Z")
+let $keyword := "hello"
+for $t in dataset dsTweet
+where $t.create_at >= $ts_start and $t.create_at < $ts_end
+ and $t.misc > $keyword
+return $t
+
diff --git a/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-inverted-index-intersect.aql b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-inverted-index-intersect.aql
new file mode 100644
index 0000000..2947d61
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-inverted-index-intersect.aql
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Tests keyword index and the ngram indexe should trigger intersection rule
+ * Success : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ message: string,
+ create_at: datetime,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id;
+
+create index msg_index on dsTweet(message) type ngram(3);
+create index msc_index on dsTweet(misc) type keyword;
+
+for $t in dataset dsTweet
+let $ed := edit-distance($t.message, "Suzanna Tilson")
+let $jacc := similarity-jaccard-check(word-tokens($t.misc), word-tokens("love like verizon"), 0.2f)
+where $jacc[0] and $ed <= 2
+return $t
+
diff --git a/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-rtree-intersect.aql b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-rtree-intersect.aql
new file mode 100644
index 0000000..40c9617
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/two-rtree-intersect.aql
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Tests two secondary rtree indexes should trigger intersection rule
+ * Success : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ location: point,
+ place: rectangle,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id;
+
+create index loc_index on dsTweet(location) type rtree;
+create index plc_index on dsTweet(place) type rtree;
+
+let $region := create-rectangle(create-point(-128.43007812500002,20.298506037222175), create-point(-64.26992187500002,54.56902589732035))
+for $t in dataset dsTweet
+where spatial-intersect($t.location , $region) and spatial-intersect($t.place, $region)
+return $t
+
diff --git a/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/with-primary-index-intersect.aql b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/with-primary-index-intersect.aql
new file mode 100644
index 0000000..d90f119
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries/multi-indexes/with-primary-index-intersect.aql
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Tests primary index will be chosen if it existed in the job plan, no intersection should apply
+ * Success : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type tTweet as closed {
+ id: int32,
+ location: point,
+ message: string,
+ create_at: datetime,
+ misc: string
+}
+
+create dataset dsTweet(tTweet) primary key id;
+
+create index ngram_index on dsTweet(message) type ngram(3);
+create index time_index on dsTweet(create_at) type btree;
+create index location_index on dsTweet(location) type rtree;
+
+write output to nc1:"rttest/btree-rtree-ngram-intersect.adm";
+
+let $region := create-rectangle(create-point(-128.43007812500002,20.298506037222175), create-point(-64.26992187500002,54.56902589732035))
+let $ts_start := datetime("2015-11-11T00:00:00Z")
+let $ts_end := datetime("2015-12-18T23:59:59Z")
+let $keyword := "hello"
+for $t in dataset dsTweet
+where $t.create_at >= $ts_start and $t.create_at < $ts_end
+ and spatial-intersect($t.location, $region)
+ and contains($t.message, $keyword)
+ and $t.id > 0
+return $t
+
diff --git a/asterix-app/src/test/resources/optimizerts/results/multi-indexes/btree-rtree-ngram-intersect.plan b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/btree-rtree-ngram-intersect.plan
new file mode 100644
index 0000000..2d74649
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/btree-rtree-ngram-intersect.plan
@@ -0,0 +1,37 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- INTERSECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$29(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$31(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$40(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- RTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-2nd-btree-intersect.plan b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-2nd-btree-intersect.plan
new file mode 100644
index 0000000..5adb550
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-2nd-btree-intersect.plan
@@ -0,0 +1,28 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- INTERSECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$19(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$23(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-inverted-index-intersect.plan b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-inverted-index-intersect.plan
new file mode 100644
index 0000000..7d99635
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-inverted-index-intersect.plan
@@ -0,0 +1,26 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- INTERSECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$18(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$20(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-rtree-intersect.plan b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-rtree-intersect.plan
new file mode 100644
index 0000000..05d43a9
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/two-rtree-intersect.plan
@@ -0,0 +1,32 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- INTERSECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$24(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- RTREE_SEARCH |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SPLIT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$33(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- RTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SPLIT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterix-app/src/test/resources/optimizerts/results/multi-indexes/with-primary-index-intersect.plan b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/with-primary-index-intersect.plan
new file mode 100644
index 0000000..7978f8a
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/multi-indexes/with-primary-index-intersect.plan
@@ -0,0 +1,11 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.1.ddl.aql
new file mode 100644
index 0000000..10145ed
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.1.ddl.aql
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse TinySocial if exists;
+create dataverse TinySocial;
+use dataverse TinySocial;
+
+create type TwitterUserType as open {
+ screen-name: string,
+ lang: string,
+ friends_count: int64,
+ statuses_count: int64,
+ name: string,
+ followers_count: int64,
+ sender-location: point
+}
+
+create type TweetMessageType as closed {
+ tweetid: int64,
+ user: TwitterUserType,
+ send-time: datetime,
+ referred-topics: {{ string }},
+ message-text: string
+}
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid
+hints(cardinality=100);
+
+create index twTimeIdx on TweetMessages(send-time) type btree;
+create index twLocationIdx on TweetMessages(user.sender-location) type rtree;
+create index twMessage on TweetMessages(message-text) type ngram(2);
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.2.update.aql
new file mode 100644
index 0000000..9923ff4
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.2.update.aql
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse TinySocial;
+
+load dataset TweetMessages using localfs
+(("path"="asterix_nc1://data/tinysocial/twm-nested.adm"),("format"="adm"));
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.3.query.aql
new file mode 100644
index 0000000..392008c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/intersection/tinysocial-intersect.3.query.aql
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse TinySocial;
+
+let $ts := datetime("2010-12-12T00:00:00Z")
+let $region := create-rectangle(create-point(0.0,0.0),create-point(100.0,100.0))
+let $keyword := "verizon"
+for $t in dataset TweetMessages
+where $t.send-time > $ts
+ and spatial-intersect($t.user.sender-location, $region)
+ and contains($t.message-text, $keyword)
+return $t
diff --git a/asterix-app/src/test/resources/runtimets/results/index-selection/intersection/intersection.1.adm b/asterix-app/src/test/resources/runtimets/results/index-selection/intersection/intersection.1.adm
new file mode 100644
index 0000000..44cc08a
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/index-selection/intersection/intersection.1.adm
@@ -0,0 +1 @@
+{ "tweetid": 9, "user": { "screen-name": "NathanGiesen@211", "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416, "sender-location": point("36.86,74.62") }, "send-time": datetime("2012-07-21T10:10:00.000Z"), "referred-topics": {{ "verizon", "voicemail-service" }}, "message-text": " love verizon its voicemail-service is awesome" }
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index f148a77..99b195d 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -2602,6 +2602,11 @@
<output-dir compare="Text">disjunctive-predicate-1</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="index-selection">
+ <compilation-unit name="intersection">
+ <output-dir compare="Text">intersection</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="inverted-index-join">
<test-case FilePath="inverted-index-join">