checkpoint: added rule to cancel the unnest with a nested listify
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/base/RuleCollections.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/base/RuleCollections.java
index 7130c26..027a3b5 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/base/RuleCollections.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/base/RuleCollections.java
@@ -20,6 +20,7 @@
import edu.uci.ics.asterix.optimizer.rules.AsterixInlineVariablesRule;
import edu.uci.ics.asterix.optimizer.rules.ByNameToByIndexFieldAccessRule;
+import edu.uci.ics.asterix.optimizer.rules.CancelUnnestWithNestedListifyRule;
import edu.uci.ics.asterix.optimizer.rules.CheckFilterExpressionTypeRule;
import edu.uci.ics.asterix.optimizer.rules.ConstantFoldingRule;
import edu.uci.ics.asterix.optimizer.rules.CountVarToCountOneRule;
@@ -143,6 +144,7 @@
condPushDownAndJoinInference.add(new PushSelectDownRule());
condPushDownAndJoinInference.add(new RemoveRedundantListifyRule());
+ condPushDownAndJoinInference.add(new CancelUnnestWithNestedListifyRule());
condPushDownAndJoinInference.add(new SimpleUnnestToProductRule());
condPushDownAndJoinInference.add(new ComplexUnnestToProductRule());
condPushDownAndJoinInference.add(new ComplexJoinInferenceRule());
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/CancelUnnestWithNestedListifyRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/CancelUnnestWithNestedListifyRule.java
new file mode 100644
index 0000000..8ef4847
--- /dev/null
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/CancelUnnestWithNestedListifyRule.java
@@ -0,0 +1,257 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.asterix.aql.util.FunctionUtils;
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnpartitionedPropertyComputer;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * This rule cancels the unnest with the nested listify. Formally, the following plan<br/>
+ *
+ * <pre>
+ * unnset $x <- $y
+ * group-by($k){
+ * aggregate $y <- listify($z)
+ * ...
+ * }
+ * </pre>
+ *
+ * will be converted into<br/>
+ *
+ * <pre>
+ * assign $x <- $z
+ * sort($k)
+ * </pre>
+ *
+ * When the positional variable exists, for example the original plan is like<br/>
+ *
+ * <pre>
+ * unnset $x at $p <- $y
+ * group-by($k){
+ * aggregate $y <- listify($z)
+ * ...
+ * }
+ * </pre>
+ *
+ * will be converted into<br/>
+ *
+ * <pre>
+ * group-by($k){
+ * running-aggregate $p <- tid()
+ * }
+ * </pre>
+ */
+public class CancelUnnestWithNestedListifyRule implements IAlgebraicRewriteRule {
+
+ /* (non-Javadoc)
+ * @see edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule#rewritePost(org.apache.commons.lang3.mutable.Mutable, edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext)
+ */
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ // apply it only at the top of the plan
+ ILogicalOperator op = opRef.getValue();
+ if (context.checkIfInDontApplySet(this, op)) {
+ return false;
+ }
+ Set<LogicalVariable> varSet = new HashSet<LogicalVariable>();
+ return applyRuleDown(opRef, varSet, context);
+ }
+
+ private boolean applyRuleDown(Mutable<ILogicalOperator> opRef, Set<LogicalVariable> varSet,
+ IOptimizationContext context) throws AlgebricksException {
+ boolean changed = applies(opRef, varSet, context);
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ VariableUtilities.getUsedVariables(op, varSet);
+ if (op.hasNestedPlans()) {
+ AbstractOperatorWithNestedPlans aonp = (AbstractOperatorWithNestedPlans) op;
+ for (ILogicalPlan p : aonp.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ if (applyRuleDown(r, varSet, context)) {
+ changed = true;
+ }
+ context.addToDontApplySet(this, r.getValue());
+ }
+ }
+ }
+ for (Mutable<ILogicalOperator> i : op.getInputs()) {
+ if (applyRuleDown(i, varSet, context)) {
+ changed = true;
+ }
+ context.addToDontApplySet(this, i.getValue());
+ }
+ return changed;
+ }
+
+ private boolean applies(Mutable<ILogicalOperator> opRef, Set<LogicalVariable> varUsedAbove,
+ IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+ if (op1.getOperatorTag() != LogicalOperatorTag.UNNEST) {
+ return false;
+ }
+ UnnestOperator unnest1 = (UnnestOperator) op1;
+ ILogicalExpression expr = unnest1.getExpressionRef().getValue();
+ LogicalVariable unnestedVar;
+ switch (expr.getExpressionTag()) {
+ case VARIABLE:
+ unnestedVar = ((VariableReferenceExpression) expr).getVariableReference();
+ break;
+ case FUNCTION_CALL:
+ if (((AbstractFunctionCallExpression) expr).getFunctionIdentifier() != AsterixBuiltinFunctions.SCAN_COLLECTION) {
+ return false;
+ }
+ AbstractFunctionCallExpression functionCall = (AbstractFunctionCallExpression) expr;
+ ILogicalExpression functionCallArgExpr = functionCall.getArguments().get(0).getValue();
+ if (functionCallArgExpr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ return false;
+ }
+ unnestedVar = ((VariableReferenceExpression) functionCallArgExpr).getVariableReference();
+ break;
+ default:
+ return false;
+ }
+ if (varUsedAbove.contains(unnestedVar)) {
+ return false;
+ }
+
+ Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
+ AbstractLogicalOperator r = (AbstractLogicalOperator) opRef2.getValue();
+
+ if (r.getOperatorTag() != LogicalOperatorTag.GROUP) {
+ return false;
+ }
+
+ // go inside of a group-by plan
+ GroupByOperator gby = (GroupByOperator) r;
+ if (gby.getNestedPlans().size() != 1) {
+ return false;
+ }
+ if (gby.getNestedPlans().get(0).getRoots().size() != 1) {
+ return false;
+ }
+ AbstractLogicalOperator nestedPlanRoot = (AbstractLogicalOperator) gby.getNestedPlans().get(0).getRoots()
+ .get(0).getValue();
+ if (nestedPlanRoot.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+ return false;
+ }
+ AggregateOperator agg = (AggregateOperator) nestedPlanRoot;
+
+ if (agg.getVariables().size() > 1) {
+ return false;
+ }
+ LogicalVariable aggVar = agg.getVariables().get(0);
+ ILogicalExpression aggFun = agg.getExpressions().get(0).getValue();
+ if (!aggVar.equals(unnestedVar)
+ || ((AbstractLogicalExpression) aggFun).getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return false;
+ }
+ AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) aggFun;
+ if (!AsterixBuiltinFunctions.LISTIFY.equals(f.getFunctionIdentifier())) {
+ return false;
+ }
+ if (f.getArguments().size() != 1) {
+ return false;
+ }
+ ILogicalExpression arg0 = f.getArguments().get(0).getValue();
+ if (((AbstractLogicalExpression) arg0).getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ return false;
+ }
+ LogicalVariable paramVar = ((VariableReferenceExpression) arg0).getVariableReference();
+
+ ArrayList<LogicalVariable> assgnVars = new ArrayList<LogicalVariable>(1);
+ assgnVars.add(unnest1.getVariable());
+ ArrayList<Mutable<ILogicalExpression>> assgnExprs = new ArrayList<Mutable<ILogicalExpression>>(1);
+ assgnExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(paramVar)));
+ AssignOperator assign = new AssignOperator(assgnVars, assgnExprs);
+
+ LogicalVariable posVar = unnest1.getPositionalVariable();
+ if (posVar == null) {
+
+ // create assignment for group-by keys
+ ArrayList<LogicalVariable> gbyKeyAssgnVars = new ArrayList<LogicalVariable>();
+ ArrayList<Mutable<ILogicalExpression>> gbyKeyAssgnExprs = new ArrayList<Mutable<ILogicalExpression>>();
+ for (int i = 0; i < gby.getGroupByList().size(); i++) {
+ if (gby.getGroupByList().get(i).first != null) {
+ gbyKeyAssgnVars.add(gby.getGroupByList().get(i).first);
+ gbyKeyAssgnExprs.add(gby.getGroupByList().get(i).second);
+ }
+ }
+
+ AssignOperator gbyKeyAssign = new AssignOperator(gbyKeyAssgnVars, gbyKeyAssgnExprs);
+ gbyKeyAssign.getInputs().add(gby.getInputs().get(0));
+
+ context.computeAndSetTypeEnvironmentForOperator(gbyKeyAssign);
+
+ // add sort to replace group-by
+ List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExprs = new ArrayList<Pair<IOrder, Mutable<ILogicalExpression>>>();
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> k : gby.getGroupByList()) {
+ orderExprs.add(new Pair<IOrder, Mutable<ILogicalExpression>>(OrderOperator.ASC_ORDER, k.second));
+ }
+
+ OrderOperator order = new OrderOperator(orderExprs);
+ order.getInputs().add(new MutableObject<ILogicalOperator>(gbyKeyAssign));
+
+ context.computeAndSetTypeEnvironmentForOperator(order);
+
+ assign.getInputs().add(new MutableObject<ILogicalOperator>(order));
+ opRef.setValue(assign);
+ } else {
+
+ return false;
+ }
+
+ context.computeAndSetTypeEnvironmentForOperator(assign);
+
+ return true;
+ }
+}
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/RemoveRedundantListifyRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/RemoveRedundantListifyRule.java
index 5ae5285..51511aa 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/RemoveRedundantListifyRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/RemoveRedundantListifyRule.java
@@ -134,7 +134,7 @@
return false;
}
- Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
+ Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
AbstractLogicalOperator r = (AbstractLogicalOperator) opRef2.getValue();
if (r.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
diff --git a/asterix-app/src/test/resources/runtimets/queries/flwor/at00/at00.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/flwor/at00/at00.1.ddl.aql
new file mode 100644
index 0000000..66b5ca2
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/flwor/at00/at00.1.ddl.aql
@@ -0,0 +1,12 @@
+drop dataverse temp if exists;
+create dataverse temp;
+use dataverse temp;
+
+create type TestType as closed {
+ "lid": int32,
+ "uid": int32,
+ "timestamp": datetime,
+ "aid": int32
+}
+
+create dataset test(TestType) primary key lid;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/flwor/at00/at00.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/flwor/at00/at00.2.update.aql
new file mode 100644
index 0000000..78b0e5b
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/flwor/at00/at00.2.update.aql
@@ -0,0 +1,4 @@
+use dataverse temp;
+
+load dataset test using localfs
+(("path"="nc1:///Users/jarodwen/Downloads/app_event_100_2000.dat"),("format"="adm"));
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/flwor/at00/at00.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/flwor/at00/at00.3.query.aql
new file mode 100644
index 0000000..f5459fb
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/flwor/at00/at00.3.query.aql
@@ -0,0 +1,7 @@
+use dataverse temp;
+
+for $i in dataset('test')
+limit 300
+group by $lid := $i.lid with $i
+for $j in $i
+return {"lid": $lid, "item": $j}
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/flwor/at00/at00.1.adm b/asterix-app/src/test/resources/runtimets/results/flwor/at00/at00.1.adm
new file mode 100644
index 0000000..fbf7b88
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/flwor/at00/at00.1.adm
@@ -0,0 +1,10 @@
+{ "num": 1, "name": "BramHatch", "user-since": datetime("2010-10-16T10:10:00.000Z") }
+{ "num": 2, "name": "EmoryUnk", "user-since": datetime("2012-07-10T10:10:00.000Z") }
+{ "num": 3, "name": "IsbelDull", "user-since": datetime("2011-01-22T10:10:00.000Z") }
+{ "num": 4, "name": "MargaritaStoddard", "user-since": datetime("2012-08-20T10:10:00.000Z") }
+{ "num": 5, "name": "NicholasStroh", "user-since": datetime("2010-12-27T10:10:00.000Z") }
+{ "num": 6, "name": "NilaMilliron", "user-since": datetime("2008-01-01T10:10:00.000Z") }
+{ "num": 7, "name": "SuzannaTillson", "user-since": datetime("2012-08-07T10:10:00.000Z") }
+{ "num": 8, "name": "VonKemble", "user-since": datetime("2010-01-05T10:10:00.000Z") }
+{ "num": 9, "name": "WillisWynne", "user-since": datetime("2005-01-17T10:10:00.000Z") }
+{ "num": 10, "name": "WoodrowNehling", "user-since": datetime("2005-09-20T10:10:00.000Z") }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index 5ad90d5..a59447f 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -15,6 +15,11 @@
<test-suite xmlns="urn:xml.testframework.asterix.ics.uci.edu" ResultOffsetPath="results" QueryOffsetPath="queries" QueryFileExtension=".aql">
<test-group name="flwor">
<test-case FilePath="flwor">
+ <compilation-unit name="at00">
+ <output-dir compare="Text">at00</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="flwor">
<compilation-unit name="at01">
<output-dir compare="Text">at01</output-dir>
</compilation-unit>
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/functions/AsterixBuiltinFunctions.java b/asterix-om/src/main/java/edu/uci/ics/asterix/om/functions/AsterixBuiltinFunctions.java
index b6bdb4b..d38964c 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/om/functions/AsterixBuiltinFunctions.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/functions/AsterixBuiltinFunctions.java
@@ -353,6 +353,7 @@
FunctionConstants.ASTERIX_NS, "counthashed-gram-tokens", 3);
public final static FunctionIdentifier TID = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "tid", 0);
+ public final static FunctionIdentifier GTID = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "gtid", 0);
// constructors:
public final static FunctionIdentifier BOOLEAN_CONSTRUCTOR = new FunctionIdentifier(FunctionConstants.ASTERIX_NS,