[ASTERIXDB-3332][COMP] Consolidate projection and filters
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
Consolidate projected fields and filter expressions when
scanning (or searching) the same dataset.
Change-Id: I3d8122e71a949c18b39efa8d92ea169173bbdb61
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18063
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Wail Alkowaileet <wael.y.k@gmail.com>
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
diff --git a/hyracks-fullstack/NOTICE b/hyracks-fullstack/NOTICE
index 6e9c46b..e9bb9a4 100644
--- a/hyracks-fullstack/NOTICE
+++ b/hyracks-fullstack/NOTICE
@@ -1,5 +1,5 @@
Apache Hyracks and Algebricks
-Copyright 2015-2023 The Apache Software Foundation
+Copyright 2015-2024 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/DefaultProjectionFiltrationInfo.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/DefaultProjectionFiltrationInfo.java
index ff13007..077a597 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/DefaultProjectionFiltrationInfo.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/DefaultProjectionFiltrationInfo.java
@@ -32,6 +32,11 @@
}
@Override
+ public void substituteFilterVariable(LogicalVariable oldVar, LogicalVariable newVar) {
+ // NoOp
+ }
+
+ @Override
public IProjectionFiltrationInfo createCopy() {
return INSTANCE;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionFiltrationInfo.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionFiltrationInfo.java
index 149731f..4310676 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionFiltrationInfo.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IProjectionFiltrationInfo.java
@@ -20,6 +20,7 @@
import java.io.IOException;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.prettyprint.AlgebricksStringBuilderWriter;
import com.fasterxml.jackson.core.JsonGenerator;
@@ -29,6 +30,15 @@
* {@link org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator}
*/
public interface IProjectionFiltrationInfo {
+
+ /**
+ * Substitute filter expression variables
+ *
+ * @param oldVar old variable
+ * @param newVar new variable
+ */
+ void substituteFilterVariable(LogicalVariable oldVar, LogicalVariable newVar);
+
/**
* @return a copy of the {@link IProjectionFiltrationInfo}
*/
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/AllVariablesSubstituteVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/AllVariablesSubstituteVisitor.java
new file mode 100644
index 0000000..06d2181
--- /dev/null
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/AllVariablesSubstituteVisitor.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+
+/**
+ * Substitute all variables with the provided one
+ * Example substitute all variables with $$myVar:
+ * add($$1, $$2) --> add($$myVar, $$myVar)
+ */
+public class AllVariablesSubstituteVisitor implements ILogicalExpressionVisitor<Void, LogicalVariable> {
+ @Override
+ public Void visitConstantExpression(ConstantExpression expr, LogicalVariable arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitVariableReferenceExpression(VariableReferenceExpression expr, LogicalVariable arg)
+ throws AlgebricksException {
+ expr.setVariable(arg);
+ return null;
+ }
+
+ @Override
+ public Void visitAggregateFunctionCallExpression(AggregateFunctionCallExpression expr, LogicalVariable arg)
+ throws AlgebricksException {
+ visitArgs(expr.getArguments(), arg);
+ return null;
+ }
+
+ @Override
+ public Void visitScalarFunctionCallExpression(ScalarFunctionCallExpression expr, LogicalVariable arg)
+ throws AlgebricksException {
+ visitArgs(expr.getArguments(), arg);
+ return null;
+ }
+
+ @Override
+ public Void visitStatefulFunctionCallExpression(StatefulFunctionCallExpression expr, LogicalVariable arg)
+ throws AlgebricksException {
+ visitArgs(expr.getArguments(), arg);
+ return null;
+ }
+
+ @Override
+ public Void visitUnnestingFunctionCallExpression(UnnestingFunctionCallExpression expr, LogicalVariable arg)
+ throws AlgebricksException {
+ visitArgs(expr.getArguments(), arg);
+ return null;
+ }
+
+ private void visitArgs(List<Mutable<ILogicalExpression>> args, LogicalVariable variable)
+ throws AlgebricksException {
+ for (Mutable<ILogicalExpression> funcArg : args) {
+ funcArg.getValue().accept(this, variable);
+ }
+ }
+}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/EquivalentVariableExpressionComparatorVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/EquivalentVariableExpressionComparatorVisitor.java
new file mode 100644
index 0000000..233e181
--- /dev/null
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/EquivalentVariableExpressionComparatorVisitor.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+
+/**
+ * Allows to compare two expression with the assumption that all variables are equivalent (e.g., two different
+ * variables originated from two different data-scan operators but on the same dataset)
+ */
+public class EquivalentVariableExpressionComparatorVisitor
+ implements ILogicalExpressionVisitor<Boolean, ILogicalExpression> {
+ public static final EquivalentVariableExpressionComparatorVisitor INSTANCE =
+ new EquivalentVariableExpressionComparatorVisitor();
+
+ private EquivalentVariableExpressionComparatorVisitor() {
+ }
+
+ @Override
+ public Boolean visitConstantExpression(ConstantExpression expr, ILogicalExpression arg) throws AlgebricksException {
+ return expr.equals(arg);
+ }
+
+ @Override
+ public Boolean visitVariableReferenceExpression(VariableReferenceExpression expr, ILogicalExpression arg)
+ throws AlgebricksException {
+ if (arg.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ return Boolean.TRUE;
+ }
+
+ return Boolean.FALSE;
+ }
+
+ @Override
+ public Boolean visitAggregateFunctionCallExpression(AggregateFunctionCallExpression expr, ILogicalExpression arg)
+ throws AlgebricksException {
+ return equals(expr, arg);
+ }
+
+ @Override
+ public Boolean visitScalarFunctionCallExpression(ScalarFunctionCallExpression expr, ILogicalExpression arg)
+ throws AlgebricksException {
+ return equals(expr, arg);
+ }
+
+ @Override
+ public Boolean visitStatefulFunctionCallExpression(StatefulFunctionCallExpression expr, ILogicalExpression arg)
+ throws AlgebricksException {
+ return equals(expr, arg);
+ }
+
+ @Override
+ public Boolean visitUnnestingFunctionCallExpression(UnnestingFunctionCallExpression expr, ILogicalExpression arg)
+ throws AlgebricksException {
+ return equals(expr, arg);
+ }
+
+ public Boolean equals(AbstractFunctionCallExpression expr1, ILogicalExpression expr2) throws AlgebricksException {
+ if (!(expr2 instanceof AbstractFunctionCallExpression)) {
+ return false;
+ } else {
+ AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) expr2;
+ boolean equal = expr1.getFunctionIdentifier().equals(fce.getFunctionIdentifier());
+ if (!equal) {
+ return false;
+ }
+ List<Mutable<ILogicalExpression>> arguments = expr1.getArguments();
+ int argumentCount = arguments.size();
+ List<Mutable<ILogicalExpression>> fceArguments = fce.getArguments();
+ if (argumentCount != fceArguments.size()) {
+ return false;
+ }
+ for (int i = 0; i < argumentCount; i++) {
+ ILogicalExpression argument = arguments.get(i).getValue();
+ ILogicalExpression fceArgument = fceArguments.get(i).getValue();
+ if (argument.accept(this, fceArgument) == Boolean.FALSE) {
+ return false;
+ }
+ }
+ return Arrays.deepEquals(expr1.getOpaqueParameters(), fce.getOpaqueParameters());
+ }
+ }
+}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
index 7ceb812..d0b0608 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
@@ -140,6 +140,7 @@
substUsedVariables(op.getMinFilterVars(), pair.first, pair.second);
substUsedVariables(op.getMaxFilterVars(), pair.first, pair.second);
}
+ op.getProjectionFiltrationInfo().substituteFilterVariable(pair.first, pair.second);
return null;
}
@@ -317,6 +318,7 @@
} else {
substUsedVariablesInExpr(op.getSelectCondition(), pair.first, pair.second);
}
+ op.getProjectionFiltrationInfo().substituteFilterVariable(pair.first, pair.second);
return null;
}
@@ -327,6 +329,7 @@
if (producedVarFound) {
substProducedVarInTypeEnvironment(op, pair);
}
+ op.getProjectionFiltrationInfo().substituteFilterVariable(pair.first, pair.second);
return null;
}