[ASTERIXDB-2354][COMP] Partition constraint propagation for binary operators
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Fixes partition constraint propagation for binary operators.
Previously only constraint from the second branch was used and
constraint from the first branch was ignored.
Now constraints from both branches are composed into a single one.
Change-Id: Ia9930479af9d0f67d6ed3ed490b9f05dc1cd8e3a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2568
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2354/query-ASTERIXDB-2354.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2354/query-ASTERIXDB-2354.1.ddl.sqlpp
new file mode 100644
index 0000000..2812a2c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2354/query-ASTERIXDB-2354.1.ddl.sqlpp
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse tpch if exists;
+create dataverse tpch;
+
+use tpch;
+
+create type tpch.OrderType as
+ closed {
+ o_orderkey : bigint,
+ o_custkey : bigint,
+ o_orderstatus : string,
+ o_totalprice : double,
+ o_orderdate : string,
+ o_orderpriority : string,
+ o_clerk : string,
+ o_shippriority : bigint,
+ o_comment : string
+};
+
+create dataset Orders(OrderType) primary key o_orderkey;
+
+create index OrdersIdx on Orders (o_custkey) type btree;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2354/query-ASTERIXDB-2354.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2354/query-ASTERIXDB-2354.2.query.sqlpp
new file mode 100644
index 0000000..1a78363
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2354/query-ASTERIXDB-2354.2.query.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use tpch;
+
+SELECT ds.DatasetName as v1
+FROM Metadata.`Dataset` ds
+WHERE ds.DatasetName LIKE "Orders%"
+UNION ALL
+SELECT ds.DatasetName v1, idx.DatasetName v2, idx.IndexName v3
+FROM Metadata.`Index` idx, Metadata.`Dataset` ds
+WHERE ds.DatasetName LIKE "Orders%" and idx.DatasetName LIKE "Orders%"
+ORDER BY v1, v2, v3
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2354/query-ASTERIXDB-2354.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2354/query-ASTERIXDB-2354.1.adm
new file mode 100644
index 0000000..e2ba372
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2354/query-ASTERIXDB-2354.1.adm
@@ -0,0 +1,3 @@
+{ "v1": "Orders" }
+{ "v1": "Orders", "v2": "Orders", "v3": "Orders" }
+{ "v1": "Orders", "v2": "Orders", "v3": "OrdersIdx" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 3b1363c..3823234 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -3715,6 +3715,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-2354">
+ <output-dir compare="Text">query-ASTERIXDB-2354</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
<compilation-unit name="query-ASTERIXDB-2355">
<output-dir compare="Text">none</output-dir>
<expected-error><![CDATA[ASX1001: Syntax error: In line 22 >> %%%<< Encountered "%" at column 2.]]></expected-error>
diff --git a/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksAbsolutePartitionConstraint.java b/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksAbsolutePartitionConstraint.java
index 067579e..b6443c4 100644
--- a/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksAbsolutePartitionConstraint.java
+++ b/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksAbsolutePartitionConstraint.java
@@ -20,11 +20,15 @@
import java.util.Arrays;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.exceptions.ErrorCode;
+
public class AlgebricksAbsolutePartitionConstraint extends AlgebricksPartitionConstraint {
private final String[] locations;
public AlgebricksAbsolutePartitionConstraint(String[] locations) {
- this.locations = locations;
+ this.locations = locations.clone();
+ Arrays.sort(locations);
}
@Override
@@ -38,7 +42,26 @@
@Override
public String toString() {
- return Arrays.toString(locations);
+ return getPartitionConstraintType().toString() + ':' + Arrays.toString(locations);
}
+ @Override
+ public AlgebricksPartitionConstraint compose(AlgebricksPartitionConstraint that) throws AlgebricksException {
+ switch (that.getPartitionConstraintType()) {
+ case COUNT:
+ AlgebricksCountPartitionConstraint thatCount = (AlgebricksCountPartitionConstraint) that;
+ if (locations.length <= thatCount.getCount()) {
+ return this;
+ }
+ break;
+ case ABSOLUTE:
+ AlgebricksAbsolutePartitionConstraint thatAbsolute = (AlgebricksAbsolutePartitionConstraint) that;
+ if (Arrays.equals(locations, thatAbsolute.locations)) {
+ return this;
+ }
+ break;
+ }
+
+ throw AlgebricksException.create(ErrorCode.CANNOT_COMPOSE_PART_CONSTRAINTS, toString(), that.toString());
+ }
}
diff --git a/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksCountPartitionConstraint.java b/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksCountPartitionConstraint.java
index fbafdee..2fc4804 100644
--- a/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksCountPartitionConstraint.java
+++ b/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksCountPartitionConstraint.java
@@ -18,6 +18,9 @@
*/
package org.apache.hyracks.algebricks.common.constraints;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.exceptions.ErrorCode;
+
public class AlgebricksCountPartitionConstraint extends AlgebricksPartitionConstraint {
private final int count;
@@ -35,4 +38,21 @@
return count;
}
+ @Override
+ public String toString() {
+ return getPartitionConstraintType().toString() + ':' + count;
+ }
+
+ @Override
+ public AlgebricksPartitionConstraint compose(AlgebricksPartitionConstraint that) throws AlgebricksException {
+ switch (that.getPartitionConstraintType()) {
+ case COUNT:
+ AlgebricksCountPartitionConstraint thatCount = (AlgebricksCountPartitionConstraint) that;
+ return count <= thatCount.count ? this : that;
+ case ABSOLUTE:
+ return that.compose(this);
+ }
+
+ throw AlgebricksException.create(ErrorCode.CANNOT_COMPOSE_PART_CONSTRAINTS, toString(), that.toString());
+ }
}
diff --git a/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraint.java b/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraint.java
index ada1390..71575d5 100644
--- a/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraint.java
+++ b/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraint.java
@@ -18,6 +18,8 @@
*/
package org.apache.hyracks.algebricks.common.constraints;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+
public abstract class AlgebricksPartitionConstraint {
public enum PartitionConstraintType {
ABSOLUTE,
@@ -25,4 +27,7 @@
}
public abstract PartitionConstraintType getPartitionConstraintType();
+
+ public abstract AlgebricksPartitionConstraint compose(AlgebricksPartitionConstraint that)
+ throws AlgebricksException;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
index 16992e7..526add1 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
@@ -185,7 +185,8 @@
return resultOps;
}
- private void setAllPartitionConstraints(Map<IConnectorDescriptor, TargetConstraint> tgtConstraints) {
+ private void setAllPartitionConstraints(Map<IConnectorDescriptor, TargetConstraint> tgtConstraints)
+ throws AlgebricksException {
List<OperatorDescriptorId> roots = jobSpec.getRoots();
setSpecifiedPartitionConstraints();
for (OperatorDescriptorId rootId : roots) {
@@ -243,8 +244,8 @@
}
private void setPartitionConstraintsBottomup(OperatorDescriptorId opId,
- Map<IConnectorDescriptor, TargetConstraint> tgtConstraints, IOperatorDescriptor parentOp,
- boolean finalPass) {
+ Map<IConnectorDescriptor, TargetConstraint> tgtConstraints, IOperatorDescriptor parentOp, boolean finalPass)
+ throws AlgebricksException {
List<IConnectorDescriptor> opInputs = jobSpec.getOperatorInputMap().get(opId);
AlgebricksPartitionConstraint opConstraint = null;
IOperatorDescriptor opDesc = jobSpec.getOperatorMap().get(opId);
@@ -260,10 +261,10 @@
if (constraint != null) {
switch (constraint) {
case ONE:
- opConstraint = countOneLocation;
+ opConstraint = composePartitionConstraints(opConstraint, countOneLocation);
break;
case SAME_COUNT:
- opConstraint = partitionConstraintMap.get(src);
+ opConstraint = composePartitionConstraints(opConstraint, partitionConstraintMap.get(src));
break;
}
}
@@ -439,4 +440,9 @@
}
return false;
}
+
+ private static AlgebricksPartitionConstraint composePartitionConstraints(AlgebricksPartitionConstraint pc1,
+ AlgebricksPartitionConstraint pc2) throws AlgebricksException {
+ return pc1 == null ? pc2 : pc2 == null ? pc1 : pc1.compose(pc2);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
index 0691005..51afac1 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
@@ -151,6 +151,7 @@
// Compilation error codes.
public static final int RULECOLLECTION_NOT_INSTANCE_OF_LIST = 10000;
+ public static final int CANNOT_COMPOSE_PART_CONSTRAINTS = 10001;
private static class Holder {
private static final Map<Integer, String> errorMessageMap;
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
index 1846062..452d379 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
@@ -133,3 +133,4 @@
114 = Node (%1$s) is not active
10000 = The given rule collection %1$s is not an instance of the List class.
+10001 = Cannot compose partition constraint %1$s with %2$s