Merge branch 'gerrit/goldfish' into 'master'
Change-Id: I088ea2f4a99970b30cdd15f723f05e2db1da9277
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/query-ASTERIXDB-3512.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/query-ASTERIXDB-3512.sqlpp
new file mode 100644
index 0000000..d51df3b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/query-ASTERIXDB-3512.sqlpp
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type dt1 as {id: int};
+
+create dataset collection1(dt1) PRIMARY KEY id;
+create dataset collection2(dt1) PRIMARY KEY id;
+
+CREATE INDEX idx1 ON collection2 (date: STRING, oid: STRING, sym: STRING, reg: BIGINT);
+
+with tmp1 AS (
+ SELECT S.sym
+ FROM collection1 AS S
+ WHERE S.date < print_date(current_date(), 'YYYY-MM-DD')),
+ tmp2 AS (
+ SELECT O.sym
+ FROM collection2 AS O
+ WHERE O.date < print_date(current_date(), 'YYYY-MM-DD'))
+SELECT sl1.sym
+FROM tmp1 AS sl1
+UNION ALL
+SELECT sl2.sym
+FROM tmp2 as sl2;
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/query-ASTERIXDB-3512.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/query-ASTERIXDB-3512.plan
new file mode 100644
index 0000000..9160a43
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/query-ASTERIXDB-3512.plan
@@ -0,0 +1,28 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- UNION_ALL |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.collection1) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (test.collection2.collection2) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$117(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (test.collection2.idx1) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.1.ddl.sqlpp
new file mode 100644
index 0000000..980d4d4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.1.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description: This test case is to verify the fix for ASTERIXDB-3512
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type dt1 as {id: int};
+
+create dataset collection1(dt1) PRIMARY KEY id;
+create dataset collection2(dt1) PRIMARY KEY id;
+
+CREATE INDEX idx1 ON collection2 (date: STRING, oid: STRING, sym: STRING, reg: BIGINT);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.2.update.sqlpp
new file mode 100644
index 0000000..41879a7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.2.update.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+upsert into collection1
+[
+ {
+ "id":1,
+ "sym":10,
+ "date": "2024-01-01"
+ }
+];
+
+upsert into collection2
+[
+ {
+ "id":1,
+ "sym":10,
+ "date": "2024-01-01"
+ }
+];
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.3.query.sqlpp
new file mode 100644
index 0000000..60f91ad
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.3.query.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description: This test case is to verify the fix for ASTERIXDB-3512
+ */
+
+use test;
+
+with tmp1 AS (
+ SELECT S.sym
+ FROM collection1 AS S
+ WHERE S.date < print_date(current_date(), 'YYYY-MM-DD')),
+ tmp2 AS (
+ SELECT O.sym
+ FROM collection2 AS O
+ WHERE O.date < print_date(current_date(), 'YYYY-MM-DD'))
+SELECT sl1.sym
+FROM tmp1 AS sl1
+UNION ALL
+SELECT sl2.sym
+FROM tmp2 as sl2;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.3.adm
new file mode 100644
index 0000000..9ea757d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.3.adm
@@ -0,0 +1,2 @@
+{ "sym": 10 }
+{ "sym": 10 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
index c4f73b0..2370d93 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
@@ -7517,6 +7517,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-3512">
+ <output-dir compare="Text">query-ASTERIXDB-3512</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
<compilation-unit name="remove_listify">
<output-dir compare="Text">remove_listify</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 949c0a3..aadc42b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -117,8 +117,8 @@
<expected-error>HYR0132: Extra field in the result, field 'second' does not exist at 'nested' in the schema</expected-error>
<expected-error>HYR0131: Result does not follow the schema, group type expected but found primitive type at 'nested'</expected-error>
<expected-error>HYR0131: Result does not follow the schema, primitive type expected but found group type at 'name'</expected-error>
- <expected-error>ASX1201: Storage units expected for the field 'row-group-size' (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...). Provided 'random'</expected-error>
- <expected-error>ASX1201: Storage units expected for the field 'page-size' (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...). Provided 'random'</expected-error>
+ <expected-error>ASX1206: Storage units expected for the field 'row-group-size' (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...). Provided 'random'</expected-error>
+ <expected-error>ASX1206: Storage units expected for the field 'page-size' (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...). Provided 'random'</expected-error>
<expected-error>ASX1202: Unsupported compression scheme rar. Supported schemes for parquet are [gzip, snappy, zstd]</expected-error>
<expected-error>ASX1001: Syntax error</expected-error>
<expected-error>ASX1204: 'binary' type not supported in parquet format</expected-error>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 7c89bc9..ea7cc3b 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -303,12 +303,12 @@
DUPLICATE_FIELD_IN_PRIMARY_KEY(1198),
INCOMPATIBLE_FIELDS_IN_PRIMARY_KEY(1199),
PREFIX_SHOULD_NOT_START_WITH_SLASH(1200),
- ILLEGAL_SIZE_PROVIDED(1201),
+ INVALID_DELTA_TABLE_FORMAT(1201),
UNSUPPORTED_WRITER_COMPRESSION_SCHEME(1202),
INVALID_PARQUET_SCHEMA(1203),
TYPE_UNSUPPORTED_PARQUET_WRITE(1204),
INVALID_PARQUET_WRITER_VERSION(1205),
- INVALID_DELTA_TABLE_FORMAT(1206),
+ ILLEGAL_SIZE_PROVIDED(1206),
// Feed errors
DATAFLOW_ILLEGAL_STATE(3001),
UTIL_DATAFLOW_UTILS_TUPLE_TOO_LARGE(3002),
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index d1e01f6..3e2f124 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -305,12 +305,12 @@
1198 = Duplicate field '%1$s' in primary key
1199 = Fields '%1$s' and '%2$s' are incompatible for primary key
1200 = Prefix should not start with "/". Prefix: '%1$s'
-1201 = Storage units expected for the field '%1$s' (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...). Provided '%2$s'
+1201 = Supported file format for 'delta' tables is 'parquet', but '%1$s' was provided.
1202 = Unsupported compression scheme %1$s. Supported schemes for %2$s are %3$s
1203 = Invalid schema provided: '%1$s'
1204 = '%1$s' type not supported in parquet format
1205 = Invalid Parquet Writer Version provided '%1$s'. Supported values: %2$s
-1206 = Supported file format for 'delta' tables is 'parquet', but '%1$s' was provided.
+1206 = Storage units expected for the field '%1$s' (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...). Provided '%2$s'
# Feed Errors
3001 = Illegal state.
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
index 956f0b7..84346fd 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
@@ -42,6 +42,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
@@ -163,9 +164,15 @@
if (!canRemoveOperator(op, opInSubplan, parentOp)) {
break;
}
- op = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
- opRef.setValue(op);
- assignVarsSetForThisOp = removeAssignVarFromConsideration(opRef);
+ if (op.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
+ op = new EmptyTupleSourceOperator();
+ context.computeAndSetTypeEnvironmentForOperator(op);
+ opRef.setValue(op);
+ } else {
+ op = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ opRef.setValue(op);
+ assignVarsSetForThisOp = removeAssignVarFromConsideration(opRef);
+ }
isTransformed = true;
}
@@ -445,7 +452,7 @@
private static boolean canRemoveOperator(ILogicalOperator op, boolean opInsideSubplan, ILogicalOperator parentOp) {
LogicalOperatorTag opTag = op.getOperatorTag();
- if (opTag == LogicalOperatorTag.AGGREGATE || opTag == LogicalOperatorTag.UNIONALL) {
+ if ((opTag == LogicalOperatorTag.AGGREGATE && opInsideSubplan) || opTag == LogicalOperatorTag.UNIONALL) {
return false;
}
if (!opInsideSubplan) {