ASTERIXDB-1157: Pushdown Limit
- Limit Pushdown into an Order (ExternalSort) operator.
- ConstantFoldingRule is not applied on the record-related operations.
Change-Id: I19aa4ce402b1834d9f68320acb72d7635a41a837
Reviewed-on: https://asterix-gerrit.ics.uci.edu/617
Reviewed-by: Yingyi Bu <buyingyi@gmail.com>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterix-app/src/test/resources/optimizerts/queries/orderby_limit_01.aql b/asterix-app/src/test/resources/optimizerts/queries/orderby_limit_01.aql
new file mode 100644
index 0000000..bf9aea3
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries/orderby_limit_01.aql
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : This test is intended to verify that the primary BTree index is used
+ * : in the optimized query plan and LIMIT push-down is applied on an external sort with a non primary key field.
+ * Expected Result : Success
+ */
+
+// Please note this is a Positive test and the BTree index should be used in the plan.
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+write output to asterix_nc1:"rttest/orderby_limit_01.adm";
+
+create type MyRecord as closed {
+ id: int64,
+ docid: int64,
+ val1: int64,
+ title: string,
+ point: point,
+ kwds: string,
+ line1: line,
+ line2: line,
+ poly1: polygon,
+ poly2: polygon,
+ rec: rectangle,
+ circle: circle
+}
+
+create dataset MyData(MyRecord)
+ primary key id;
+
+create index btree_index_docid on MyData(docid) type btree;
+create index btree_index_val1 on MyData(val1) type btree;
+create index rtree_index_point on MyData(point) type rtree;
+create index rtree_index_rec on MyData(rec) type rtree;
+create index ngram_index_title on MyData(title) type ngram(3);
+create index keyword_index_title on MyData(title) type keyword;
+
+for $o in dataset('MyData')
+where $o.id < 50
+order by $o.docid
+limit 2
+return {"pk":$o.id, "sk":$o.docid}
diff --git a/asterix-app/src/test/resources/optimizerts/queries/orderby_limit_offset_01.aql b/asterix-app/src/test/resources/optimizerts/queries/orderby_limit_offset_01.aql
new file mode 100644
index 0000000..9059c35
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries/orderby_limit_offset_01.aql
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : This test is intended to verify that the primary BTree index is used
+ * : in the optimized query plan and LIMIT with an offset push-down is applied on an external sort with a non primary key field..
+ * Expected Result : Success
+ */
+
+// Please note this is a Positive test and the BTree index should be used in the plan.
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+write output to asterix_nc1:"rttest/orderby_limit_offset_01.adm";
+
+create type MyRecord as closed {
+ id: int64,
+ docid: int64,
+ val1: int64,
+ title: string,
+ point: point,
+ kwds: string,
+ line1: line,
+ line2: line,
+ poly1: polygon,
+ poly2: polygon,
+ rec: rectangle,
+ circle: circle
+}
+
+create dataset MyData(MyRecord)
+ primary key id;
+
+create index btree_index_docid on MyData(docid) type btree;
+create index btree_index_val1 on MyData(val1) type btree;
+create index rtree_index_point on MyData(point) type rtree;
+create index rtree_index_rec on MyData(rec) type rtree;
+create index ngram_index_title on MyData(title) type ngram(3);
+create index keyword_index_title on MyData(title) type keyword;
+
+for $o in dataset('MyData')
+where $o.id < 50
+order by $o.docid
+limit 2 offset 2
+return {"pk":$o.id, "sk":$o.docid}
diff --git a/asterix-app/src/test/resources/optimizerts/queries/orderby_limit_primary_index_01.aql b/asterix-app/src/test/resources/optimizerts/queries/orderby_limit_primary_index_01.aql
new file mode 100644
index 0000000..429cf09
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries/orderby_limit_primary_index_01.aql
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : This test is intended to verify that the primary BTree index is used
+ * : in the optimized query plan and LIMIT push-down is applied on an external sort.
+ * Expected Result : Success
+ */
+
+// Please note this is a Positive test and the BTree index should be used in the plan.
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+write output to asterix_nc1:"rttest/orderby_limit_primary_index_01.adm";
+
+create type MyRecord as closed {
+ id: int64,
+ docid: int64,
+ val1: int64,
+ title: string,
+ point: point,
+ kwds: string,
+ line1: line,
+ line2: line,
+ poly1: polygon,
+ poly2: polygon,
+ rec: rectangle,
+ circle: circle
+}
+
+create dataset MyData(MyRecord)
+ primary key id;
+
+create index btree_index_docid on MyData(docid) type btree;
+create index btree_index_val1 on MyData(val1) type btree;
+create index rtree_index_point on MyData(point) type rtree;
+create index rtree_index_rec on MyData(rec) type rtree;
+create index ngram_index_title on MyData(title) type ngram(3);
+create index keyword_index_title on MyData(title) type keyword;
+
+for $o in dataset('MyData')
+where $o.id < 50
+order by $o.id
+limit 2
+return {"pk":$o.id, "sk":$o.docid}
diff --git a/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-primary-limit-orderby-01.plan b/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-primary-limit-orderby-01.plan
new file mode 100644
index 0000000..e5eca71
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-primary-limit-orderby-01.plan
@@ -0,0 +1,17 @@
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ -- STREAM_PROJECT |UNPARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- STREAM_LIMIT |UNPARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$10(ASC) ] |PARTITIONED|
+ -- STREAM_LIMIT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$10(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-primary-limit-orderby-02.plan b/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-primary-limit-orderby-02.plan
new file mode 100644
index 0000000..e7039b2
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/btree-index/btree-primary-limit-orderby-02.plan
@@ -0,0 +1,12 @@
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ -- STREAM_PROJECT |UNPARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- STREAM_LIMIT |UNPARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$9(ASC) ] |PARTITIONED|
+ -- STREAM_LIMIT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterix-app/src/test/resources/optimizerts/results/inlined_q18_large_volume_customer.plan b/asterix-app/src/test/resources/optimizerts/results/inlined_q18_large_volume_customer.plan
index 8b205d6..92f9a29 100644
--- a/asterix-app/src/test/resources/optimizerts/results/inlined_q18_large_volume_customer.plan
+++ b/asterix-app/src/test/resources/optimizerts/results/inlined_q18_large_volume_customer.plan
@@ -6,7 +6,7 @@
-- SORT_MERGE_EXCHANGE [$$12(DESC), $$11(ASC) ] |PARTITIONED|
-- STREAM_LIMIT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$12(DESC), $$11(ASC)] |PARTITIONED|
+ -- STABLE_SORT [topK: 100] [$$12(DESC), $$11(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- PRE_CLUSTERED_GROUP_BY[$$72, $$73] |PARTITIONED|
{
diff --git a/asterix-app/src/test/resources/optimizerts/results/orderby_limit_01.plan b/asterix-app/src/test/resources/optimizerts/results/orderby_limit_01.plan
new file mode 100644
index 0000000..6f139d6
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/orderby_limit_01.plan
@@ -0,0 +1,17 @@
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ -- STREAM_PROJECT |UNPARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- STREAM_LIMIT |UNPARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$10(ASC) ] |PARTITIONED|
+ -- STREAM_LIMIT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [topK: 2] [$$10(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterix-app/src/test/resources/optimizerts/results/orderby_limit_offset_01.plan b/asterix-app/src/test/resources/optimizerts/results/orderby_limit_offset_01.plan
new file mode 100644
index 0000000..ee46a5c
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/orderby_limit_offset_01.plan
@@ -0,0 +1,17 @@
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ -- STREAM_PROJECT |UNPARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- STREAM_LIMIT |UNPARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$10(ASC) ] |PARTITIONED|
+ -- STREAM_LIMIT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [topK: 4] [$$10(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterix-app/src/test/resources/optimizerts/results/orderby_limit_primary_index_01.plan b/asterix-app/src/test/resources/optimizerts/results/orderby_limit_primary_index_01.plan
new file mode 100644
index 0000000..e7039b2
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/results/orderby_limit_primary_index_01.plan
@@ -0,0 +1,12 @@
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ -- STREAM_PROJECT |UNPARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- STREAM_LIMIT |UNPARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$9(ASC) ] |PARTITIONED|
+ -- STREAM_LIMIT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterix-app/src/test/resources/optimizerts/results/q03_shipping_priority.plan b/asterix-app/src/test/resources/optimizerts/results/q03_shipping_priority.plan
index 0618b41..c4c0c05 100644
--- a/asterix-app/src/test/resources/optimizerts/results/q03_shipping_priority.plan
+++ b/asterix-app/src/test/resources/optimizerts/results/q03_shipping_priority.plan
@@ -6,7 +6,7 @@
-- SORT_MERGE_EXCHANGE [$$49(DESC), $$4(ASC) ] |PARTITIONED|
-- STREAM_LIMIT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$49(DESC), $$4(ASC)] |PARTITIONED|
+ -- STABLE_SORT [topK: 10] [$$49(DESC), $$4(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$55, $$56, $$57] |PARTITIONED|
{
diff --git a/asterix-app/src/test/resources/optimizerts/results/q2.plan b/asterix-app/src/test/resources/optimizerts/results/q2.plan
index 14382c1..87aa995 100644
--- a/asterix-app/src/test/resources/optimizerts/results/q2.plan
+++ b/asterix-app/src/test/resources/optimizerts/results/q2.plan
@@ -6,7 +6,7 @@
-- SORT_MERGE_EXCHANGE [$$26(DESC) ] |PARTITIONED|
-- STREAM_LIMIT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$26(DESC)] |PARTITIONED|
+ -- STABLE_SORT [topK: 5] [$$26(DESC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- PRE_CLUSTERED_GROUP_BY[$$32] |PARTITIONED|
{
diff --git a/asterix-app/src/test/resources/optimizerts/results/sort-cust.plan b/asterix-app/src/test/resources/optimizerts/results/sort-cust.plan
index 66e6049..6faec89 100644
--- a/asterix-app/src/test/resources/optimizerts/results/sort-cust.plan
+++ b/asterix-app/src/test/resources/optimizerts/results/sort-cust.plan
@@ -6,7 +6,7 @@
-- SORT_MERGE_EXCHANGE [$$7(ASC) ] |PARTITIONED|
-- STREAM_LIMIT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$7(ASC)] |PARTITIONED|
+ -- STABLE_SORT [topK: 10] [$$7(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|