[ASTERIXDB-1946][STO][IDX] Create BTreeIndex for Correlated Datasets
Implemented create seconary BTree index for datasets using
correlated merge policy. Instead of creating one component
for the new index, this change creates one component for each
component of the primary index to maintain the correlation.
The current implementation assumes when a secondary index is being
created, the dataset is locked with no modifications.
Change-Id: I2a3435e6720f07bd6a5092d4d9ce42e8d4b7894c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1813
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <buyingyi@gmail.com>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 65268b1..f7327b4 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -952,6 +952,11 @@
// #. create the index artifact in NC.
runJob(hcc, spec);
+ // #. flush the internal dataset for correlated policy
+ if (ds.isCorrelated() && ds.getDatasetType() == DatasetType.INTERNAL) {
+ FlushDatasetUtil.flushDataset(hcc, metadataProvider, dataverseName, datasetName, datasetName);
+ }
+
mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.1.ddl.aql
new file mode 100644
index 0000000..e4a402e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.1.ddl.aql
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-secondary-index-nullable.aql
+ * Description : This test is intended to test deletion from correlated secondary btree indexes that
+ * are built on nullable fields
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type AddressType as closed {
+ number: int64,
+ street: string,
+ city: string
+}
+
+create type CustomerType as closed {
+ cid: int64,
+ name: string,
+ age: int64?,
+ address: AddressType?,
+ interests: {{string}},
+ children: [ { name: string, age: int64? } ]
+}
+
+create dataset Customers(CustomerType) primary key cid
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+create feed CustomerFeed
+using localfs
+(("path"="asterix_nc1://data/semistructured/co1k/customer.adm"),
+("format"="adm"),
+("type-name"="CustomerType"),
+("tuple-interval"="10"));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.2.update.aql
new file mode 100644
index 0000000..4fcc858
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.2.update.aql
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-secondary-index-nullable.aql
+ * Description : This test is intended to test deletion from correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+use dataverse test;
+
+set "wait-for-completion-feed" "true";
+
+connect feed CustomerFeed to dataset Customers;
+
+start feed CustomerFeed;
+
+delete $c from dataset Customers where $c.cid>=200;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.3.ddl.aql
new file mode 100644
index 0000000..c19ef10
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.3.ddl.aql
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+create index age_index on Customers(age);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.4.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.4.query.aql
new file mode 100644
index 0000000..4b8ff9a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.4.query.aql
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-secondary-index-nullable.aql
+ * Description : This test is intended to test deletion from correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+use dataverse test;
+
+for $c in dataset('Customers')
+where $c.age < 20
+order by $c.cid
+return $c
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.1.ddl.aql
new file mode 100644
index 0000000..c540b1f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.1.ddl.aql
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-correlated-secondary-index-open.aql
+ * Description : This test is intended to test deletion from correlated secondary btree indexes
+ * that are built on open fields
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type AddressType as closed {
+ number: int64,
+ street: string,
+ city: string
+}
+
+create type CustomerType as closed {
+ cid: int64,
+ name: string,
+ age: int64?,
+ address: AddressType?,
+ interests: {{string}},
+ children: [ { name: string, age: int64? } ]
+}
+
+create type CustomerOpenType as open {
+ cid: int64,
+ name: string,
+ address: AddressType?,
+ interests: {{string}},
+ children: [ { name: string, age: int64? } ]
+}
+
+create dataset Customers(CustomerType) primary key cid
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+create dataset CustomersOpen(CustomerOpenType) primary key cid
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.2.update.aql
new file mode 100644
index 0000000..1c24546
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.2.update.aql
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse test;
+
+load dataset Customers
+using localfs
+(("path"="asterix_nc1://data/semistructured/co1k/customer.adm"),("format"="adm"));
+
+insert into dataset test.CustomersOpen (
+ for $x in dataset test.Customers
+ return {
+ "cid": $x.cid,
+ "name": $x.name,
+ "age": $x.age,
+ "address": $x.address,
+ "interests": $x.interests,
+ "children": $x.children
+ }
+);
+
+delete $c from dataset CustomersOpen where $c.cid>=200;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.3.ddl.aql
new file mode 100644
index 0000000..7c358d2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.3.ddl.aql
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+create index age_index on CustomersOpen(age:int32?) enforced;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.4.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.4.query.aql
new file mode 100644
index 0000000..33d6039
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.4.query.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse test;
+
+for $c in dataset('CustomersOpen')
+where $c.age < 20
+order by $c.cid
+return $c
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.1.ddl.aql
new file mode 100644
index 0000000..c846c06
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.1.ddl.aql
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-correlated-secondary-index-nullable.aql
+ * Description : This test is intended to test insertion into correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type AddressType as closed {
+ number: int64,
+ street: string,
+ city: string
+}
+
+create type CustomerType as closed {
+ cid: int64,
+ name: string,
+ age: int64?,
+ address: AddressType?,
+ interests: {{string}},
+ children: [ { name: string, age: int64? } ]
+}
+
+create dataset Customers(CustomerType) primary key cid
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+create dataset CustomersMini(CustomerType) primary key cid
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.2.update.aql
new file mode 100644
index 0000000..833c400
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.2.update.aql
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-correlated-secondary-index-nullable.aql
+ * Description : This test is intended to test insertion into correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+use dataverse test;
+
+load dataset Customers
+using localfs
+(("path"="asterix_nc1://data/semistructured/co1k/customer.adm"),("format"="adm"));
+
+insert into dataset CustomersMini
+(
+ for $c in dataset('Customers')
+ where $c.cid < 200
+ return {
+ "cid": $c.cid,
+ "name": $c.name,
+ "age": $c.age,
+ "address": $c.address,
+ "interests": $c.interests,
+ "children": $c.children
+ }
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.3.ddl.aql
new file mode 100644
index 0000000..f54c11d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.3.ddl.aql
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+create index age_index on CustomersMini(age);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.4.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.4.query.aql
new file mode 100644
index 0000000..6b70584
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.4.query.aql
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-correlated-secondary-index-nullable.aql
+ * Description : This test is intended to test insertion into correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+
+use dataverse test;
+
+for $c in dataset('CustomersMini')
+where $c.age < 20
+order by $c.cid
+return $c
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.1.ddl.aql
new file mode 100644
index 0000000..09e4d94
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.1.ddl.aql
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-correlated-secondary-index-open.aql
+ * Description : This test is intended to test insertion into correlated secondary btree indexes
+ * that are built on open fields
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type AddressType as closed {
+ number: int64,
+ street: string,
+ city: string
+}
+
+create type CustomerType as closed {
+ cid: int64,
+ name: string,
+ age: int64?,
+ address: AddressType?,
+ interests: {{string}},
+ children: [ { name: string, age: int64? } ]
+}
+
+create type CustomerOpenType as open {
+ cid: int64,
+ name: string,
+ address: AddressType?,
+ interests: {{string}},
+ children: [ { name: string, age: int64? } ]
+}
+
+create dataset Customers(CustomerType) primary key cid
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+create dataset CustomersOpen(CustomerOpenType) primary key cid
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.2.update.aql
new file mode 100644
index 0000000..8883bbf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.2.update.aql
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-correlated-secondary-index-open.aql
+ * Description : This test is intended to test insertion into correlated secondary btree indexes
+ * that are built on open fields
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+use dataverse test;
+
+load dataset Customers
+using localfs
+(("path"="asterix_nc1://data/semistructured/co1k/customer.adm"),("format"="adm"));
+
+insert into dataset CustomersOpen
+(
+ for $c in dataset('Customers')
+ where $c.cid < 200
+ return {
+ "cid": $c.cid,
+ "name": $c.name,
+ "age": $c.age,
+ "address": $c.address,
+ "interests": $c.interests,
+ "children": $c.children
+ }
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.3.ddl.aql
new file mode 100644
index 0000000..38baed5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.3.ddl.aql
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+create index age_index on CustomersOpen(age:int32?) enforced;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.4.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.4.query.aql
new file mode 100644
index 0000000..a0ffb86
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.4.query.aql
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-correlated-secondary-index-open.aql
+ * Description : This test is intended to test insertion into correlated secondary btree indexes
+ * that are built on open fields
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+use dataverse test;
+
+for $c in dataset('CustomersOpen')
+where $c.age < 20
+order by $c.cid
+return $c
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql
new file mode 100644
index 0000000..ab50500
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : using-correlated-prefix-merge-policy-with-feed.aql
+ * Description : This test is inteded to test the correlated prefix merge policy, and create secondary
+ * index for datasets using this policy
+ * Expected Result : Success
+ * Date : June 9 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type LineItemType as closed {
+ l_orderkey: int64,
+ l_partkey: int64,
+ l_suppkey: int64,
+ l_linenumber: int64,
+ l_quantity: int64,
+ l_extendedprice: double,
+ l_discount: double,
+ l_tax: double,
+ l_returnflag: string,
+ l_linestatus: string,
+ l_shipdate: string,
+ l_commitdate: string,
+ l_receiptdate: string,
+ l_shipinstruct: string,
+ l_shipmode: string,
+ l_comment: string
+}
+
+create dataset LineItem(LineItemType)
+ primary key l_orderkey, l_linenumber using compaction policy correlated-prefix
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+create feed LineItemFeed
+using localfs
+(("path"="asterix_nc1://data/tpch0.001/lineitem.tbl"),
+("format"="delimited-text"),
+("delimiter"="|"),
+("type-name"="LineItemType"),
+("tuple-interval"="10"));
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql
new file mode 100644
index 0000000..f5cf29b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse test;
+
+set wait-for-completion-feed "true";
+
+connect feed LineItemFeed to dataset LineItem;
+
+start feed LineItemFeed;
+
+delete $l from dataset LineItem where $l.l_suppkey>=2 or $l.l_linenumber>1;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.ddl.aql
new file mode 100644
index 0000000..4a96c91
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.ddl.aql
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+create index idx_LineItem_partkey on LineItem(l_linenumber);
+create index idx_LineItem_suppkey on LineItem(l_suppkey);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql
new file mode 100644
index 0000000..e4c670a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse test;
+
+for $c in dataset('LineItem')
+where $c.l_suppkey<150
+order by $c.l_orderkey, $c.l_linenumber
+return $c
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.1.ddl.aql
new file mode 100644
index 0000000..511775d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.1.ddl.aql
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test filters with insert pipeline in the existence of a correlated secondary b-tree
+ * Expected Res : Success
+ * Date : 16 June 2017
+ */
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+create type FacebookMessageType as closed {
+ message-id: int64,
+ author-id: int64,
+ in-response-to: int64?,
+ sender-location: point?,
+ message: string,
+ send-time: datetime
+}
+
+create dataset FacebookMessages(FacebookMessageType)
+primary key message-id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+
+create dataset FacebookMessages2(FacebookMessageType)
+primary key message-id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"))
+with filter on send-time;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.2.update.aql
new file mode 100644
index 0000000..0e0575a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.2.update.aql
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+load dataset FacebookMessages using localfs
+(("path"="asterix_nc1://data/fbm-with-send-time.adm"),("format"="adm"));
+
+insert into dataset FacebookMessages2 (
+for $m in dataset('FacebookMessages')
+ return $m
+);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.3.ddl.aql
new file mode 100644
index 0000000..1bfb269
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.3.ddl.aql
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+create index fbAuthorIdx on FacebookMessages2(author-id) type btree;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.4.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.4.query.aql
new file mode 100644
index 0000000..df3092c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.4.query.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+for $m in dataset('FacebookMessages2')
+where $m.author-id = 1
+and $m.send-time > datetime("2012-08-20T10:10:00")
+and $m.send-time < datetime("2012-11-20T10:10:00")
+return $m
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.1.ddl.aql
new file mode 100644
index 0000000..6834cde
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.1.ddl.aql
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : This test is intended to test inserting into a temporary dataset that has a correlated
+ * secondary index and scan the data.
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+create type test.Emp as closed {
+id: int64,
+fname: string,
+lname: string,
+age: int64,
+dept: string
+}
+
+create temporary dataset test.employee(Emp) primary key id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.2.update.aql
new file mode 100644
index 0000000..c68a59e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.2.update.aql
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : This test is intended to test inserting into a temporary dataset that has a correlated
+ * secondary index and scan the data.
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+use dataverse test;
+
+load dataset test.employee
+using localfs
+(("path"="asterix_nc1://data/names.adm"),("format"="delimited-text"),("delimiter"="|"));
+
+
+insert into dataset test.employee (
+for $x in dataset test.employee
+return {
+ "id": $x.id + 10000,
+ "fname": $x.fname,
+ "lname": $x.lname,
+ "age": $x.age,
+ "dept": $x.dept
+}
+);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.3.ddl.aql
new file mode 100644
index 0000000..00b0850
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.3.ddl.aql
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : This test is intended to test inserting into a temporary dataset that has a correlated
+ * secondary index and scan the data.
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+use dataverse test;
+
+create index idx_employee_first_name on test.employee(fname);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.4.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.4.query.aql
new file mode 100644
index 0000000..195f11e7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.4.query.aql
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : This test is intended to test inserting into a temporary dataset that has a correlated
+ * secondary index and scan the data.
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+use dataverse test;
+
+for $l in dataset('test.employee')
+order by $l.id
+return $l
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.1.ddl.aql
new file mode 100644
index 0000000..5bb3985
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.1.ddl.aql
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Upsert into a dataset which has a b-tree correlated secondary index
+ * Expected Res : Success
+ * Date : June 8 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TestType as closed{
+id:int32,
+age:int32,
+name:string,
+salary:double
+};
+
+create dataset UpsertTo("TestType") primary key id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+create dataset UpsertFrom("TestType") primary key id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.2.update.aql
new file mode 100644
index 0000000..2f9ac1e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.2.update.aql
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Upsert into a dataset which has a b-tree correlated secondary index
+ * Expected Res : Success
+ * Date : June 8 2017
+ */
+
+use dataverse test;
+// load first dataset
+load dataset UpsertTo using
+localfs(("format"="delimited-text"),
+ ("path"="asterix_nc1://data/upsert/raw-data/overlapping.data"),
+ ("delimiter"=","));
+// load second dataset
+load dataset UpsertFrom using
+localfs(("format"="delimited-text"),
+ ("path"="asterix_nc1://data/upsert/raw-data/test-data.txt,asterix_nc1://data/upsert/raw-data/more-data.txt"),
+ ("delimiter"=","));
+
+// upsert UpsertFrom into UpsertTo
+use dataverse test;
+upsert into dataset UpsertTo(
+ for $x in dataset UpsertFrom
+ return $x
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.3.ddl.aql
new file mode 100644
index 0000000..e46c96b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.3.ddl.aql
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Upsert into a dataset which has a b-tree correlated secondary index
+ * Expected Res : Success
+ * Date : June 08 2017
+ */
+
+use dataverse test;
+
+create index ageindex on UpsertTo('age');
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.4.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.4.query.aql
new file mode 100644
index 0000000..35711c1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.4.query.aql
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Upsert into a dataset which has a b-tree correlated secondary index
+ * Expected Res : Success
+ * Date : June 8 2017
+ */
+
+ // So far this one doesn't use the btree index, need another query
+use dataverse test;
+for $x in dataset UpsertTo
+where $x.age >5
+return $x;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.1.ddl.sqlpp
new file mode 100644
index 0000000..64a9d9b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.1.ddl.sqlpp
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-correlated-secondary-index-nullable.aql
+ * Description : This test is intended to test deletion from correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date : June 7 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+ number : bigint,
+ street : string,
+ city : string
+}
+
+create type test.CustomerType as
+ closed {
+ cid : bigint,
+ name : string,
+ age : bigint?,
+ address : AddressType?,
+ interests : {{string}},
+ children : [{
+ name : string,
+ age : bigint?
+ }
+]
+}
+
+create dataset Customers(CustomerType) primary key cid
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+
+create feed CustomerFeed
+using localfs
+(("path"="asterix_nc1://data/semistructured/co1k/customer.adm"),
+("format"="adm"),
+("type-name"="CustomerType"),
+("tuple-interval"="10"));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.2.update.sqlpp
new file mode 100644
index 0000000..57b7529
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.2.update.sqlpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-correlated-secondary-index-nullable.aql
+ * Description : This test is intended to test deletion from correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date : June 07 2017
+ */
+
+use test;
+
+set `wait-for-completion-feed` "true";
+
+connect feed CustomerFeed to dataset Customers;
+
+start feed CustomerFeed;
+
+delete from Customers
+ where cid >= 200;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.3.ddl.sqlpp
new file mode 100644
index 0000000..8934610
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.3.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index age_index on Customers (age) type btree;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.4.query.sqlpp
new file mode 100644
index 0000000..1b71786
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.4.query.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-correlated-secondary-index-nullable.aql
+ * Description : This test is intended to test deletion from correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date : June 07 2017
+ */
+
+use test;
+
+select element c
+from Customers as c
+where (c.age < 20)
+order by c.cid
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.1.ddl.sqlpp
new file mode 100644
index 0000000..7d54035
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.1.ddl.sqlpp
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-correlated-secondary-index-open.sqlpp
+ * Description : This test is intended to test deletion from secondary correlated btree indexes that
+ * are built on open fields
+ * Expected Result : Success
+ * Date : June 7 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+ number : bigint,
+ street : string,
+ city : string
+}
+
+create type test.CustomerType as
+ closed {
+ cid : bigint,
+ name : string,
+ age : bigint?,
+ address : AddressType?,
+ interests : {{string}},
+ children : [{
+ name : string,
+ age : bigint?
+ }
+]
+}
+
+create type test.CustomerOpenType as
+{
+ cid : bigint,
+ name : string,
+ address : AddressType?,
+ interests : {{string}},
+ children : [{
+ name : string,
+ age : bigint?
+ }
+]
+}
+
+create dataset Customers(CustomerType) primary key cid
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset CustomersOpen(CustomerOpenType) primary key cid
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.2.update.sqlpp
new file mode 100644
index 0000000..47f0031
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.2.update.sqlpp
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+load dataset Customers using localfs ((`path`=`asterix_nc1://data/semistructured/co1k/customer.adm`),(`format`=`adm`));
+
+insert into test.CustomersOpen
+select element {'cid':x.cid,'name':x.name,'age':x.age,'address':x.address,'interests':x.interests,'children':x.children}
+from `test.Customers` as x
+;
+
+delete from CustomersOpen
+ where cid >= 200;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.3.ddl.sqlpp
new file mode 100644
index 0000000..f3da8bc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.3.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index age_index on CustomersOpen (age:integer?) type btree enforced;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.4.query.sqlpp
new file mode 100644
index 0000000..c6edd1b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element c
+from CustomersOpen as c
+where (c.age < 20)
+order by c.cid
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.1.ddl.sqlpp
new file mode 100644
index 0000000..ab982b3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.1.ddl.sqlpp
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-secondary-index-nullable.aql
+ * Description : This test is intended to test insertion into correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date : June 7 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+ number : bigint,
+ street : string,
+ city : string
+}
+
+create type test.CustomerType as
+ closed {
+ cid : bigint,
+ name : string,
+ age : bigint?,
+ address : AddressType?,
+ interests : {{string}},
+ children : [{
+ name : string,
+ age : bigint?
+ }
+]
+}
+
+create dataset Customers(CustomerType) primary key cid
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset CustomersMini(CustomerType) primary key cid
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.2.update.sqlpp
new file mode 100644
index 0000000..29268ee
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.2.update.sqlpp
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-secondary-index-nullable.aql
+ * Description : This test is intended to test insertion into correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date : June 7 2017
+ */
+
+use test;
+
+
+load dataset Customers using localfs ((`path`=`asterix_nc1://data/semistructured/co1k/customer.adm`),(`format`=`adm`));
+
+
+insert into CustomersMini
+select element {'cid':c.cid,'name':c.name,'age':c.age,'address':c.address,'interests':c.interests,'children':c.children}
+from Customers as c
+where (c.cid < 200)
+;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.3.ddl.sqlpp
new file mode 100644
index 0000000..eef04e4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.3.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+create index age_index on CustomersMini (age) type btree;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.4.query.sqlpp
new file mode 100644
index 0000000..69312f0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.4.query.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-secondary-index-nullable.aql
+ * Description : This test is intended to test insertion into correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date : June 7 2017
+ */
+
+use test;
+
+
+select element c
+from CustomersMini as c
+where (c.age < 20)
+order by c.cid
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.1.ddl.sqlpp
new file mode 100644
index 0000000..d43a96d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.1.ddl.sqlpp
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-correlated-secondary-index-open.aql
+ * Description : This test is intended to test insertion into secondary btree indexes that are built on open fields
+ * Expected Result : Success
+ * Date : June 7 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+ number : bigint,
+ street : string,
+ city : string
+}
+
+create type test.CustomerType as
+ closed {
+ cid : bigint,
+ name : string,
+ age : bigint?,
+ address : AddressType?,
+ interests : {{string}},
+ children : [{
+ name : string,
+ age : bigint?
+ }
+]
+}
+
+create type test.CustomerOpenType as
+{
+ cid : bigint,
+ name : string,
+ address : AddressType?,
+ interests : {{string}},
+ children : [{
+ name : string,
+ age : bigint?
+ }
+]
+}
+
+create dataset Customers(CustomerType) primary key cid
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset CustomersOpen(CustomerOpenType) primary key cid
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.2.update.sqlpp
new file mode 100644
index 0000000..1019c35
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.2.update.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-correlated-secondary-index-open.aql
+ * Description : This test is intended to test insertion into secondary btree indexes that are built on open fields
+ * Expected Result : Success
+ * Date : June 7 2017
+ */
+
+use test;
+
+
+load dataset Customers using localfs ((`path`=`asterix_nc1://data/semistructured/co1k/customer.adm`),(`format`=`adm`));
+
+insert into CustomersOpen
+select element {'cid':c.cid,'name':c.name,'age':c.age,'address':c.address,'interests':c.interests,'children':c.children}
+from Customers as c
+where (c.cid < 200)
+;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.3.ddl.sqlpp
new file mode 100644
index 0000000..f3da8bc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.3.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index age_index on CustomersOpen (age:integer?) type btree enforced;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.4.query.sqlpp
new file mode 100644
index 0000000..4550bbf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.4.query.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-btree-correlated-secondary-index-open.aql
+ * Description : This test is intended to test insertion into secondary btree indexes that are built on open fields
+ * Expected Result : Success
+ * Date : June 7 2017
+ */
+
+use test;
+
+
+select element c
+from CustomersOpen as c
+where (c.age < 20)
+order by c.cid
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.sqlpp
new file mode 100644
index 0000000..f86b5a3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.sqlpp
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : using-correlated-prefix-merge-policy-with-feed.sqlpp
+ * Description : This test is inteded to test the correlated prefix merge policy, and create secondary
+ * index for datasets using this policy
+ * Expected Result : Success
+ * Date : June 9 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type LineItemType as closed {
+ l_orderkey: int64,
+ l_partkey: int64,
+ l_suppkey: int64,
+ l_linenumber: int64,
+ l_quantity: int64,
+ l_extendedprice: double,
+ l_discount: double,
+ l_tax: double,
+ l_returnflag: string,
+ l_linestatus: string,
+ l_shipdate: string,
+ l_commitdate: string,
+ l_receiptdate: string,
+ l_shipinstruct: string,
+ l_shipmode: string,
+ l_comment: string
+}
+
+create dataset LineItem(LineItemType)
+ primary key l_orderkey, l_linenumber using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create feed LineItemFeed
+using localfs
+(("path"="asterix_nc1://data/tpch0.001/lineitem.tbl"),
+("format"="delimited-text"),
+("delimiter"="|"),
+("type-name"="LineItemType"),
+("tuple-interval"="10"));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.sqlpp
new file mode 100644
index 0000000..6443388
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+set `wait-for-completion-feed` "true";
+
+connect feed LineItemFeed to dataset LineItem;
+
+start feed LineItemFeed;
+
+delete from LineItem
+ where l_suppkey >= 2 or l_linenumber > 1;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.ddl.sqlpp
new file mode 100644
index 0000000..fff1e39
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.ddl.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+create index idx_LineItem_partkey on LineItem (l_linenumber) type btree;
+
+create index idx_LineItem_suppkey on LineItem (l_suppkey) type btree;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.5.query.sqlpp
new file mode 100644
index 0000000..9a02491
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.5.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select element c
+from LineItem as c
+where (c.l_suppkey < 150)
+order by c.l_orderkey,c.l_linenumber;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.1.ddl.sqlpp
new file mode 100644
index 0000000..7568e18
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.1.ddl.sqlpp
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test filters with insert pipeline in the existence of a correlated secondary b-tree
+ * Expected Res : Success
+ * Date : June 7 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.FacebookMessageType as
+ closed {
+ `message-id` : bigint,
+ `author-id` : bigint,
+ `in-response-to` : bigint?,
+ `sender-location` : point?,
+ message : string,
+ `send-time` : datetime
+}
+
+create dataset FacebookMessages(FacebookMessageType) primary key `message-id`
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset FacebookMessages2(FacebookMessageType) primary key `message-id`
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`))
+ with filter on `send-time`;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.2.update.sqlpp
new file mode 100644
index 0000000..3a14ccb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.2.update.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+load dataset FacebookMessages using localfs ((`path`=`asterix_nc1://data/fbm-with-send-time.adm`),(`format`=`adm`));
+
+insert into FacebookMessages2
+select element m
+from FacebookMessages as m
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.3.ddl.sqlpp
new file mode 100644
index 0000000..cbc3aa8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.3.ddl.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test filters with insert pipeline in the existence of a correlated secondary b-tree
+ * Expected Res : Success
+ * Date : June 7 2017
+ */
+
+use test;
+
+create index fbAuthorIdx on FacebookMessages2 (`author-id`) type btree;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.4.query.sqlpp
new file mode 100644
index 0000000..f6086e1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.4.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element m
+from FacebookMessages2 as m
+where ((m.`author-id` = 1) and (m.`send-time` > test.datetime('2012-08-20T10:10:00')) and (m.`send-time` < test.datetime('2012-11-20T10:10:00')))
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.1.ddl.sqlpp
new file mode 100644
index 0000000..cea4c40
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.1.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+* Description : Index creation and selection for non-enforced indexes on correlated datasets
+* Expected Res : Success
+* Date : 18 Jun 2017
+*/
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type TestOpenType as open {
+ c_id: int64
+}
+
+create dataset TestOpen(TestOpenType)
+primary key c_id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`="16384"),(`max-tolerance-component-count`="3"));
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.10.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.10.query.sqlpp
new file mode 100644
index 0000000..bb643f6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.10.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select t.c_x as res
+from TestOpen t
+where t.c_d >= 3.25
+order by t.c_x;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.11.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.11.query.sqlpp
new file mode 100644
index 0000000..85bb0ea
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.11.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select t.c_x as res
+from TestOpen t
+where t.c_i8 > 499 and t.c_i8 < 99999
+order by t.c_x;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.2.update.sqlpp
new file mode 100644
index 0000000..eba8642
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.2.update.sqlpp
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+insert into TestOpen ({
+ "c_id": 1,
+ "c_x": 1,
+ "c_s": "hello",
+ "c_i64": 2,
+ "c_i8": 2,
+ "c_d": 2
+});
+insert into TestOpen ({
+ "c_id": 2,
+ "c_x": 2,
+ "c_s": 2,
+ "c_i64": "2",
+ "c_i8": 2.5,
+ "c_d": 3
+});
+insert into TestOpen ({
+ "c_id": 3,
+ "c_x": 3,
+ "c_s": "world",
+ "c_i64": 2,
+ "c_i8": 4,
+ "c_d": 3.125
+});
+insert into TestOpen ({
+ "c_id": 4,
+ "c_x": 4,
+ "c_s": null,
+ "c_i64": null,
+ "c_i8": 500,
+ "c_d": 3.25
+});
+insert into TestOpen ({
+ "c_id": 5,
+ "c_x": 5,
+ "c_s": "hello",
+ "c_i64": 2.25,
+ "c_i8": 10000.25,
+ "c_d": 3.5
+});
+insert into TestOpen ({
+ "c_id": 6,
+ "c_x": 6,
+ "c_s": false,
+ "c_i64": false,
+ "c_i8": 2e100,
+ "c_d": 2e100
+});
+insert into TestOpen ({
+ "c_id": 7,
+ "c_x": 7,
+ "c_s": "world",
+ "c_i64": 3
+});
+insert into TestOpen ({
+ "c_id": 8,
+ "c_x": 8
+});
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.3.ddl.sqlpp
new file mode 100644
index 0000000..efec192
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.3.ddl.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+create index idx_s on TestOpen(c_s:string);
+
+create index idx_i64 on TestOpen(c_i64:int64);
+
+create index idx_i8 on TestOpen(c_i8:int8);
+
+create index idx_d on TestOpen(c_d:double);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.4.query.sqlpp
new file mode 100644
index 0000000..fd0f674
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.4.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select t.c_x as res
+from TestOpen t
+where t.c_s = 'world'
+order by t.c_x;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.5.query.sqlpp
new file mode 100644
index 0000000..179d0c0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.5.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select t.c_x as res
+from TestOpen t
+where t.c_i64 = 2
+order by t.c_x;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.6.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.6.query.sqlpp
new file mode 100644
index 0000000..558769c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.6.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select t.c_x as res
+from TestOpen t
+where t.c_i64 > 2
+order by t.c_x;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.7.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.7.query.sqlpp
new file mode 100644
index 0000000..66fd4bd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.7.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select t.c_x as res
+from TestOpen t
+where t.c_i64 > 2.0
+order by t.c_x;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.8.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.8.query.sqlpp
new file mode 100644
index 0000000..1743ba7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.8.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select t.c_x as res
+from TestOpen t
+where t.c_i8 > 2
+order by t.c_x;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.9.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.9.query.sqlpp
new file mode 100644
index 0000000..0d6e0e3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.9.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select t.c_x as res
+from TestOpen t
+where t.c_i8 > 2.5
+order by t.c_x;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.1.ddl.sqlpp
new file mode 100644
index 0000000..c591a8e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.1.ddl.sqlpp
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : This test is intended to test inserting into a temporary dataset that has a correlated
+ * secondary index and scan the data at the same time where we insert a materializing to prevent the possibility
+ * of deadlatch.
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+create type test.Emp as
+ closed {
+ id : bigint,
+ fname : string,
+ lname : string,
+ age : bigint,
+ dept : string
+}
+
+create temporary dataset test.employee(Emp) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.2.update.sqlpp
new file mode 100644
index 0000000..969f8ef
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.2.update.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : This test is intended to test inserting into a temporary dataset that has a correlated
+ * secondary index and scan the data at the same time where we insert a materializing to prevent the possibility
+ * of deadlatch.
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+use test;
+
+
+load dataset test.employee using localfs ((`path`=`asterix_nc1://data/names.adm`),(`format`=`delimited-text`),(`delimiter`=`|`));
+
+insert into test.employee
+select element {'id':(x.id + 10000),'fname':x.fname,'lname':x.lname,'age':x.age,'dept':x.dept}
+from `test.employee` as x
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.3.ddl.sqlpp
new file mode 100644
index 0000000..0b0405c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.3.ddl.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : This test is intended to test inserting into a temporary dataset that has a correlated
+ * secondary index and scan the data at the same time where we insert a materializing to prevent the possibility
+ * of deadlatch.
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+use test;
+
+create index idx_employee_first_name on test.employee (fname) type btree;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.4.query.sqlpp
new file mode 100644
index 0000000..a179e16
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.4.query.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : This test is intended to test inserting into a temporary dataset that has a correlated
+ * secondary index and scan the data at the same time where we insert a materializing to prevent the possibility
+ * of deadlatch.
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+use test;
+
+
+select element l
+from `test.employee` as l
+order by l.id
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-btree/primary-secondary-btree.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-btree/primary-secondary-btree.1.ddl.sqlpp
new file mode 100644
index 0000000..071dc1f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-btree/primary-secondary-btree.1.ddl.sqlpp
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Upsert into a dataset which has a correlated b-tree secondary index
+ * Expected Res : Success
+ * Date : June 8 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type TestType as closed{
+id:int32,
+age:int32,
+name:string,
+salary:double
+};
+
+create dataset UpsertTo(TestType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset UpsertFrom(TestType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-btree/primary-secondary-btree.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-btree/primary-secondary-btree.2.update.sqlpp
new file mode 100644
index 0000000..deb491f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-btree/primary-secondary-btree.2.update.sqlpp
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Upsert into a dataset which has a correlated b-tree secondary index
+ * Expected Res : Success
+ * Date : June 8 2017
+ */
+
+
+use test;
+// load first dataset
+load dataset UpsertTo using
+localfs(("format"="delimited-text"),
+ ("path"="asterix_nc1://data/upsert/raw-data/overlapping.data"),
+ ("delimiter"=","));
+// load second dataset
+load dataset UpsertFrom using
+localfs(("format"="delimited-text"),
+ ("path"="asterix_nc1://data/upsert/raw-data/test-data.txt,asterix_nc1://data/upsert/raw-data/more-data.txt"),
+ ("delimiter"=","));
+
+// upsert UpsertFrom into UpsertTo
+upsert into UpsertTo(
+ select value x
+ from UpsertFrom x
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-btree/primary-secondary-btree.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-btree/primary-secondary-btree.3.ddl.sqlpp
new file mode 100644
index 0000000..6eea3b8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-btree/primary-secondary-btree.3.ddl.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Upsert into a dataset which has a correlated b-tree secondary index
+ * Expected Res : Success
+ * Date : June 8 2017
+ */
+
+use test;
+
+create index ageindex on UpsertTo(age);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-btree/primary-secondary-btree.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-btree/primary-secondary-btree.4.query.sqlpp
new file mode 100644
index 0000000..64c8acf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-btree/primary-secondary-btree.4.query.sqlpp
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Upsert into a dataset which has a correlated b-tree secondary index
+ * Expected Res : Success
+ * Date : June 8 2017
+ */
+
+
+ // So far this one doesn't use the btree index, need another query
+use test;
+
+select value x
+from UpsertTo x
+where x.age > 5;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.10.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.10.adm
new file mode 100644
index 0000000..04080b7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.10.adm
@@ -0,0 +1,3 @@
+{ "res": 4 }
+{ "res": 5 }
+{ "res": 6 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.11.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.11.adm
new file mode 100644
index 0000000..429a13d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.11.adm
@@ -0,0 +1,2 @@
+{ "res": 4 }
+{ "res": 5 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.4.adm
new file mode 100644
index 0000000..f36e389
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.4.adm
@@ -0,0 +1,2 @@
+{ "res": 3 }
+{ "res": 7 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.5.adm
new file mode 100644
index 0000000..9897674
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.5.adm
@@ -0,0 +1,2 @@
+{ "res": 1 }
+{ "res": 3 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.6.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.6.adm
new file mode 100644
index 0000000..c1c06bd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.6.adm
@@ -0,0 +1,2 @@
+{ "res": 5 }
+{ "res": 7 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.7.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.7.adm
new file mode 100644
index 0000000..c1c06bd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.7.adm
@@ -0,0 +1,2 @@
+{ "res": 5 }
+{ "res": 7 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.8.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.8.adm
new file mode 100644
index 0000000..e6e7ad6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.8.adm
@@ -0,0 +1,5 @@
+{ "res": 2 }
+{ "res": 3 }
+{ "res": 4 }
+{ "res": 5 }
+{ "res": 6 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.9.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.9.adm
new file mode 100644
index 0000000..db56195
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/open-index-non-enforced/correlated-index-selection/btree-index-01/btree-index-01.9.adm
@@ -0,0 +1,4 @@
+{ "res": 3 }
+{ "res": 4 }
+{ "res": 5 }
+{ "res": 6 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
index aad1832..0b31f17 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -337,6 +337,11 @@
<output-dir compare="Text">primary-secondary-btree</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="primary-correlated-secondary-btree">
+ <output-dir compare="Text">primary-secondary-btree</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="upsert">
<compilation-unit name="primary-secondary-inverted">
<output-dir compare="Text">primary-secondary-inverted</output-dir>
@@ -1696,6 +1701,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit name="using-correlated-prefix-merge-policy-with-feed">
+ <output-dir compare="Text">using-correlated-prefix-merge-policy</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="using-no-merge-policy">
<output-dir compare="Text">using-no-merge-policy</output-dir>
</compilation-unit>
@@ -1989,6 +1999,12 @@
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit
+ name="scan-delete-btree-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="scan-delete-rtree-secondary-index-nullable">
<output-dir compare="Text">scan-delete-rtree-secondary-index-nullable</output-dir>
</compilation-unit>
@@ -2004,6 +2020,12 @@
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit
+ name="scan-insert-btree-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="scan-insert-rtree-secondary-index-nullable">
<output-dir compare="Text">scan-insert-rtree-secondary-index-nullable</output-dir>
</compilation-unit>
@@ -2083,6 +2105,12 @@
<output-dir compare="Text">scan-delete-btree-secondary-index-open</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="dml">
+ <compilation-unit
+ name="scan-delete-btree-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="dml">
<compilation-unit name="scan-delete-inverted-index-ngram-secondary-index-open">
<output-dir compare="Text">scan-delete-inverted-index-ngram-secondary-index-open</output-dir>
@@ -2103,6 +2131,11 @@
<output-dir compare="Text">scan-insert-btree-secondary-index-open</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-btree-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="dml">
<compilation-unit name="scan-insert-inverted-index-ngram-secondary-index-open">
<output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index-open</output-dir>
@@ -7215,6 +7248,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="filters">
+ <compilation-unit name="insert-with-secondary-correlated-btree">
+ <output-dir compare="Text">insert-with-secondary-btree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
<compilation-unit name="insert-with-secondary-inverted-ngram">
<output-dir compare="Text">insert-with-secondary-inverted-ngram</output-dir>
</compilation-unit>
@@ -7395,6 +7433,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="temp-dataset">
+ <compilation-unit name="insert-and-scan-dataset-with-correlated-index">
+ <output-dir compare="Text">insert-and-scan-dataset-with-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="temp-dataset">
<compilation-unit name="temp_primary_plus_ngram_flush">
<output-dir compare="Text">temp_primary_plus_ngram_flush</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 9903fd0..33cf582 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -1590,6 +1590,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit name="using-correlated-prefix-merge-policy-with-feed">
+ <output-dir compare="Text">using-correlated-prefix-merge-policy</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="using-no-merge-policy">
<output-dir compare="Text">using-no-merge-policy</output-dir>
</compilation-unit>
@@ -1883,6 +1888,12 @@
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit
+ name="scan-delete-btree-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="scan-delete-rtree-secondary-index-nullable">
<output-dir compare="Text">scan-delete-rtree-secondary-index-nullable</output-dir>
</compilation-unit>
@@ -1972,6 +1983,11 @@
<output-dir compare="Text">scan-delete-btree-secondary-index-open</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-btree-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="dml">
<compilation-unit name="scan-delete-inverted-index-ngram-secondary-index-open">
<output-dir compare="Text">scan-delete-inverted-index-ngram-secondary-index-open</output-dir>
@@ -1993,6 +2009,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit name="scan-insert-btree-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="scan-insert-inverted-index-ngram-secondary-index-open">
<output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index-open</output-dir>
</compilation-unit>
@@ -3546,6 +3567,11 @@
<output-dir compare="Text">btree-index-01</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="open-index-non-enforced/correlated-index-selection">
+ <compilation-unit name="btree-index-01">
+ <output-dir compare="Text">btree-index-01</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="open-index-non-enforced/index-join">
<test-case FilePath="open-index-non-enforced/index-join">
@@ -4063,6 +4089,11 @@
<output-dir compare="Text">scan-insert-btree-secondary-index-nullable</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-btree-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="nested-index-dml">
<compilation-unit name="scan-insert-rtree-secondary-index-nullable">
<output-dir compare="Text">scan-insert-rtree-secondary-index-nullable</output-dir>
@@ -8505,6 +8536,11 @@
<output-dir compare="Text">insert-with-secondary-btree</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="insert-with-secondary-correlated-btree">
+ <output-dir compare="Text">insert-with-secondary-btree</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="filters">
<compilation-unit name="insert-with-secondary-inverted-ngram">
<output-dir compare="Text">insert-with-secondary-inverted-ngram</output-dir>
@@ -8670,6 +8706,11 @@
<output-dir compare="Text">insert-and-scan-dataset-with-index</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="temp-dataset">
+ <compilation-unit name="insert-and-scan-dataset-with-correlated-index">
+ <output-dir compare="Text">insert-and-scan-dataset-with-index</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="unnest">
<compilation-unit name="left-outer-unnest">
<output-dir compare="Text">left-outer-unnest</output-dir>
@@ -8788,6 +8829,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="upsert">
+ <compilation-unit name="primary-correlated-secondary-btree">
+ <output-dir compare="Text">primary-secondary-btree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
<compilation-unit name="primary-secondary-inverted">
<output-dir compare="Text">primary-secondary-inverted</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.1.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.1.script.aql
new file mode 100644
index 0000000..7d441cd
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.1.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+create_and_start.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.10.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.10.script.aql
new file mode 100644
index 0000000..40df6fb
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.10.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+stop_and_delete.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.2.ddl.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.2.ddl.aql
new file mode 100644
index 0000000..618c6b1
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.2.ddl.aql
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : primary_plus_default_correlated_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+drop dataverse recovery if exists;
+create dataverse recovery;
+use dataverse recovery;
+
+/* For raw Fragile data */
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float,
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+};
+
+/* For cleaned Fragile data */
+create type FragileType as closed {
+ row_id: int32,
+ sid: int32,
+ date: date,
+ day: int32,
+ time: time,
+ bpm: int32,
+ RR: float,
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+};
+
+/* Create dataset for loading raw Fragile data */
+create dataset Fragile_raw (FragileTypeRaw)
+primary key row_id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+/* Create dataset for cleaned Fragile data */
+create dataset Fragile (FragileType)
+primary key row_id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.3.update.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.3.update.aql
new file mode 100644
index 0000000..6a18e6e
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.3.update.aql
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : primary_plus_default_correlated_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+use dataverse recovery;
+
+load dataset Fragile_raw using localfs
+(("path"="asterix_nc1://../../../../../../asterix-app/data/csv/fragile_02.adm"),("format"="adm")) pre-sorted;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.4.txneu.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.4.txneu.aql
new file mode 100644
index 0000000..aa5d6fb
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.4.txneu.aql
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : primary_plus_default_correlated_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+use dataverse recovery;
+
+/* Load Fragile data from raw dataset into cleaned dataset */
+insert into dataset Fragile (
+ for $t in dataset Fragile_raw
+ return {
+ "row_id": $t.row_id % 28000,
+ "sid": $t.sid,
+ "date": date($t.date),
+ "day": $t.day,
+ "time": parse-time($t.time, "h:m:s"),
+ "bpm": $t.bpm,
+ "RR": $t.RR,
+ "text": $t.text,
+ "location": $t.location,
+ "text2": $t.text2
+ }
+);
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.5.ddl.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.5.ddl.aql
new file mode 100644
index 0000000..1af5b9f
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.5.ddl.aql
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : primary_plus_default_correlated_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+use dataverse recovery;
+
+/* Create default secondary index on dataset clean Fragile */
+create index cfSidIdx on Fragile(sid);
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.6.txnqbc.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.6.txnqbc.aql
new file mode 100644
index 0000000..bbfd14c
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.6.txnqbc.aql
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : primary_plus_default_correlated_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile where $x.sid=1 return $x);
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.7.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.7.script.aql
new file mode 100644
index 0000000..4583455
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.7.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.8.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.8.script.aql
new file mode 100644
index 0000000..7087cd3
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.8.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+stop_and_start.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.9.txnqar.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.9.txnqar.aql
new file mode 100644
index 0000000..bbfd14c
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_default_correlated_secondary_index/primary_plus_default_correlated_secondary_index.9.txnqar.aql
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : primary_plus_default_correlated_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : June 8 2017
+ */
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile where $x.sid=1 return $x);
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_default_correlated_secondary_index/create_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_default_correlated_secondary_index/create_and_start.sh
new file mode 100755
index 0000000..789914b
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_default_correlated_secondary_index/create_and_start.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix create -n asterix -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_default_correlated_secondary_index/kill_cc_and_nc.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_default_correlated_secondary_index/kill_cc_and_nc.sh
new file mode 100755
index 0000000..4b876be
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_default_correlated_secondary_index/kill_cc_and_nc.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+ps -ef | awk '/java.*org\.apache\.hyracks\.control\.[cn]c\.[CN]CDriver/ {print $2}' | xargs -n 1 kill -9
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_default_correlated_secondary_index/stop_and_delete.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_default_correlated_secondary_index/stop_and_delete.sh
new file mode 100755
index 0000000..eb1c01e
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_default_correlated_secondary_index/stop_and_delete.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix delete -n asterix;
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_default_correlated_secondary_index/stop_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_default_correlated_secondary_index/stop_and_start.sh
new file mode 100755
index 0000000..2fb80ce
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_default_correlated_secondary_index/stop_and_start.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix start -n asterix;
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
index 07e2d6d..f039708 100644
--- a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
@@ -56,6 +56,12 @@
</test-case>
<test-case FilePath="recover_after_abort">
+ <compilation-unit name="primary_plus_default_correlated_secondary_index">
+ <output-dir compare="Text">primary_plus_default_secondary_index</output-dir>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="recover_after_abort">
<compilation-unit name="primary_plus_rtree_index">
<output-dir compare="Text">primary_plus_rtree_index</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index 3aed427..9131692 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -30,6 +30,7 @@
import org.apache.asterix.active.ActiveLifecycleListener;
import org.apache.asterix.active.IActiveEntityEventsListener;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
+import org.apache.asterix.common.context.CorrelatedPrefixMergePolicyFactory;
import org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.common.dataflow.NoOpFrameOperationCallbackFactory;
import org.apache.asterix.common.exceptions.CompilationException;
@@ -657,6 +658,10 @@
return getDatasetDetails().isTemp();
}
+ public boolean isCorrelated() {
+ return CorrelatedPrefixMergePolicyFactory.NAME.equals(compactionPolicyFactory);
+ }
+
@Override
public List<List<String>> getPrimaryKeys() {
if (getDatasetType() == DatasetType.EXTERNAL) {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
index f2a0709..9e55a97 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
@@ -101,42 +101,43 @@
public static JobSpecification buildDropIndexJobSpec(Index index, MetadataProvider metadataProvider,
Dataset dataset) throws AlgebricksException {
- SecondaryIndexOperationsHelper secondaryIndexHelper =
- SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider,
- physicalOptimizationConfig);
+ SecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper
+ .createIndexOperationsHelper(dataset, index, metadataProvider, physicalOptimizationConfig);
return secondaryIndexHelper.buildDropJobSpec();
}
public static JobSpecification buildSecondaryIndexCreationJobSpec(Dataset dataset, Index index,
MetadataProvider metadataProvider) throws AlgebricksException {
- SecondaryIndexOperationsHelper secondaryIndexHelper =
- SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider,
- physicalOptimizationConfig);
+ SecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper
+ .createIndexOperationsHelper(dataset, index, metadataProvider, physicalOptimizationConfig);
return secondaryIndexHelper.buildCreationJobSpec();
}
public static JobSpecification buildSecondaryIndexLoadingJobSpec(Dataset dataset, Index index,
MetadataProvider metadataProvider) throws AlgebricksException {
- SecondaryIndexOperationsHelper secondaryIndexHelper =
- SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider,
- physicalOptimizationConfig);
- return secondaryIndexHelper.buildLoadingJobSpec();
+ return buildSecondaryIndexLoadingJobSpec(dataset, index, metadataProvider, null);
}
public static JobSpecification buildSecondaryIndexLoadingJobSpec(Dataset dataset, Index index,
MetadataProvider metadataProvider, List<ExternalFile> files) throws AlgebricksException {
- SecondaryIndexOperationsHelper secondaryIndexHelper =
- SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider,
- physicalOptimizationConfig);
- secondaryIndexHelper.setExternalFiles(files);
+ SecondaryIndexOperationsHelper secondaryIndexHelper;
+ if (dataset.isCorrelated()) {
+ secondaryIndexHelper = SecondaryCorrelatedTreeIndexOperationsHelper.createIndexOperationsHelper(dataset,
+ index, metadataProvider, physicalOptimizationConfig);
+ } else {
+ secondaryIndexHelper = SecondaryTreeIndexOperationsHelper.createIndexOperationsHelper(dataset, index,
+ metadataProvider, physicalOptimizationConfig);
+ }
+ if (files != null) {
+ secondaryIndexHelper.setExternalFiles(files);
+ }
return secondaryIndexHelper.buildLoadingJobSpec();
}
public static JobSpecification buildSecondaryIndexCompactJobSpec(Dataset dataset, Index index,
MetadataProvider metadataProvider) throws AlgebricksException {
- SecondaryIndexOperationsHelper secondaryIndexHelper =
- SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider,
- physicalOptimizationConfig);
+ SecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper
+ .createIndexOperationsHelper(dataset, index, metadataProvider, physicalOptimizationConfig);
return secondaryIndexHelper.buildCompactJobSpec();
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
new file mode 100644
index 0000000..7daa150
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.utils;
+
+import java.util.List;
+
+import org.apache.asterix.common.config.DatasetConfig.DatasetType;
+import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.runtime.operators.LSMSecondaryIndexBulkLoadOperatorDescriptor;
+import org.apache.asterix.runtime.utils.RuntimeUtils;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.core.jobgen.impl.ConnectorPolicyAssignmentPolicy;
+import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+import org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import org.apache.hyracks.algebricks.data.ISerializerDeserializerProvider;
+import org.apache.hyracks.algebricks.data.ITypeTraitProvider;
+import org.apache.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.algebricks.runtime.operators.base.SinkRuntimeFactory;
+import org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
+import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
+import org.apache.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
+
+/**
+ * This class is used to build secondary LSMBTree index for correlated datasets.
+ *
+ * @author luochen
+ *
+ */
+public class SecondaryCorrelatedBTreeOperationsHelper extends SecondaryCorrelatedTreeIndexOperationsHelper {
+
+ protected SecondaryCorrelatedBTreeOperationsHelper(Dataset dataset, Index index,
+ PhysicalOptimizationConfig physOptConf, MetadataProvider metadataProvider) throws AlgebricksException {
+ super(dataset, index, physOptConf, metadataProvider);
+ }
+
+ @Override
+ public JobSpecification buildLoadingJobSpec() throws AlgebricksException {
+ JobSpecification spec = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext());
+
+ boolean isOverridingKeyFieldTypes = index.isOverridingKeyFieldTypes();
+
+ assert dataset.getDatasetType() == DatasetType.INTERNAL;
+
+ // only handle internal datasets
+ // Create dummy key provider for feeding the primary index scan.
+ JobId jobId = IndexUtil.bindJobEventListener(spec, metadataProvider);
+
+ // Create dummy key provider for feeding the primary index scan.
+ IOperatorDescriptor keyProviderOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset, metadataProvider);
+
+ // Create primary index scan op.
+ IOperatorDescriptor primaryScanOp = createPrimaryIndexScanDiskComponentsOp(spec, metadataProvider,
+ getTaggedRecordDescriptor(dataset.getPrimaryRecordDescriptor(metadataProvider)), jobId);
+
+ // Assign op.
+ IOperatorDescriptor sourceOp = primaryScanOp;
+ if (isOverridingKeyFieldTypes && !enforcedItemType.equals(itemType)) {
+ sourceOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced());
+ spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0);
+ }
+ RecordDescriptor taggedSecondaryRecDesc = getTaggedRecordDescriptor(secondaryRecDesc);
+ AlgebricksMetaOperatorDescriptor asterixAssignOp =
+ createAssignOp(spec, index.getKeyFieldNames().size(), taggedSecondaryRecDesc);
+
+ // Generate compensate tuples for upsert
+ IOperatorDescriptor processorOp =
+ createTupleProcessorOp(spec, taggedSecondaryRecDesc, getNumSecondaryKeys(), numPrimaryKeys, false);
+
+ ExternalSortOperatorDescriptor sortOp = createSortOp(spec,
+ getTaggedSecondaryComparatorFactories(secondaryComparatorFactories), taggedSecondaryRecDesc);
+
+ // Create secondary BTree bulk load op.
+ LSMSecondaryIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec,
+ metadataProvider, taggedSecondaryRecDesc, getNumSecondaryKeys(), numPrimaryKeys, false);
+
+ AlgebricksMetaOperatorDescriptor metaOp =
+ new AlgebricksMetaOperatorDescriptor(spec, 1, 0, new IPushRuntimeFactory[] { new SinkRuntimeFactory() },
+ new RecordDescriptor[] { taggedSecondaryRecDesc });
+ // Connect the operators.
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, asterixAssignOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, processorOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), processorOp, 0, sortOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), secondaryBulkLoadOp, 0, metaOp, 0);
+ spec.addRoot(metaOp);
+ spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+ return spec;
+ }
+
+ @Override
+ protected int getNumSecondaryKeys() {
+ return index.getKeyFieldNames().size();
+ }
+
+ @Override
+ @SuppressWarnings("rawtypes")
+ protected void setSecondaryRecDescAndComparators() throws AlgebricksException {
+ int numSecondaryKeys = index.getKeyFieldNames().size();
+ secondaryFieldAccessEvalFactories = new IScalarEvaluatorFactory[numSecondaryKeys + numFilterFields];
+ secondaryComparatorFactories = new IBinaryComparatorFactory[numSecondaryKeys + numPrimaryKeys];
+ secondaryBloomFilterKeyFields = new int[numSecondaryKeys];
+ ISerializerDeserializer[] secondaryRecFields =
+ new ISerializerDeserializer[numPrimaryKeys + numSecondaryKeys + numFilterFields];
+ ISerializerDeserializer[] enforcedRecFields =
+ new ISerializerDeserializer[1 + numPrimaryKeys + (dataset.hasMetaPart() ? 1 : 0) + numFilterFields];
+ ITypeTraits[] enforcedTypeTraits =
+ new ITypeTraits[1 + numPrimaryKeys + (dataset.hasMetaPart() ? 1 : 0) + numFilterFields];
+ secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys];
+ ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider();
+ ITypeTraitProvider typeTraitProvider = metadataProvider.getFormat().getTypeTraitProvider();
+ IBinaryComparatorFactoryProvider comparatorFactoryProvider =
+ metadataProvider.getFormat().getBinaryComparatorFactoryProvider();
+ // Record column is 0 for external datasets, numPrimaryKeys for internal ones
+ int recordColumn = NUM_TAG_FIELDS + numPrimaryKeys;
+ boolean isOverridingKeyTypes = index.isOverridingKeyFieldTypes();
+ for (int i = 0; i < numSecondaryKeys; i++) {
+ ARecordType sourceType;
+ int sourceColumn;
+ List<Integer> keySourceIndicators = index.getKeyFieldSourceIndicators();
+ if (keySourceIndicators == null || keySourceIndicators.get(i) == 0) {
+ sourceType = itemType;
+ sourceColumn = recordColumn;
+ } else {
+ sourceType = metaType;
+ sourceColumn = recordColumn + 1;
+ }
+ secondaryFieldAccessEvalFactories[i] = metadataProvider.getFormat().getFieldAccessEvaluatorFactory(
+ isOverridingKeyTypes ? enforcedItemType : sourceType, index.getKeyFieldNames().get(i),
+ sourceColumn);
+ Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(i),
+ index.getKeyFieldNames().get(i), sourceType);
+ IAType keyType = keyTypePair.first;
+ anySecondaryKeyIsNullable = anySecondaryKeyIsNullable || keyTypePair.second;
+ ISerializerDeserializer keySerde = serdeProvider.getSerializerDeserializer(keyType);
+ secondaryRecFields[i] = keySerde;
+ secondaryComparatorFactories[i] = comparatorFactoryProvider.getBinaryComparatorFactory(keyType, true);
+ secondaryTypeTraits[i] = typeTraitProvider.getTypeTrait(keyType);
+ secondaryBloomFilterKeyFields[i] = i;
+ }
+ // Add serializers and comparators for primary index fields.
+ for (int i = 0; i < numPrimaryKeys; i++) {
+ secondaryRecFields[numSecondaryKeys + i] = primaryRecDesc.getFields()[i];
+ enforcedRecFields[i] = primaryRecDesc.getFields()[i];
+ secondaryTypeTraits[numSecondaryKeys + i] = primaryRecDesc.getTypeTraits()[i];
+ enforcedTypeTraits[i] = primaryRecDesc.getTypeTraits()[i];
+ secondaryComparatorFactories[numSecondaryKeys + i] = primaryComparatorFactories[i];
+ }
+
+ enforcedRecFields[numPrimaryKeys] = serdeProvider.getSerializerDeserializer(itemType);
+ enforcedTypeTraits[numPrimaryKeys] = typeTraitProvider.getTypeTrait(itemType);
+ if (dataset.hasMetaPart()) {
+ enforcedRecFields[numPrimaryKeys + 1] = serdeProvider.getSerializerDeserializer(metaType);
+ enforcedTypeTraits[numPrimaryKeys + 1] = typeTraitProvider.getTypeTrait(metaType);
+ }
+
+ if (numFilterFields > 0) {
+ secondaryFieldAccessEvalFactories[numSecondaryKeys] = metadataProvider.getFormat()
+ .getFieldAccessEvaluatorFactory(itemType, filterFieldName, recordColumn);
+ Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
+ IAType type = keyTypePair.first;
+ ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);
+ secondaryRecFields[numPrimaryKeys + numSecondaryKeys] = serde;
+ enforcedRecFields[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)] = serde;
+ enforcedTypeTraits[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)] =
+ typeTraitProvider.getTypeTrait(type);
+ }
+ secondaryRecDesc = new RecordDescriptor(secondaryRecFields, secondaryTypeTraits);
+ enforcedRecDesc = new RecordDescriptor(enforcedRecFields, enforcedTypeTraits);
+
+ }
+}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java
new file mode 100644
index 0000000..775efbe
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java
@@ -0,0 +1,324 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.utils;
+
+import org.apache.asterix.common.config.DatasetConfig.DatasetType;
+import org.apache.asterix.common.context.ITransactionSubsystemProvider;
+import org.apache.asterix.common.context.TransactionSubsystemProvider;
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.transactions.IRecoveryManager;
+import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.dataflow.data.nontagged.MissingWriterFactory;
+import org.apache.asterix.formats.nontagged.BinaryComparatorFactoryProvider;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.FunctionManagerHolder;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.runtime.operators.LSMSecondaryIndexBulkLoadOperatorDescriptor;
+import org.apache.asterix.runtime.operators.LSMSecondaryIndexCreationTupleProcessorOperatorDescriptor;
+import org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexInstantSearchOperationCallbackFactory;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+import org.apache.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.algebricks.runtime.evaluators.ColumnAccessEvalFactory;
+import org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
+import org.apache.hyracks.algebricks.runtime.operators.std.AssignRuntimeFactory;
+import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.data.std.primitive.BooleanPointable;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.dataflow.common.data.marshalling.BooleanSerializerDeserializer;
+import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import org.apache.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory;
+import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeDiskComponentScanOperatorDescriptor;
+
+/**
+ * This class is used to build secondary LSM index for correlated datasets.
+ *
+ * @author luochen
+ *
+ */
+public abstract class SecondaryCorrelatedTreeIndexOperationsHelper extends SecondaryTreeIndexOperationsHelper {
+
+ protected final static int COMPONENT_POS_OFFSET = 0;
+
+ protected final static int ANTI_MATTER_OFFSET = 1;
+
+ protected final static int NUM_TAG_FIELDS = 2;
+
+ /**
+ * Make sure tuples are in the descending order w.r.t. component_pos.
+ * In the disk component list of an index, components are ordered from newest to oldest.
+ * This descending order ensures older components can be bulk loaded first and get a smaller (older)
+ * component file timestamp.
+ */
+ protected final static IBinaryComparatorFactory COMPONENT_POS_COMPARATOR_FACTORY = new IBinaryComparatorFactory() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IBinaryComparator createBinaryComparator() {
+ return new IBinaryComparator() {
+ final IBinaryComparator comparator =
+ BinaryComparatorFactoryProvider.INTEGER_POINTABLE_INSTANCE.createBinaryComparator();
+
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) throws HyracksDataException {
+ return -comparator.compare(b1, s1, l1, b2, s2, l2);
+ }
+ };
+ }
+ };
+
+ protected SecondaryCorrelatedTreeIndexOperationsHelper(Dataset dataset, Index index,
+ PhysicalOptimizationConfig physOptConf, MetadataProvider metadataProvider) throws AlgebricksException {
+ super(dataset, index, physOptConf, metadataProvider);
+ }
+
+ protected RecordDescriptor getTaggedRecordDescriptor(RecordDescriptor recDescriptor) {
+ @SuppressWarnings("rawtypes")
+ ISerializerDeserializer[] fields =
+ new ISerializerDeserializer[recDescriptor.getFields().length + NUM_TAG_FIELDS];
+ ITypeTraits[] traits = null;
+ if (recDescriptor.getTypeTraits() != null) {
+ traits = new ITypeTraits[recDescriptor.getTypeTraits().length + NUM_TAG_FIELDS];
+ }
+
+ //component position field
+ fields[COMPONENT_POS_OFFSET] = IntegerSerializerDeserializer.INSTANCE;
+ if (traits != null) {
+ traits[COMPONENT_POS_OFFSET] = IntegerPointable.TYPE_TRAITS;
+ }
+
+ //anti-matter field
+ fields[ANTI_MATTER_OFFSET] = BooleanSerializerDeserializer.INSTANCE;
+ if (traits != null) {
+ traits[ANTI_MATTER_OFFSET] = BooleanPointable.TYPE_TRAITS;
+ }
+
+ for (int i = NUM_TAG_FIELDS; i < fields.length; i++) {
+ fields[i] = recDescriptor.getFields()[i - NUM_TAG_FIELDS];
+ if (traits != null && i < traits.length) {
+ traits[i] = recDescriptor.getTypeTraits()[i - NUM_TAG_FIELDS];
+ }
+ }
+
+ return new RecordDescriptor(fields, traits);
+ }
+
+ protected IBinaryComparatorFactory[] getTaggedSecondaryComparatorFactories(
+ IBinaryComparatorFactory[] secondaryComparatorFactories) {
+ IBinaryComparatorFactory[] resultFactories =
+ new IBinaryComparatorFactory[secondaryComparatorFactories.length + 1];
+
+ // order component ids from largest (oldest) to smallest (newest)
+ // this is necessary since during bulk-loading, we need to create older components first
+ resultFactories[COMPONENT_POS_OFFSET] = COMPONENT_POS_COMPARATOR_FACTORY;
+
+ for (int i = 1; i < resultFactories.length; i++) {
+ resultFactories[i] = secondaryComparatorFactories[i - 1];
+ }
+
+ return resultFactories;
+ }
+
+ @Override
+ protected AlgebricksMetaOperatorDescriptor createCastOp(JobSpecification spec, DatasetType dsType,
+ boolean strictCast) throws AlgebricksException {
+ IFunctionDescriptor castFuncDesc = FunctionManagerHolder.getFunctionManager()
+ .lookupFunction(strictCast ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX);
+ castFuncDesc.setImmutableStates(enforcedItemType, itemType);
+
+ int[] outColumns = new int[1];
+
+ // tags(2) + primary keys + record + meta part(?)
+ int[] projectionList = new int[NUM_TAG_FIELDS + (dataset.hasMetaPart() ? 2 : 1) + numPrimaryKeys];
+ int recordIdx = NUM_TAG_FIELDS + numPrimaryKeys;
+
+ //here we only consider internal dataset
+ assert dsType == DatasetType.INTERNAL;
+
+ outColumns[0] = NUM_TAG_FIELDS + numPrimaryKeys;
+
+ int projCount = 0;
+ for (int i = 0; i < NUM_TAG_FIELDS; i++) {
+ projectionList[projCount++] = i;
+ }
+
+ //set primary keys and the record
+ for (int i = 0; i <= numPrimaryKeys; i++) {
+ projectionList[projCount++] = NUM_TAG_FIELDS + i;
+ }
+ if (dataset.hasMetaPart()) {
+ projectionList[NUM_TAG_FIELDS + numPrimaryKeys + 1] = NUM_TAG_FIELDS + numPrimaryKeys + 1;
+ }
+ IScalarEvaluatorFactory[] castEvalFact =
+ new IScalarEvaluatorFactory[] { new ColumnAccessEvalFactory(recordIdx) };
+ IScalarEvaluatorFactory[] sefs = new IScalarEvaluatorFactory[1];
+ sefs[0] = castFuncDesc.createEvaluatorFactory(castEvalFact);
+ AssignRuntimeFactory castAssign = new AssignRuntimeFactory(outColumns, sefs, projectionList);
+ return new AlgebricksMetaOperatorDescriptor(spec, 1, 1, new IPushRuntimeFactory[] { castAssign },
+ new RecordDescriptor[] { getTaggedRecordDescriptor(enforcedRecDesc) });
+ }
+
+ /**
+ * It differs from its base class in that we need to add the extra tags to the fields.
+ */
+ @Override
+ protected AlgebricksMetaOperatorDescriptor createAssignOp(JobSpecification spec, int numSecondaryKeyFields,
+ RecordDescriptor secondaryRecDesc) throws AlgebricksException {
+ int[] outColumns = new int[numSecondaryKeyFields + numFilterFields];
+ int[] projectionList = new int[NUM_TAG_FIELDS + numSecondaryKeyFields + numPrimaryKeys + numFilterFields];
+ for (int i = 0; i < numSecondaryKeyFields + numFilterFields; i++) {
+ outColumns[i] = NUM_TAG_FIELDS + numPrimaryKeys + i;
+ }
+ int projCount = 0;
+
+ //set tag fields
+ for (int i = 0; i < NUM_TAG_FIELDS; i++) {
+ projectionList[projCount++] = i;
+ }
+
+ //set secondary keys
+ for (int i = 0; i < numSecondaryKeyFields; i++) {
+ projectionList[projCount++] = NUM_TAG_FIELDS + numPrimaryKeys + i;
+ }
+
+ //set primary keys
+ for (int i = 0; i < numPrimaryKeys; i++) {
+ projectionList[projCount++] = NUM_TAG_FIELDS + i;
+ }
+
+ //set filter fields
+ if (numFilterFields > 0) {
+ projectionList[projCount] = NUM_TAG_FIELDS + numPrimaryKeys + numSecondaryKeyFields;
+ }
+
+ IScalarEvaluatorFactory[] sefs = new IScalarEvaluatorFactory[secondaryFieldAccessEvalFactories.length];
+ for (int i = 0; i < secondaryFieldAccessEvalFactories.length; ++i) {
+ sefs[i] = secondaryFieldAccessEvalFactories[i];
+ }
+ AssignRuntimeFactory assign = new AssignRuntimeFactory(outColumns, sefs, projectionList);
+ AlgebricksMetaOperatorDescriptor asterixAssignOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 1,
+ new IPushRuntimeFactory[] { assign }, new RecordDescriptor[] { secondaryRecDesc });
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, asterixAssignOp,
+ primaryPartitionConstraint);
+ return asterixAssignOp;
+ }
+
+ protected IOperatorDescriptor createTupleProcessorOp(JobSpecification spec, RecordDescriptor taggedSecondaryRecDesc,
+ int numSecondaryKeyFields, int numPrimaryKeyFields, boolean hasBuddyBTree) {
+ IOperatorDescriptor op = new LSMSecondaryIndexCreationTupleProcessorOperatorDescriptor(spec,
+ taggedSecondaryRecDesc, MissingWriterFactory.INSTANCE, NUM_TAG_FIELDS, numSecondaryKeyFields,
+ numPrimaryKeyFields, hasBuddyBTree);
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, op, primaryPartitionConstraint);
+ return op;
+ }
+
+ @Override
+ protected ExternalSortOperatorDescriptor createSortOp(JobSpecification spec,
+ IBinaryComparatorFactory[] taggedSecondaryComparatorFactories, RecordDescriptor taggedSecondaryRecDesc) {
+ int[] taggedSortFields = new int[taggedSecondaryComparatorFactories.length];
+ //component pos
+ taggedSortFields[COMPONENT_POS_OFFSET] = COMPONENT_POS_OFFSET;
+ //not sorting on anti-matter field
+ for (int i = 1; i < taggedSortFields.length; i++) {
+ taggedSortFields[i] = i + 1;
+ }
+ ExternalSortOperatorDescriptor sortOp =
+ new ExternalSortOperatorDescriptor(spec, physOptConf.getMaxFramesExternalSort(), taggedSortFields,
+ taggedSecondaryComparatorFactories, taggedSecondaryRecDesc);
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, sortOp, primaryPartitionConstraint);
+ return sortOp;
+ }
+
+ protected LSMSecondaryIndexBulkLoadOperatorDescriptor createTreeIndexBulkLoadOp(JobSpecification spec,
+ MetadataProvider metadataProvider, RecordDescriptor taggedSecondaryRecDesc, int numSecondaryKeys,
+ int numPrimaryKeys, boolean hasBuddyBtree) throws AlgebricksException {
+ IndexDataflowHelperFactory primaryIndexHelperFactory = new IndexDataflowHelperFactory(
+ metadataProvider.getStorageComponentProvider().getStorageManager(), primaryFileSplitProvider);
+
+ IndexDataflowHelperFactory secondaryIndexHelperFactory = new IndexDataflowHelperFactory(
+ metadataProvider.getStorageComponentProvider().getStorageManager(), secondaryFileSplitProvider);
+
+ LSMSecondaryIndexBulkLoadOperatorDescriptor treeIndexBulkLoadOp =
+ new LSMSecondaryIndexBulkLoadOperatorDescriptor(spec, taggedSecondaryRecDesc, primaryIndexHelperFactory,
+ secondaryIndexHelperFactory, NUM_TAG_FIELDS, numSecondaryKeys, numPrimaryKeys, hasBuddyBtree);
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, treeIndexBulkLoadOp,
+ secondaryPartitionConstraint);
+ return treeIndexBulkLoadOp;
+ }
+
+ protected IOperatorDescriptor createPrimaryIndexScanDiskComponentsOp(JobSpecification spec,
+ MetadataProvider metadataProvider, RecordDescriptor outRecDesc, JobId jobId) throws AlgebricksException {
+ ITransactionSubsystemProvider txnSubsystemProvider = TransactionSubsystemProvider.INSTANCE;
+ boolean temp = dataset.getDatasetDetails().isTemp();
+ ISearchOperationCallbackFactory searchCallbackFactory = temp ? NoOpOperationCallbackFactory.INSTANCE
+ : new PrimaryIndexInstantSearchOperationCallbackFactory(jobId, dataset.getDatasetId(),
+ dataset.getPrimaryBloomFilterFields(), txnSubsystemProvider,
+ IRecoveryManager.ResourceType.LSM_BTREE);
+ IndexDataflowHelperFactory indexHelperFactory = new IndexDataflowHelperFactory(
+ metadataProvider.getStorageComponentProvider().getStorageManager(), primaryFileSplitProvider);
+ LSMBTreeDiskComponentScanOperatorDescriptor primaryScanOp = new LSMBTreeDiskComponentScanOperatorDescriptor(
+ spec, outRecDesc, indexHelperFactory, searchCallbackFactory);
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, primaryScanOp,
+ primaryPartitionConstraint);
+ return primaryScanOp;
+ }
+
+ public static SecondaryIndexOperationsHelper createIndexOperationsHelper(Dataset dataset, Index index,
+ MetadataProvider metadataProvider, PhysicalOptimizationConfig physOptConf) throws AlgebricksException {
+
+ SecondaryIndexOperationsHelper indexOperationsHelper = null;
+ switch (index.getIndexType()) {
+ case BTREE:
+ indexOperationsHelper =
+ new SecondaryCorrelatedBTreeOperationsHelper(dataset, index, physOptConf, metadataProvider);
+ break;
+ case RTREE:
+ //TODO RTree
+ case SINGLE_PARTITION_WORD_INVIX:
+ case SINGLE_PARTITION_NGRAM_INVIX:
+ case LENGTH_PARTITIONED_WORD_INVIX:
+ case LENGTH_PARTITIONED_NGRAM_INVIX:
+ //TODO Inverted Index
+ //TODO This will be fixed soon
+ throw new UnsupportedOperationException();
+ default:
+ throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_INDEX_TYPE, index.getIndexType());
+ }
+ indexOperationsHelper.init();
+ return indexOperationsHelper;
+ }
+
+}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AbstractLSMSecondaryIndexCreationNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AbstractLSMSecondaryIndexCreationNodePushable.java
new file mode 100644
index 0000000..6b050af
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AbstractLSMSecondaryIndexCreationNodePushable.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.operators;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.BooleanPointable;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
+
+public abstract class AbstractLSMSecondaryIndexCreationNodePushable
+ extends AbstractUnaryInputUnaryOutputOperatorNodePushable {
+ protected final IHyracksTaskContext ctx;
+ protected final RecordDescriptor inputRecDesc;
+
+ // with tag fields
+ protected final FrameTupleReference tuple;
+
+ protected final int partition;
+ protected final int numTagFields;
+ protected final int numSecondaryKeys;
+ protected final int numPrimaryKeys;
+
+ protected final boolean hasBuddyBTree;
+
+ protected FrameTupleAccessor accessor;
+
+ public AbstractLSMSecondaryIndexCreationNodePushable(IHyracksTaskContext ctx, int partition,
+ RecordDescriptor inputRecDesc, int numTagFields, int numSecondaryKeys, int numPrimaryKeys,
+ boolean hasBuddyBTree) {
+ this.ctx = ctx;
+ this.inputRecDesc = inputRecDesc;
+ this.tuple = new FrameTupleReference();
+
+ this.partition = partition;
+ this.numTagFields = numTagFields;
+ this.numSecondaryKeys = numSecondaryKeys;
+ this.numPrimaryKeys = numPrimaryKeys;
+ this.hasBuddyBTree = hasBuddyBTree;
+
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ accessor = new FrameTupleAccessor(inputRecDesc);
+ try {
+ writer.open();
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ writer.fail();
+ }
+
+ protected int getComponentPos(ITupleReference tuple) {
+ return IntegerPointable.getInteger(tuple.getFieldData(0), tuple.getFieldStart(0));
+ }
+
+ protected boolean isAntiMatterTuple(ITupleReference tuple) {
+ return BooleanPointable.getBoolean(tuple.getFieldData(1), tuple.getFieldStart(1));
+ }
+
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadNodePushable.java
new file mode 100644
index 0000000..f09bcd2
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadNodePushable.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.operators;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.asterix.runtime.operators.LSMSecondaryIndexCreationTupleProcessorNodePushable.DeletedTupleCounter;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.LongPointable;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
+import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentId;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+
+/**
+ * This operator node is used to bulk load incoming tuples (scanned from the primary index)
+ * into multiple disk components of the secondary index.
+ * Incoming tuple format:
+ * [component pos, anti-matter flag, secondary keys, primary keys, filter values]
+ */
+public class LSMSecondaryIndexBulkLoadNodePushable extends AbstractLSMSecondaryIndexCreationNodePushable {
+ // with tag fields
+
+ private final PermutingTupleReference sourceTuple;
+ private final PermutingTupleReference deletedKeyTuple;
+
+ private final IIndexDataflowHelper primaryIndexHelper;
+ private final IIndexDataflowHelper secondaryIndexHelper;
+
+ private ILSMIndex primaryIndex;
+ private ILSMIndex secondaryIndex;
+
+ private ILSMDiskComponent component;
+ private ILSMDiskComponentBulkLoader componentBulkLoader;
+ private int currentComponentPos = -1;
+
+ private ILSMDiskComponent[] diskComponents;
+
+ public LSMSecondaryIndexBulkLoadNodePushable(IHyracksTaskContext ctx, int partition, RecordDescriptor inputRecDesc,
+ IIndexDataflowHelperFactory primaryIndexHelperFactory,
+ IIndexDataflowHelperFactory secondaryIndexHelperFactory, int numTagFields, int numSecondaryKeys,
+ int numPrimaryKeys, boolean hasBuddyBTree) throws HyracksDataException {
+ super(ctx, partition, inputRecDesc, numTagFields, numSecondaryKeys, numPrimaryKeys, hasBuddyBTree);
+
+ this.primaryIndexHelper =
+ primaryIndexHelperFactory.create(ctx.getJobletContext().getServiceContext(), partition);
+ this.secondaryIndexHelper =
+ secondaryIndexHelperFactory.create(ctx.getJobletContext().getServiceContext(), partition);
+
+ int[] sourcePermutation = new int[inputRecDesc.getFieldCount() - numTagFields];
+ for (int i = 0; i < sourcePermutation.length; i++) {
+ sourcePermutation[i] = i + numTagFields;
+ }
+ sourceTuple = new PermutingTupleReference(sourcePermutation);
+
+ int[] deletedKeyPermutation = new int[inputRecDesc.getFieldCount() - numTagFields - numSecondaryKeys];
+ for (int i = 0; i < deletedKeyPermutation.length; i++) {
+ deletedKeyPermutation[i] = i + numTagFields + numSecondaryKeys;
+ }
+ deletedKeyTuple = new PermutingTupleReference(deletedKeyPermutation);
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ super.open();
+ primaryIndexHelper.open();
+ primaryIndex = (ILSMIndex) primaryIndexHelper.getIndexInstance();
+ diskComponents = new ILSMDiskComponent[primaryIndex.getImmutableComponents().size()];
+ secondaryIndexHelper.open();
+ secondaryIndex = (ILSMIndex) secondaryIndexHelper.getIndexInstance();
+
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ HyracksDataException closeException = null;
+ try {
+ endCurrentComponent();
+ } catch (HyracksDataException e) {
+ closeException = e;
+ }
+
+ activateComponents();
+
+ try {
+ if (primaryIndexHelper != null) {
+ primaryIndexHelper.close();
+ }
+ } catch (HyracksDataException e) {
+ if (closeException == null) {
+ closeException = e;
+ } else {
+ closeException.addSuppressed(e);
+ }
+ }
+
+ try {
+ if (secondaryIndexHelper != null) {
+ secondaryIndexHelper.close();
+ }
+ } catch (HyracksDataException e) {
+ if (closeException == null) {
+ closeException = e;
+ } else {
+ closeException.addSuppressed(e);
+ }
+ }
+
+ try {
+ // will definitely be called regardless of exceptions
+ writer.close();
+ } catch (HyracksDataException th) {
+ if (closeException == null) {
+ closeException = th;
+ } else {
+ closeException.addSuppressed(th);
+ }
+ }
+
+ if (closeException != null) {
+ throw closeException;
+ }
+ }
+
+ @Override
+ public void flush() throws HyracksDataException {
+ writer.flush();
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ writer.fail();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ accessor.reset(buffer);
+ int tupleCount = accessor.getTupleCount();
+ for (int i = 0; i < tupleCount; i++) {
+ try {
+ // if both previous value and new value are null, then we skip
+ tuple.reset(accessor, i);
+ int componentPos = getComponentPos(tuple);
+ if (componentPos != currentComponentPos) {
+ loadNewComponent(componentPos);
+ currentComponentPos = componentPos;
+ }
+
+ if (isAntiMatterTuple(tuple)) {
+ addAntiMatterTuple(tuple);
+ } else {
+ addMatterTuple(tuple);
+ }
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+ }
+
+ private void endCurrentComponent() throws HyracksDataException {
+ if (component != null) {
+ // set disk component id
+
+ componentBulkLoader.end();
+ diskComponents[currentComponentPos] = component;
+
+ componentBulkLoader = null;
+ component = null;
+ }
+ }
+
+ private void loadNewComponent(int componentPos) throws HyracksDataException {
+ endCurrentComponent();
+
+ component = secondaryIndex.createBulkLoadTarget();
+ componentBulkLoader = secondaryIndex.createComponentBulkLoader(component, 1.0f, false,
+ getNumDeletedTuples(componentPos), false, true, true);
+
+ }
+
+ private void addAntiMatterTuple(ITupleReference tuple) throws HyracksDataException {
+ if (hasBuddyBTree) {
+ deletedKeyTuple.reset(tuple);
+ componentBulkLoader.delete(deletedKeyTuple);
+ } else {
+ sourceTuple.reset(tuple);
+ componentBulkLoader.delete(sourceTuple);
+ }
+ }
+
+ private void addMatterTuple(ITupleReference tuple) throws HyracksDataException {
+ sourceTuple.reset(tuple);
+ componentBulkLoader.add(sourceTuple);
+
+ }
+
+ private void activateComponents() throws HyracksDataException {
+ List<ILSMDiskComponent> primaryComponents = primaryIndex.getImmutableComponents();
+ for (int i = diskComponents.length - 1; i >= 0; i--) {
+ // start from the oldest component to the newest component
+ if (diskComponents[i] != null && diskComponents[i].getComponentSize() > 0) {
+ secondaryIndex.getIOOperationCallback().afterOperation(LSMOperationType.FLUSH, null, diskComponents[i]);
+
+ // setting component id has to be place between afterOperation and addBulkLoadedComponent,
+ // since afterOperation would set a flush component id (but it's not invalid)
+ // and addBulkLoadedComponent would finalize the component
+ ILSMDiskComponentId primaryComponentId = primaryComponents.get(i).getComponentId();
+ //set component id
+ diskComponents[i].getMetadata().put(ILSMDiskComponentId.COMPONENT_ID_MIN_KEY,
+ LongPointable.FACTORY.createPointable(primaryComponentId.getMinId()));
+ diskComponents[i].getMetadata().put(ILSMDiskComponentId.COMPONENT_ID_MAX_KEY,
+ LongPointable.FACTORY.createPointable(primaryComponentId.getMaxId()));
+
+ ((AbstractLSMIndex) secondaryIndex).getLsmHarness().addBulkLoadedComponent(diskComponents[i]);
+
+ }
+ }
+ }
+
+ private int getNumDeletedTuples(int componentPos) {
+ DeletedTupleCounter counter = (DeletedTupleCounter) ctx.getStateObject(partition);
+ return counter.get(componentPos);
+ }
+
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadOperatorDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadOperatorDescriptor.java
new file mode 100644
index 0000000..804b700
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadOperatorDescriptor.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.operators;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
+import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
+import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+
+public class LSMSecondaryIndexBulkLoadOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ private final IIndexDataflowHelperFactory primaryIndexHelperFactory;
+ private final IIndexDataflowHelperFactory secondaryIndexHelperFactory;
+
+ private final int numTagFields;
+ private final int numSecondaryKeys;
+ private final int numPrimaryKeys;
+
+ private final boolean hasBuddyBTree;
+
+ public LSMSecondaryIndexBulkLoadOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor outRecDesc,
+ IIndexDataflowHelperFactory primaryIndexHelperFactory,
+ IIndexDataflowHelperFactory secondaryIndexHelperFactory, int numTagFields, int numSecondaryKeys,
+ int numPrimaryKeys, boolean hasBuddyBTree) {
+ super(spec, 1, 1);
+ this.outRecDescs[0] = outRecDesc;
+ this.primaryIndexHelperFactory = primaryIndexHelperFactory;
+ this.secondaryIndexHelperFactory = secondaryIndexHelperFactory;
+ this.numTagFields = numTagFields;
+ this.numSecondaryKeys = numSecondaryKeys;
+ this.numPrimaryKeys = numPrimaryKeys;
+ this.hasBuddyBTree = hasBuddyBTree;
+
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
+ return new LSMSecondaryIndexBulkLoadNodePushable(ctx, partition,
+ recordDescProvider.getInputRecordDescriptor(getActivityId(), 0), primaryIndexHelperFactory,
+ secondaryIndexHelperFactory, numTagFields, numSecondaryKeys, numPrimaryKeys, hasBuddyBTree);
+ }
+}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexCreationTupleProcessorNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexCreationTupleProcessorNodePushable.java
new file mode 100644
index 0000000..b2a2fa4
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexCreationTupleProcessorNodePushable.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.operators;
+
+import java.io.DataOutput;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.TypeTagUtil;
+import org.apache.hyracks.api.comm.VSizeFrame;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.IMissingWriter;
+import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.dataflow.common.data.marshalling.BooleanSerializerDeserializer;
+import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import org.apache.hyracks.dataflow.common.utils.TupleUtils;
+import org.apache.hyracks.dataflow.std.base.AbstractStateObject;
+
+/**
+ * This operator node is used for post-processing tuples scanned from the primary index
+ * when creating a new secondary index.
+ * The incoming tuples could be either a matter-tuple or an anti-matter tuple.
+ * For an anti-matter tuple, it outputs a corresponding anti-matter tuple
+ * (if the secondary index has a buddy btree, then the result tuple only has primary key;
+ * otherwise, the result tuple has both secondary keys and primary key)
+ * For each matter-tuple (here old secondary keys refer to the secondary keys of the previous
+ * matter tuple with the same primary key)
+ * -If old secondary keys == new secondary keys
+ * --If with buddy btree
+ * ---generate a deleted-tuple
+ * --generate a new key
+ * -else
+ * --If old secondary keys are null?
+ * ---do nothing
+ * --else
+ * ---delete old secondary keys
+ * --If new keys are null?
+ * ---do nothing
+ * --else
+ * ---insert new keys
+ *
+ * Incoming tuple format:
+ * [component pos, anti-matter flag, secondary keys, primary keys, filter values]
+ */
+public class LSMSecondaryIndexCreationTupleProcessorNodePushable extends AbstractLSMSecondaryIndexCreationNodePushable {
+ // prevSourceTuple stores the previous matter tuple
+ private final ArrayTupleBuilder prevMatterTupleBuilder;
+ private final ArrayTupleReference prevMatterTuple = new ArrayTupleReference();
+
+ private boolean hasPrevMatterTuple;
+
+ private FrameTupleAppender appender;
+ private ArrayTupleBuilder tb;
+ private DataOutput dos;
+ private final IMissingWriter missingWriter;
+ private DeletedTupleCounter deletedTupleCounter;
+
+ public static class DeletedTupleCounter extends AbstractStateObject {
+ private final Map<Integer, Integer> map = new HashMap<>();
+
+ public DeletedTupleCounter(JobId jobId, int partition) {
+ super(jobId, partition);
+ }
+
+ public int get(int componentPos) {
+ Integer value = map.get(componentPos);
+ return value != null ? value : 0;
+ }
+
+ public void inc(int componentPos) {
+ map.put(componentPos, get(componentPos) + 1);
+ }
+ }
+
+ public LSMSecondaryIndexCreationTupleProcessorNodePushable(IHyracksTaskContext ctx, int partition,
+ RecordDescriptor inputRecDesc, IMissingWriterFactory missingWriterFactory, int numTagFields,
+ int numSecondaryKeys, int numPrimaryKeys, boolean hasBuddyBTree) throws HyracksDataException {
+
+ super(ctx, partition, inputRecDesc, numTagFields, numSecondaryKeys, numPrimaryKeys, hasBuddyBTree);
+
+ this.prevMatterTupleBuilder = new ArrayTupleBuilder(inputRecDesc.getFieldCount());
+
+ if (this.hasBuddyBTree) {
+ missingWriter = missingWriterFactory.createMissingWriter();
+ } else {
+ missingWriter = null;
+ }
+
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ super.open();
+ deletedTupleCounter = new DeletedTupleCounter(ctx.getJobletContext().getJobId(), partition);
+ ctx.setStateObject(deletedTupleCounter);
+ try {
+ tb = new ArrayTupleBuilder(recordDesc.getFieldCount());
+ dos = tb.getDataOutput();
+ appender = new FrameTupleAppender(new VSizeFrame(ctx), true);
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ HyracksDataException closeException = null;
+ try {
+ if (appender.getTupleCount() > 0) {
+ appender.write(writer, true);
+ }
+ } catch (HyracksDataException e) {
+ closeException = e;
+ }
+ try {
+ // will definitely be called regardless of exceptions
+ writer.close();
+ } catch (HyracksDataException th) {
+ if (closeException == null) {
+ closeException = th;
+ } else {
+ closeException.addSuppressed(th);
+ }
+ }
+ if (closeException != null) {
+ throw closeException;
+ }
+ }
+
+ @Override
+ public void flush() throws HyracksDataException {
+ appender.flush(writer);
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ writer.fail();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ accessor.reset(buffer);
+ int tupleCount = accessor.getTupleCount();
+ for (int i = 0; i < tupleCount; i++) {
+ try {
+ // if both previous value and new value are null, then we skip
+ tuple.reset(accessor, i);
+ if (isAntiMatterTuple(tuple)) {
+ processAntiMatterTuple(tuple);
+ hasPrevMatterTuple = false;
+ } else {
+ processMatterTuple(tuple);
+ // save the matter tuple
+ TupleUtils.copyTuple(prevMatterTupleBuilder, tuple, recordDesc.getFieldCount());
+ prevMatterTuple.reset(prevMatterTupleBuilder.getFieldEndOffsets(),
+ prevMatterTupleBuilder.getByteArray());
+ hasPrevMatterTuple = true;
+ }
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+ }
+
+ private void processMatterTuple(ITupleReference tuple) throws HyracksDataException {
+
+ boolean isNewValueMissing = isSecondaryKeyMissing(tuple);
+ boolean isOldValueMissing = !hasPrevMatterTuple || !equalPrimaryKeys(tuple, prevMatterTuple)
+ || isSecondaryKeyMissing(prevMatterTuple);
+
+ if (isNewValueMissing && isOldValueMissing) {
+ // if both values are missing, then do nothing
+ return;
+ }
+ // At least one is not null
+ if (!isOldValueMissing && equalSecondaryKeys(prevMatterTuple, tuple)) {
+ if (hasBuddyBTree) {
+ // if the index has buddy btree, then we have to delete the index entry
+ // from the older disk components
+ writeAntiMatterTuple(prevMatterTuple, getComponentPos(tuple));
+ }
+ // we need to write the new tuple anyway
+ writeMatterTuple(tuple);
+ return;
+ }
+ if (!isOldValueMissing) {
+ // we need to delete the previous entry
+ writeAntiMatterTuple(prevMatterTuple, getComponentPos(tuple));
+ }
+ if (!isNewValueMissing) {
+ // we need to insert the new entry
+ writeMatterTuple(tuple);
+ }
+
+ }
+
+ private void processAntiMatterTuple(ITupleReference tuple) throws HyracksDataException {
+ boolean isNewValueMissing = isSecondaryKeyMissing(tuple);
+ // if the secondary value is missing (which means the secondary value of the previous matter tuple
+ // is also missing), we then simply ignore this tuple since there is nothing to delete
+ if (!isNewValueMissing) {
+ writeAntiMatterTuple(tuple, getComponentPos(tuple));
+ }
+ }
+
+ private void writeAntiMatterTuple(ITupleReference tuple, int componentPos) throws HyracksDataException {
+ deletedTupleCounter.inc(componentPos);
+ tb.reset();
+ // write tag fields
+ tb.addField(IntegerSerializerDeserializer.INSTANCE, componentPos);
+ tb.addField(BooleanSerializerDeserializer.INSTANCE, true);
+ if (hasBuddyBTree) {
+ // the output tuple does not have secondary keys (only primary keys + filter values)
+ // write secondary keys (missing)
+ for (int i = 0; i < numSecondaryKeys; i++) {
+ missingWriter.writeMissing(dos);
+ tb.addFieldEndOffset();
+ }
+ } else {
+ for (int i = numTagFields; i < numTagFields + numSecondaryKeys; i++) {
+ tb.addField(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
+ }
+ }
+
+ // write all remaining fields
+ for (int i = numTagFields + numSecondaryKeys; i < recordDesc.getFieldCount(); i++) {
+ tb.addField(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
+ }
+
+ FrameUtils.appendToWriter(writer, appender, tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
+ }
+
+ private void writeMatterTuple(ITupleReference tuple) throws HyracksDataException {
+ // simply output the original tuple to the writer
+ TupleUtils.copyTuple(tb, tuple, recordDesc.getFieldCount());
+ FrameUtils.appendToWriter(writer, appender, tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
+ }
+
+ private boolean isSecondaryKeyMissing(ITupleReference tuple) {
+ for (int i = numTagFields; i < numTagFields + numSecondaryKeys; i++) {
+ if (TypeTagUtil.isType(tuple, i, ATypeTag.SERIALIZED_MISSING_TYPE_TAG)
+ || TypeTagUtil.isType(tuple, i, ATypeTag.SERIALIZED_NULL_TYPE_TAG)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean equalPrimaryKeys(ITupleReference tuple1, ITupleReference tuple2) {
+ for (int i = numTagFields + numSecondaryKeys; i < numTagFields + numPrimaryKeys + numSecondaryKeys; i++) {
+ if (!equalField(tuple1, tuple2, i)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private boolean equalSecondaryKeys(ITupleReference tuple1, ITupleReference tuple2) {
+ for (int i = numTagFields; i < numTagFields + numSecondaryKeys; i++) {
+ if (!equalField(tuple1, tuple2, i)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private boolean equalField(ITupleReference tuple1, ITupleReference tuple2, int fIdx) {
+ return LSMSecondaryUpsertOperatorNodePushable.equals(tuple1.getFieldData(fIdx), tuple1.getFieldStart(fIdx),
+ tuple1.getFieldLength(fIdx), tuple2.getFieldData(fIdx), tuple2.getFieldStart(fIdx),
+ tuple2.getFieldLength(fIdx));
+ }
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexCreationTupleProcessorOperatorDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexCreationTupleProcessorOperatorDescriptor.java
new file mode 100644
index 0000000..68e7603
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexCreationTupleProcessorOperatorDescriptor.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.operators;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
+import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory;
+import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
+import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+
+public class LSMSecondaryIndexCreationTupleProcessorOperatorDescriptor
+ extends AbstractSingleActivityOperatorDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ private final IMissingWriterFactory missingWriterFactory;
+
+ private final int numTagFields;
+ private final int numSecondaryKeys;
+ private final int numPrimaryKeys;
+
+ private final boolean hasBuddyBTree;
+
+ public LSMSecondaryIndexCreationTupleProcessorOperatorDescriptor(IOperatorDescriptorRegistry spec,
+ RecordDescriptor outRecDesc, IMissingWriterFactory missingWriterFactory, int numTagFields,
+ int numSecondaryKeys, int numPrimaryKeys, boolean hasBuddyBTree) {
+ super(spec, 1, 1);
+ this.outRecDescs[0] = outRecDesc;
+ this.missingWriterFactory = missingWriterFactory;
+ this.numTagFields = numTagFields;
+ this.numSecondaryKeys = numSecondaryKeys;
+ this.numPrimaryKeys = numPrimaryKeys;
+ this.hasBuddyBTree = hasBuddyBTree;
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
+ return new LSMSecondaryIndexCreationTupleProcessorNodePushable(ctx, partition,
+ recordDescProvider.getInputRecordDescriptor(getActivityId(), 0), missingWriterFactory, numTagFields,
+ numSecondaryKeys, numPrimaryKeys, hasBuddyBTree);
+ }
+}
diff --git a/asterixdb/asterix-runtime/src/test/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexCreationTupleProcessorTest.java b/asterixdb/asterix-runtime/src/test/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexCreationTupleProcessorTest.java
new file mode 100644
index 0000000..1db124a
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/test/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexCreationTupleProcessorTest.java
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.operators;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.dataflow.data.nontagged.MissingWriterFactory;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.TypeTagUtil;
+import org.apache.asterix.runtime.operators.LSMSecondaryIndexCreationTupleProcessorNodePushable.DeletedTupleCounter;
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.comm.IFrameWriter;
+import org.apache.hyracks.api.comm.VSizeFrame;
+import org.apache.hyracks.api.context.IHyracksJobletContext;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.state.IStateObject;
+import org.apache.hyracks.api.dataflow.value.IMissingWriter;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.BooleanPointable;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.dataflow.common.data.marshalling.BooleanSerializerDeserializer;
+import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import org.apache.hyracks.dataflow.common.utils.TupleUtils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+public class LSMSecondaryIndexCreationTupleProcessorTest {
+
+ private static final int FRAME_SIZE = 512;
+
+ private final ISerializerDeserializer[] fields =
+ { IntegerSerializerDeserializer.INSTANCE, BooleanSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+
+ private final RecordDescriptor recDesc = new RecordDescriptor(fields);
+
+ private final int numTagFields = 2;
+
+ private final int numSecondaryKeys = 1;
+
+ private final int numPrimaryKeys = 1;
+
+ private class ResultFrameWriter implements IFrameWriter {
+ FrameTupleAccessor resultAccessor = new FrameTupleAccessor(recDesc);
+
+ FrameTupleReference tuple = new FrameTupleReference();
+ final List<ITupleReference> resultTuples;
+
+ public ResultFrameWriter(List<ITupleReference> resultTuples) {
+ this.resultTuples = resultTuples;
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ resultAccessor.reset(buffer);
+ int count = resultAccessor.getTupleCount();
+ for (int i = 0; i < count; i++) {
+ tuple.reset(resultAccessor, i);
+ resultTuples.add(TupleUtils.copyTuple(tuple));
+ }
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ }
+ }
+
+ @Test
+ public void testBasic() {
+ List<Object[]> inputs = new ArrayList<>();
+ inputs.add(new Object[] { 0, false, 1, 1 });
+ inputs.add(new Object[] { 1, false, 1, 2 });
+ inputs.add(new Object[] { 2, false, 2, 3 });
+
+ List<Object[]> outputs = new ArrayList<>();
+ outputs.add(new Object[] { 0, false, 1, 1 });
+ outputs.add(new Object[] { 1, false, 1, 2 });
+ outputs.add(new Object[] { 2, false, 2, 3 });
+
+ List<Object[]> buddyOutputs = new ArrayList<>();
+ buddyOutputs.add(new Object[] { 0, false, 1, 1 });
+ buddyOutputs.add(new Object[] { 1, false, 1, 2 });
+ buddyOutputs.add(new Object[] { 2, false, 2, 3 });
+ try {
+ runTest(inputs, outputs, new int[] { 0, 0, 0 }, false);
+ runTest(inputs, buddyOutputs, new int[] { 0, 0, 0 }, true);
+ } catch (HyracksDataException e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testUpsertWithSameSecondary() {
+ List<Object[]> inputs = new ArrayList<>();
+ inputs.add(new Object[] { 1, false, 1, 1 });
+ inputs.add(new Object[] { 0, false, 1, 1 });
+
+ List<Object[]> outputs = new ArrayList<>();
+ outputs.add(new Object[] { 1, false, 1, 1 });
+ outputs.add(new Object[] { 0, false, 1, 1 });
+
+ List<Object[]> buddyOutputs = new ArrayList<>();
+ buddyOutputs.add(new Object[] { 1, false, 1, 1 });
+ buddyOutputs.add(new Object[] { 0, true, null, 1 });
+ buddyOutputs.add(new Object[] { 0, false, 1, 1 });
+
+ try {
+ runTest(inputs, outputs, new int[] { 0, 0 }, false);
+ runTest(inputs, buddyOutputs, new int[] { 1, 0 }, true);
+ } catch (HyracksDataException e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testUpsertWithDifferentSecondary() {
+ List<Object[]> inputs = new ArrayList<>();
+ inputs.add(new Object[] { 1, false, 2, 1 });
+ inputs.add(new Object[] { 0, false, 1, 1 });
+
+ List<Object[]> outputs = new ArrayList<>();
+ outputs.add(new Object[] { 1, false, 2, 1 });
+ outputs.add(new Object[] { 0, true, 2, 1 });
+ outputs.add(new Object[] { 0, false, 1, 1 });
+
+ List<Object[]> buddyOutputs = new ArrayList<>();
+ buddyOutputs.add(new Object[] { 1, false, 2, 1 });
+ buddyOutputs.add(new Object[] { 0, true, null, 1 });
+ buddyOutputs.add(new Object[] { 0, false, 1, 1 });
+ try {
+ runTest(inputs, outputs, new int[] { 1, 0 }, false);
+ runTest(inputs, buddyOutputs, new int[] { 1, 0 }, true);
+ } catch (HyracksDataException e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testDelete() {
+ List<Object[]> inputs = new ArrayList<>();
+ inputs.add(new Object[] { 2, false, 1, 1 });
+ inputs.add(new Object[] { 1, true, 1, 1 });
+ inputs.add(new Object[] { 0, false, 2, 1 });
+
+ List<Object[]> outputs = new ArrayList<>();
+ outputs.add(new Object[] { 2, false, 1, 1 });
+ outputs.add(new Object[] { 1, true, 1, 1 });
+ outputs.add(new Object[] { 0, false, 2, 1 });
+
+ List<Object[]> buddyOutputs = new ArrayList<>();
+ buddyOutputs.add(new Object[] { 2, false, 1, 1 });
+ buddyOutputs.add(new Object[] { 1, true, null, 1 });
+ buddyOutputs.add(new Object[] { 0, false, 2, 1 });
+
+ try {
+ runTest(inputs, outputs, new int[] { 0, 1, 0 }, false);
+ runTest(inputs, buddyOutputs, new int[] { 0, 1, 0 }, true);
+
+ } catch (HyracksDataException e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testDeleteNullSecondary() {
+ List<Object[]> inputs = new ArrayList<>();
+ inputs.add(new Object[] { 1, false, null, 1 });
+ inputs.add(new Object[] { 0, true, null, 1 });
+
+ List<Object[]> outputs = new ArrayList<>();
+
+ List<Object[]> buddyOutputs = new ArrayList<>();
+
+ try {
+ runTest(inputs, outputs, new int[] { 0, 0 }, false);
+ runTest(inputs, buddyOutputs, new int[] { 0, 0 }, true);
+ } catch (HyracksDataException e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testNullSecondary() {
+ List<Object[]> inputs = new ArrayList<>();
+ inputs.add(new Object[] { 2, false, null, 1 });
+ inputs.add(new Object[] { 1, false, 2, 1 });
+ inputs.add(new Object[] { 0, false, null, 1 });
+
+ List<Object[]> outputs = new ArrayList<>();
+ outputs.add(new Object[] { 1, false, 2, 1 });
+ outputs.add(new Object[] { 0, true, 2, 1 });
+
+ List<Object[]> buddyOutputs = new ArrayList<>();
+ buddyOutputs.add(new Object[] { 1, false, 2, 1 });
+ buddyOutputs.add(new Object[] { 0, true, null, 1 });
+ try {
+ runTest(inputs, outputs, new int[] { 1, 0, 0 }, false);
+ runTest(inputs, buddyOutputs, new int[] { 1, 0, 0 }, true);
+ } catch (HyracksDataException e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ private void runTest(List<Object[]> inputTuples, List<Object[]> expectedTuples, int[] numDeletedTuples,
+ boolean hasBuddyBTree) throws HyracksDataException {
+ List<Object> stateObjects = new ArrayList<>();
+ IHyracksJobletContext jobletContext = Mockito.mock(IHyracksJobletContext.class);
+ IHyracksTaskContext ctx = Mockito.mock(IHyracksTaskContext.class);
+ Mockito.when(ctx.getJobletContext()).thenReturn(jobletContext);
+ Mockito.when(ctx.getInitialFrameSize()).thenReturn(FRAME_SIZE);
+ Mockito.when(ctx.allocateFrame(FRAME_SIZE)).thenAnswer(new Answer<ByteBuffer>() {
+ @Override
+ public ByteBuffer answer(InvocationOnMock invocation) throws Throwable {
+ return ByteBuffer.allocate(FRAME_SIZE);
+ }
+ });
+
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ stateObjects.add(invocation.getArguments()[0]);
+ return null;
+ }
+ }).when(ctx).setStateObject(Mockito.any(IStateObject.class));
+
+ List<ITupleReference> resultTuples = new ArrayList<>();
+ IFrameWriter resultWriter = new ResultFrameWriter(resultTuples);
+
+ LSMSecondaryIndexCreationTupleProcessorNodePushable op =
+ new LSMSecondaryIndexCreationTupleProcessorNodePushable(ctx, 0, recDesc, MissingWriterFactory.INSTANCE,
+ numTagFields, numSecondaryKeys, numPrimaryKeys, hasBuddyBTree);
+ op.setOutputFrameWriter(0, resultWriter, recDesc);
+
+ op.open();
+
+ IFrame frame = new VSizeFrame(ctx);
+ FrameTupleAppender appender = new FrameTupleAppender(frame, true);
+ // generate input tuples
+ for (Object[] inputTuple : inputTuples) {
+ ITupleReference tuple = buildTuple(inputTuple);
+ appender.append(tuple);
+ }
+ appender.write(op, true);
+ op.close();
+
+ // check results
+ Assert.assertEquals("Check the number of returned tuples", expectedTuples.size(), resultTuples.size());
+
+ for (int i = 0; i < expectedTuples.size(); i++) {
+ Object[] expectedTuple = expectedTuples.get(i);
+ ITupleReference resultTuple = resultTuples.get(i);
+
+ // check component pos
+ Assert.assertEquals("Check component position", expectedTuple[0],
+ IntegerPointable.getInteger(resultTuple.getFieldData(0), resultTuple.getFieldStart(0)));
+
+ // check anti-matter flag
+ Assert.assertEquals("Check anti-matter", expectedTuple[1],
+ BooleanPointable.getBoolean(resultTuple.getFieldData(1), resultTuple.getFieldStart(1)));
+
+ if (expectedTuple[2] == null) {
+ Assert.assertTrue("Check secondary key is empty",
+ TypeTagUtil.isType(resultTuple, 2, ATypeTag.SERIALIZED_MISSING_TYPE_TAG));
+ } else {
+ Assert.assertEquals("Check secondary key", expectedTuple[2],
+ IntegerPointable.getInteger(resultTuple.getFieldData(2), resultTuple.getFieldStart(2)));
+ }
+
+ Assert.assertEquals("Check primary key", expectedTuple[3],
+ IntegerPointable.getInteger(resultTuple.getFieldData(3), resultTuple.getFieldStart(3)));
+ }
+
+ //check num deleted tuples
+ Assert.assertEquals("Check state object", 1, stateObjects.size());
+ DeletedTupleCounter counter = (DeletedTupleCounter) stateObjects.get(0);
+ for (int i = 0; i < numDeletedTuples.length; i++) {
+ Assert.assertEquals("Check num of deleted tuples", numDeletedTuples[i], counter.get(i));
+ }
+ }
+
+ private ITupleReference buildTuple(Object[] values) throws HyracksDataException {
+ ArrayTupleBuilder builder = new ArrayTupleBuilder(numTagFields + numSecondaryKeys + numPrimaryKeys);
+ builder.addField(IntegerSerializerDeserializer.INSTANCE, (int) values[0]);
+ builder.addField(BooleanSerializerDeserializer.INSTANCE, (boolean) values[1]);
+ if (values[2] == null) {
+ IMissingWriter writer = MissingWriterFactory.INSTANCE.createMissingWriter();
+ writer.writeMissing(builder.getDataOutput());
+ builder.addFieldEndOffset();
+ } else {
+ builder.addField(IntegerSerializerDeserializer.INSTANCE, (int) values[2]);
+ }
+
+ builder.addField(IntegerSerializerDeserializer.INSTANCE, (int) values[3]);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ tuple.reset(builder.getFieldEndOffsets(), builder.getByteArray());
+ return tuple;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
index 2b92a20..3ea29bc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
@@ -63,6 +63,11 @@
</dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-dataflow-std</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-am-common</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDiskComponentScanOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDiskComponentScanOperatorDescriptor.java
new file mode 100644
index 0000000..291363d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDiskComponentScanOperatorDescriptor.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.btree.dataflow;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
+import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+
+public class LSMBTreeDiskComponentScanOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ protected final IIndexDataflowHelperFactory indexHelperFactory;
+ protected final ISearchOperationCallbackFactory searchCallbackFactory;
+
+ public LSMBTreeDiskComponentScanOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor outRecDesc,
+ IIndexDataflowHelperFactory indexHelperFactory, ISearchOperationCallbackFactory searchCallbackFactory) {
+ super(spec, 1, 1);
+ this.indexHelperFactory = indexHelperFactory;
+ this.searchCallbackFactory = searchCallbackFactory;
+ this.outRecDescs[0] = outRecDesc;
+ }
+
+ @Override
+ public LSMBTreeDiskComponentScanOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
+ return new LSMBTreeDiskComponentScanOperatorNodePushable(ctx, partition,
+ recordDescProvider.getInputRecordDescriptor(getActivityId(), 0), indexHelperFactory,
+ searchCallbackFactory);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDiskComponentScanOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDiskComponentScanOperatorNodePushable.java
new file mode 100644
index 0000000..1f71876
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDiskComponentScanOperatorNodePushable.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.dataflow;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndex;
+import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import org.apache.hyracks.storage.am.common.dataflow.IndexSearchOperatorNodePushable;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.common.ISearchPredicate;
+
+public class LSMBTreeDiskComponentScanOperatorNodePushable extends IndexSearchOperatorNodePushable {
+
+ public LSMBTreeDiskComponentScanOperatorNodePushable(IHyracksTaskContext ctx, int partition,
+ RecordDescriptor inputRecDesc, IIndexDataflowHelperFactory indexHelperFactory,
+ ISearchOperationCallbackFactory searchCallbackFactory) throws HyracksDataException {
+ super(ctx, inputRecDesc, partition, null, null, indexHelperFactory, false, false, null, searchCallbackFactory,
+ false);
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ try {
+ ((ILSMIndexAccessor) indexAccessor).scanDiskComponents(cursor);
+ writeSearchResults(0);
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ protected ISearchPredicate createSearchPredicate() {
+ // do nothing
+ // no need to create search predicate for disk component scan operation
+ return null;
+ }
+
+ @Override
+ protected void resetSearchPredicate(int tupleIndex) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected int getFieldCount() {
+ return ((ITreeIndex) index).getFieldCount() + 2;
+ }
+
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index 1c99b17..f6f4828 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -675,7 +675,8 @@
}
}
- protected ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException {
+ @Override
+ public ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException {
LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
return createDiskComponent(bulkComponentFactory, componentFileRefs.getInsertIndexFileReference(),
componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index e830b3e..ffdfe2a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -455,7 +455,8 @@
return new LSMBTreeBulkLoader(this, fillLevel, verifyInput, numElementsHint);
}
- protected ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException {
+ @Override
+ public ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException {
LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
return createDiskComponent(bulkLoadComponentFactory, componentFileRefs.getInsertIndexFileReference(),
componentFileRefs.getBloomFilterFileReference(), true);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java
index 3c9f709..ab9b5af 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.lsm.btree.impls;
-import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.data.std.primitive.BooleanPointable;
@@ -107,13 +106,17 @@
super.next();
LSMBTreeTupleReference diskTuple = (LSMBTreeTupleReference) super.getTuple();
if (diskTuple.isAntimatter()) {
- setAntiMatterTuple(diskTuple, outputElement.getCursorIndex());
+ if (setAntiMatterTuple(diskTuple, outputElement.getCursorIndex())) {
+ foundNext = true;
+ return true;
+ }
} else {
//matter tuple
setMatterTuple(diskTuple, outputElement.getCursorIndex());
+ foundNext = true;
+ return true;
}
- foundNext = true;
- return true;
+
}
return false;
@@ -139,21 +142,23 @@
}
originalTuple = new PermutingTupleReference(permutation);
}
-
//build the matter tuple
buildTuple(tupleBuilder, diskTuple, cursorIndex, MATTER_TUPLE_FLAG);
outputTuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
originalTuple.reset(outputTuple);
}
- private void setAntiMatterTuple(ITupleReference diskTuple, int cursorIndex) throws HyracksDataException {
+ private boolean setAntiMatterTuple(ITupleReference diskTuple, int cursorIndex) throws HyracksDataException {
if (originalTuple == null || cmp.compare(diskTuple, originalTuple) != 0) {
- //This shouldn't happen, because we shouldn't place an anti-matter tuple
- //into the primary index if that tuple is not there
- throw HyracksDataException.create(ErrorCode.CANNOT_FIND_MATTER_TUPLE_FOR_ANTI_MATTER_TUPLE);
+ // This could happen sometimes...
+ // Consider insert tuple A into the memory component, and then delete it immediately.
+ // We would have -A in the memory component, but there is no tuple A in the previous disk components.
+ // But in this case, we can simply ignore it for the scan purpose
+ return false;
}
buildTuple(antiMatterTupleBuilder, originalTuple, cursorIndex, ANTIMATTER_TUPLE_FLAG);
outputTuple.reset(antiMatterTupleBuilder.getFieldEndOffsets(), antiMatterTupleBuilder.getByteArray());
+ return true;
}
private void buildTuple(ArrayTupleBuilder builder, ITupleReference diskTuple, int cursorIndex,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
index 9e1f30d..ae2c93d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
@@ -159,4 +159,11 @@
boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter,
boolean cleanupEmptyComponent) throws HyracksDataException;
+ /**
+ * Creates a disk component for the bulk load operation
+ *
+ * @return
+ * @throws HyracksDataException
+ */
+ ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index d5edbb5..30266fb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -549,12 +549,13 @@
componentBulkLoader.abort();
}
- private ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException {
- LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- return createDiskInvIndexComponent(componentFactory, componentFileRefs.getInsertIndexFileReference(),
- componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(),
- true);
- }
+ }
+
+ @Override
+ public ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException {
+ LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
+ return createDiskInvIndexComponent(componentFactory, componentFileRefs.getInsertIndexFileReference(),
+ componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true);
}
protected InMemoryInvertedIndex createInMemoryInvertedIndex(IVirtualBufferCache virtualBufferCache,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 56af16d..f295f5b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -338,7 +338,8 @@
buddyBTreeFields);
}
- protected ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException {
+ @Override
+ public ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException {
LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
return createDiskComponent(componentFactory, componentFileRefs.getInsertIndexFileReference(),
componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index ae72884..19f0ca0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -311,12 +311,13 @@
}
}
- private ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException {
- LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference();
- return createDiskComponent(bulkLoaComponentFactory, relFlushFileRefs.getInsertIndexFileReference(), null,
- null, true);
- }
+ }
+ @Override
+ public ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException {
+ LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference();
+ return createDiskComponent(bulkLoaComponentFactory, relFlushFileRefs.getInsertIndexFileReference(), null, null,
+ true);
}
@Override