[ASTERIXDB-1946][STO][IDX]Create RTree/InvertedIdx for Correlated Datasets
- user model changes: allow creating secondary rtree and inverted index
for datasets using correlated merge policy
- storage format changes: no
- interface changes: no
details:
- Implmeneted create secondary RTree and Inverted Index for datasets
using correlated merge policy.
- Add tests for RTree and Inverted Index creation job
Change-Id: I100fc0b86b8a6fa36a95d77806107bad0307544e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1845
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <buyingyi@gmail.com>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable.1.ddl.sqlpp
new file mode 100644
index 0000000..74be0ef
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable.1.ddl.sqlpp
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-inverted-index-ngram-correlated-secondary-index-nullable.sqlpp
+ * Description : This test is intended to test deletion from correlated secondary ngram inverted index
+ * that are built on nullable fields.
+ * Expected Result : Success
+ * Date : June 21 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : bigint,
+ dblpid : string,
+ title : string?,
+ authors : string,
+ misc : string
+}
+
+create dataset DBLP(DBLPType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create feed DBLPFeed
+using localfs
+((`path`=`asterix_nc1://data/dblp-small/dblp-small-nulls.adm`),
+(`format`=`adm`),
+(`delimiter`=`:`),
+(`type-name`=`DBLPType`),
+(`tuple-interval`=`10`));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable.2.update.sqlpp
new file mode 100644
index 0000000..868b798
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable.2.update.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+set `wait-for-completion-feed` "true";
+
+connect feed DBLPFeed to dataset DBLP;
+
+start feed DBLPFeed;
+
+delete from DBLP
+ where id > 50;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable.3.ddl.sqlpp
new file mode 100644
index 0000000..f1d7f46
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable.3.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index ngram_index on DBLP (title) type ngram (3);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable.4.query.sqlpp
new file mode 100644
index 0000000..0ac4ba6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable/scan-delete-inverted-index-ngram-correlated-secondary-index-nullable.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element o
+from DBLP as o
+where test.contains(o.title,'Multimedia')
+order by o.id
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-open/scan-delete-inverted-index-ngram-correlated-secondary-index-open.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-open/scan-delete-inverted-index-ngram-correlated-secondary-index-open.1.ddl.sqlpp
new file mode 100644
index 0000000..c59a9d3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-open/scan-delete-inverted-index-ngram-correlated-secondary-index-open.1.ddl.sqlpp
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-inverted-index-ngram-correlated-secondary-index-open.sqlpp
+ * Description : This test is intended to test deletion from correlated secondary ngram inverted index that is built on open field.
+ * Expected Result : Success
+ * Date : June 21 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : bigint,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.DBLPOpenType as
+{
+ id : bigint,
+ dblpid : string,
+ authors : string,
+ misc : string
+}
+
+create dataset DBLP(DBLPType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset DBLPOpen(DBLPOpenType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-open/scan-delete-inverted-index-ngram-correlated-secondary-index-open.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-open/scan-delete-inverted-index-ngram-correlated-secondary-index-open.2.update.sqlpp
new file mode 100644
index 0000000..5865ca6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-open/scan-delete-inverted-index-ngram-correlated-secondary-index-open.2.update.sqlpp
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+load dataset DBLP using localfs ((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),(`format`=`delimited-text`),(`delimiter`=`:`)) pre-sorted;
+
+insert into test.DBLPOpen
+select element x
+from `test.DBLP` as x
+;
+
+delete from DBLPOpen
+ where id > 50;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-open/scan-delete-inverted-index-ngram-correlated-secondary-index-open.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-open/scan-delete-inverted-index-ngram-correlated-secondary-index-open.3.ddl.sqlpp
new file mode 100644
index 0000000..f0edc78
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-open/scan-delete-inverted-index-ngram-correlated-secondary-index-open.3.ddl.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+create index ngram_index on DBLPOpen (title:string?) type ngram (3) enforced;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-open/scan-delete-inverted-index-ngram-correlated-secondary-index-open.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-open/scan-delete-inverted-index-ngram-correlated-secondary-index-open.5.query.sqlpp
new file mode 100644
index 0000000..95ba079
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index-open/scan-delete-inverted-index-ngram-correlated-secondary-index-open.5.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select element o
+from DBLPOpen as o
+where test.contains(o.title,'Multimedia')
+order by o.id
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index/scan-delete-inverted-index-ngram-correlated-secondary-index.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index/scan-delete-inverted-index-ngram-correlated-secondary-index.1.ddl.sqlpp
new file mode 100644
index 0000000..0ca47f4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index/scan-delete-inverted-index-ngram-correlated-secondary-index.1.ddl.sqlpp
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-inverted-index-ngram-correlated-secondary-index.sqlpp
+ * Description : This test is intended to test deletion from correlated secondary ngram inverted index.
+ * Expected Result : Success
+ * Date : June 21 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : bigint,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create dataset DBLP(DBLPType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create feed DBLPFeed
+using localfs
+((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),
+(`format`=`delimited-text`),
+(`delimiter`=`:`),
+(`type-name`=`DBLPType`),
+(`tuple-interval`=`10`));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index/scan-delete-inverted-index-ngram-correlated-secondary-index.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index/scan-delete-inverted-index-ngram-correlated-secondary-index.2.update.sqlpp
new file mode 100644
index 0000000..7dbad4f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index/scan-delete-inverted-index-ngram-correlated-secondary-index.2.update.sqlpp
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+use test;
+
+
+set `wait-for-completion-feed` "true";
+
+connect feed DBLPFeed to dataset DBLP;
+
+start feed DBLPFeed;
+
+delete from DBLP
+ where id > 50;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index/scan-delete-inverted-index-ngram-correlated-secondary-index.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index/scan-delete-inverted-index-ngram-correlated-secondary-index.3.ddl.sqlpp
new file mode 100644
index 0000000..aedc027
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index/scan-delete-inverted-index-ngram-correlated-secondary-index.3.ddl.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+create index ngram_index on DBLP (title) type ngram (3);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index/scan-delete-inverted-index-ngram-correlated-secondary-index.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index/scan-delete-inverted-index-ngram-correlated-secondary-index.4.query.sqlpp
new file mode 100644
index 0000000..0ac4ba6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-ngram-correlated-secondary-index/scan-delete-inverted-index-ngram-correlated-secondary-index.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element o
+from DBLP as o
+where test.contains(o.title,'Multimedia')
+order by o.id
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-nullable/scan-delete-inverted-index-word-correlated-secondary-index-nullable.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-nullable/scan-delete-inverted-index-word-correlated-secondary-index-nullable.1.ddl.sqlpp
new file mode 100644
index 0000000..8e63f7f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-nullable/scan-delete-inverted-index-word-correlated-secondary-index-nullable.1.ddl.sqlpp
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-inverted-index-word-correlated-secondary-index-nullable.sqlpp
+ * Description : This test is intended to test deletion from correlated secondary keyword inverted index
+ * that are built on nullable fields.
+ * Expected Result : Success
+ * Date : Jun 21 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : bigint,
+ dblpid : string,
+ title : string?,
+ authors : string,
+ misc : string
+}
+
+create dataset DBLP(DBLPType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create feed DBLPFeed
+using localfs
+((`path`=`asterix_nc1://data/dblp-small/dblp-small-nulls.adm`),
+(`format`=`adm`),
+(`type-name`=`DBLPType`),
+(`tuple-interval`=`10`));
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-nullable/scan-delete-inverted-index-word-correlated-secondary-index-nullable.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-nullable/scan-delete-inverted-index-word-correlated-secondary-index-nullable.2.update.sqlpp
new file mode 100644
index 0000000..0caeb62
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-nullable/scan-delete-inverted-index-word-correlated-secondary-index-nullable.2.update.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+set `wait-for-completion-feed` "true";
+
+connect feed DBLPFeed to dataset DBLP;
+
+start feed DBLPFeed;
+
+delete from DBLP
+ where id < 50;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-nullable/scan-delete-inverted-index-word-correlated-secondary-index-nullable.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-nullable/scan-delete-inverted-index-word-correlated-secondary-index-nullable.3.ddl.sqlpp
new file mode 100644
index 0000000..c782c45
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-nullable/scan-delete-inverted-index-word-correlated-secondary-index-nullable.3.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index keyword_index on DBLP (title) type keyword;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-nullable/scan-delete-inverted-index-word-correlated-secondary-index-nullable.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-nullable/scan-delete-inverted-index-word-correlated-secondary-index-nullable.4.query.sqlpp
new file mode 100644
index 0000000..ef0cc9b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-nullable/scan-delete-inverted-index-word-correlated-secondary-index-nullable.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element o
+from DBLP as o
+with jacc as test.`similarity-jaccard-check`(test.`word-tokens`(o.title),test.`word-tokens`('Transactions for Cooperative Environments'),0.500000f)
+where jacc[0]
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-open/scan-delete-inverted-index-word-correlated-secondary-index-open.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-open/scan-delete-inverted-index-word-correlated-secondary-index-open.1.ddl.sqlpp
new file mode 100644
index 0000000..929adc2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-open/scan-delete-inverted-index-word-correlated-secondary-index-open.1.ddl.sqlpp
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-inverted-index-word-correlated-secondary-index-open.sqlpp
+ * Description : This test is intended to test deletion from correlated secondary keyword inverted index
+ * that is built on open field.
+ * Expected Result : Success
+ * Date : June 21 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : integer,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.DBLPOpenType as
+{
+ id : integer,
+ dblpid : string,
+ authors : string,
+ misc : string
+}
+
+create dataset DBLP(DBLPType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset DBLPOpen(DBLPOpenType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-open/scan-delete-inverted-index-word-correlated-secondary-index-open.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-open/scan-delete-inverted-index-word-correlated-secondary-index-open.2.update.sqlpp
new file mode 100644
index 0000000..878bdd0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-open/scan-delete-inverted-index-word-correlated-secondary-index-open.2.update.sqlpp
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+load dataset DBLP using localfs ((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),(`format`=`delimited-text`),(`delimiter`=`:`)) pre-sorted;
+
+insert into test.DBLPOpen
+select element x
+from `test.DBLP` as x
+;
+
+delete from DBLPOpen
+ where id < 50;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-open/scan-delete-inverted-index-word-correlated-secondary-index-open.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-open/scan-delete-inverted-index-word-correlated-secondary-index-open.3.ddl.sqlpp
new file mode 100644
index 0000000..1b26ca6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-open/scan-delete-inverted-index-word-correlated-secondary-index-open.3.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index keyword_index on DBLPOpen (title:string?) type keyword enforced;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-open/scan-delete-inverted-index-word-correlated-secondary-index-open.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-open/scan-delete-inverted-index-word-correlated-secondary-index-open.4.query.sqlpp
new file mode 100644
index 0000000..414345c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index-open/scan-delete-inverted-index-word-correlated-secondary-index-open.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element o
+from DBLPOpen as o
+with jacc as test.`similarity-jaccard-check`(test.`word-tokens`(o.title),test.`word-tokens`('Transactions for Cooperative Environments'),0.500000f)
+where jacc[0]
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index/scan-delete-inverted-index-word-correlated-secondary-index.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index/scan-delete-inverted-index-word-correlated-secondary-index.1.ddl.sqlpp
new file mode 100644
index 0000000..58956d7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index/scan-delete-inverted-index-word-correlated-secondary-index.1.ddl.sqlpp
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-inverted-index-word-correlated-secondary-index.sqlpp
+ * Description : This test is intended to test deletion from correlated secondary keyword inverted index.
+ * Expected Result : Success
+ * Date : June 20 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : bigint,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create dataset DBLP(DBLPType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create feed DBLPFeed
+using localfs
+((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),
+(`format`=`delimited-text`),
+(`delimiter`=`:`),
+(`type-name`=`DBLPType`),
+(`tuple-interval`=`10`));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index/scan-delete-inverted-index-word-correlated-secondary-index.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index/scan-delete-inverted-index-word-correlated-secondary-index.2.update.sqlpp
new file mode 100644
index 0000000..46809c5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index/scan-delete-inverted-index-word-correlated-secondary-index.2.update.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+set `wait-for-completion-feed` "true";
+
+connect feed DBLPFeed to dataset DBLP;
+
+start feed DBLPFeed;
+
+delete from DBLP
+ where id < 50;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index/scan-delete-inverted-index-word-correlated-secondary-index.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index/scan-delete-inverted-index-word-correlated-secondary-index.3.ddl.sqlpp
new file mode 100644
index 0000000..c782c45
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index/scan-delete-inverted-index-word-correlated-secondary-index.3.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index keyword_index on DBLP (title) type keyword;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index/scan-delete-inverted-index-word-correlated-secondary-index.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index/scan-delete-inverted-index-word-correlated-secondary-index.4.query.sqlpp
new file mode 100644
index 0000000..ef0cc9b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-inverted-index-word-correlated-secondary-index/scan-delete-inverted-index-word-correlated-secondary-index.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element o
+from DBLP as o
+with jacc as test.`similarity-jaccard-check`(test.`word-tokens`(o.title),test.`word-tokens`('Transactions for Cooperative Environments'),0.500000f)
+where jacc[0]
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-nullable/scan-delete-rtree-correlated-secondary-index-nullable.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-nullable/scan-delete-rtree-correlated-secondary-index-nullable.1.ddl.sqlpp
new file mode 100644
index 0000000..bd93783
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-nullable/scan-delete-rtree-correlated-secondary-index-nullable.1.ddl.sqlpp
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-rtree-secondary-index-nullable.sqlpp
+ * Description : This test is intended to test deletion from secondary correlated rtree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date : June 20 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.MyRecord as
+ closed {
+ id : bigint,
+ point : point?,
+ kwds : string,
+ line1 : line,
+ line2 : line,
+ poly1 : polygon,
+ poly2 : polygon,
+ rec : rectangle
+}
+
+create dataset MyData(MyRecord) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create feed MyFeed
+using localfs
+((`path`=`asterix_nc1://data/spatial/spatialDataNulls.json`),
+(`format`=`adm`),
+(`type-name`=`MyRecord`),
+(`tuple-interval`=`10`));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-nullable/scan-delete-rtree-correlated-secondary-index-nullable.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-nullable/scan-delete-rtree-correlated-secondary-index-nullable.2.update.sqlpp
new file mode 100644
index 0000000..7efdaae
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-nullable/scan-delete-rtree-correlated-secondary-index-nullable.2.update.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+set `wait-for-completion-feed` "true";
+
+connect feed MyFeed to dataset MyData;
+
+start feed MyFeed;
+
+delete from MyData
+ where id > 10;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-nullable/scan-delete-rtree-correlated-secondary-index-nullable.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-nullable/scan-delete-rtree-correlated-secondary-index-nullable.3.ddl.sqlpp
new file mode 100644
index 0000000..662b250
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-nullable/scan-delete-rtree-correlated-secondary-index-nullable.3.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index rtree_index_point on MyData (point) type rtree;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-nullable/scan-delete-rtree-correlated-secondary-index-nullable.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-nullable/scan-delete-rtree-correlated-secondary-index-nullable.4.query.sqlpp
new file mode 100644
index 0000000..f335262
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-nullable/scan-delete-rtree-correlated-secondary-index-nullable.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element {'id':o.id}
+from MyData as o
+where test.`spatial-intersect`(o.point,test.`create-polygon`([0.0,1.0,0.0,4.0,12.0,4.0,12.0,1.0]))
+order by o.id
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-open/scan-delete-rtree-correlated-secondary-index-open.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-open/scan-delete-rtree-correlated-secondary-index-open.1.ddl.sqlpp
new file mode 100644
index 0000000..002228c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-open/scan-delete-rtree-correlated-secondary-index-open.1.ddl.sqlpp
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-delete-rtree-correlated-secondary-index-open.sqlpp
+ * Description : This test is intended to test deletion from correlated secondary rtree index
+ * that is built on open field.
+ * Expected Result : Success
+ * Date : June 20 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.MyRecord as
+ closed {
+ id : bigint,
+ point : point,
+ kwds : string,
+ line1 : line,
+ line2 : line,
+ poly1 : polygon,
+ poly2 : polygon,
+ rec : rectangle,
+ circle : circle
+}
+
+create type test.MyRecordOpen as
+{
+ id : bigint,
+ kwds : string,
+ line1 : line,
+ line2 : line,
+ poly1 : polygon,
+ poly2 : polygon,
+ rec : rectangle,
+ circle : circle
+}
+
+create dataset MyData(MyRecord) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset MyDataOpen(MyRecordOpen) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-open/scan-delete-rtree-correlated-secondary-index-open.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-open/scan-delete-rtree-correlated-secondary-index-open.2.update.sqlpp
new file mode 100644
index 0000000..3fae91c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-open/scan-delete-rtree-correlated-secondary-index-open.2.update.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+load dataset MyData using localfs ((`path`=`asterix_nc1://data/spatial/spatialData.json`),(`format`=`adm`)) pre-sorted;
+
+insert into test.MyDataOpen
+select element x
+from `test.MyData` as x
+;
+
+delete from MyDataOpen
+ where id > 10;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-open/scan-delete-rtree-correlated-secondary-index-open.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-open/scan-delete-rtree-correlated-secondary-index-open.3.ddl.sqlpp
new file mode 100644
index 0000000..83625e6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-open/scan-delete-rtree-correlated-secondary-index-open.3.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index rtree_index_point on MyDataOpen (point:point?) type rtree enforced;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-open/scan-delete-rtree-correlated-secondary-index-open.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-open/scan-delete-rtree-correlated-secondary-index-open.4.query.sqlpp
new file mode 100644
index 0000000..ec7c725
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index-open/scan-delete-rtree-correlated-secondary-index-open.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element {'id':o.id}
+from MyDataOpen as o
+where test.`spatial-intersect`(o.point,test.`create-polygon`([0.0,1.0,0.0,4.0,12.0,4.0,12.0,1.0]))
+order by o.id
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index/scan-delete-rtree-correlated-secondary-index.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index/scan-delete-rtree-correlated-secondary-index.1.ddl.sqlpp
new file mode 100644
index 0000000..d2fa3db
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index/scan-delete-rtree-correlated-secondary-index.1.ddl.sqlpp
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test case Name : scan-delete-rtree-correlated-secondary-index.sqlpp
+ * Description : This test is intended to test deletion from correlated secondary rtree indexes
+ * Expected Result : Success
+ * Date : June 20 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.MyRecord as
+ closed {
+ id : bigint,
+ point : point,
+ kwds : string,
+ line1 : line,
+ line2 : line,
+ poly1 : polygon,
+ poly2 : polygon,
+ rec : rectangle,
+ circle : circle
+}
+
+create dataset MyData(MyRecord) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+
+create feed MyFeed
+using localfs
+((`path`=`asterix_nc1://data/spatial/spatialData.json`),
+(`format`=`adm`),
+(`type-name`=`MyRecord`),
+(`tuple-interval`=`10`));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index/scan-delete-rtree-correlated-secondary-index.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index/scan-delete-rtree-correlated-secondary-index.2.update.sqlpp
new file mode 100644
index 0000000..7efdaae
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index/scan-delete-rtree-correlated-secondary-index.2.update.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+set `wait-for-completion-feed` "true";
+
+connect feed MyFeed to dataset MyData;
+
+start feed MyFeed;
+
+delete from MyData
+ where id > 10;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index/scan-delete-rtree-correlated-secondary-index.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index/scan-delete-rtree-correlated-secondary-index.3.ddl.sqlpp
new file mode 100644
index 0000000..662b250
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index/scan-delete-rtree-correlated-secondary-index.3.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index rtree_index_point on MyData (point) type rtree;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index/scan-delete-rtree-correlated-secondary-index.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index/scan-delete-rtree-correlated-secondary-index.4.query.sqlpp
new file mode 100644
index 0000000..f335262
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-rtree-correlated-secondary-index/scan-delete-rtree-correlated-secondary-index.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element {'id':o.id}
+from MyData as o
+where test.`spatial-intersect`(o.point,test.`create-polygon`([0.0,1.0,0.0,4.0,12.0,4.0,12.0,1.0]))
+order by o.id
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable.1.ddl.sqlpp
new file mode 100644
index 0000000..0773e99
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable.1.ddl.sqlpp
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-insert-inverted-index-ngram-correlated-secondary-index-nullable.sqlpp
+ * Description : This test is intended to test insertion from correlated secondary ngram inverted index
+ * that are built on nullable fields.
+ * Expected Result : Success
+ * Date : June 21 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : bigint,
+ dblpid : string,
+ title : string?,
+ authors : string,
+ misc : string
+}
+
+create dataset DBLP(DBLPType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset DBLP1(DBLPType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable.2.update.sqlpp
new file mode 100644
index 0000000..dfdefc1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable.2.update.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+load dataset DBLP using localfs ((`path`=`asterix_nc1://data/dblp-small/dblp-small-nulls.adm`),(`format`=`adm`));
+
+insert into DBLP1
+select element {'id':o.id,'dblpid':o.dblpid,'title':o.title,'authors':o.authors,'misc':o.misc}
+from DBLP as o
+where test.contains(o.title,'Multimedia')
+order by o.id
+;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable.3.ddl.sqlpp
new file mode 100644
index 0000000..2854258
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable.3.ddl.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index ngram_index on DBLP (title) type ngram (3);
+
+create index ngram_index1 on DBLP1 (title) type ngram (3);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable.4.query.sqlpp
new file mode 100644
index 0000000..cfa4899
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable/scan-insert-inverted-index-ngram-correlated-secondary-index-nullable.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element o
+from DBLP1 as o
+where test.contains(o.title,'Multimedia')
+order by o.id
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-open/scan-insert-inverted-index-ngram-correlated-secondary-index-open.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-open/scan-insert-inverted-index-ngram-correlated-secondary-index-open.1.ddl.sqlpp
new file mode 100644
index 0000000..50c691a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-open/scan-insert-inverted-index-ngram-correlated-secondary-index-open.1.ddl.sqlpp
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-insert-inverted-index-ngram-correlated-secondary-index-open.sqlpp
+ * Description : This test is intended to test insertion from correlated secondary ngram inverted index
+ * that is built on open field.
+ * Expected Result : Success
+ * Date : June 21 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : bigint,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.DBLPOpenType as
+{
+ id : bigint,
+ dblpid : string,
+ authors : string,
+ misc : string
+}
+
+create dataset DBLP(DBLPType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset DBLPOpen(DBLPOpenType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-open/scan-insert-inverted-index-ngram-correlated-secondary-index-open.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-open/scan-insert-inverted-index-ngram-correlated-secondary-index-open.2.update.sqlpp
new file mode 100644
index 0000000..d696508
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-open/scan-insert-inverted-index-ngram-correlated-secondary-index-open.2.update.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+load dataset DBLP using localfs ((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),(`format`=`delimited-text`),(`delimiter`=`:`)) pre-sorted;
+
+insert into DBLPOpen
+select element {'id':o.id,'dblpid':o.dblpid,'title':o.title,'authors':o.authors,'misc':o.misc}
+from DBLP as o
+where test.contains(o.title,'Multimedia')
+order by o.id
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-open/scan-insert-inverted-index-ngram-correlated-secondary-index-open.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-open/scan-insert-inverted-index-ngram-correlated-secondary-index-open.3.ddl.sqlpp
new file mode 100644
index 0000000..a347007
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-open/scan-insert-inverted-index-ngram-correlated-secondary-index-open.3.ddl.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index ngram_index on DBLP (title) type ngram (3);
+
+create index ngram_index1 on DBLPOpen (title:string?) type ngram (3) enforced;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-open/scan-insert-inverted-index-ngram-correlated-secondary-index-open.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-open/scan-insert-inverted-index-ngram-correlated-secondary-index-open.4.query.sqlpp
new file mode 100644
index 0000000..4a2da41
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index-open/scan-insert-inverted-index-ngram-correlated-secondary-index-open.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element o
+from DBLPOpen as o
+where test.contains(o.title,'Multimedia')
+order by o.id
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index/scan-insert-inverted-index-ngram-correlated-secondary-index.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index/scan-insert-inverted-index-ngram-correlated-secondary-index.1.ddl.sqlpp
new file mode 100644
index 0000000..2010c30
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index/scan-insert-inverted-index-ngram-correlated-secondary-index.1.ddl.sqlpp
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-insert-inverted-index-ngram-correlated-secondary-index.sqlpp
+ * Description : This test is intended to test insertion from correlated secondary ngram inverted index.
+ * Expected Result : Success
+ * Date : June 21 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : bigint,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create dataset DBLP(DBLPType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset DBLP1(DBLPType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index/scan-insert-inverted-index-ngram-correlated-secondary-index.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index/scan-insert-inverted-index-ngram-correlated-secondary-index.2.update.sqlpp
new file mode 100644
index 0000000..d68e90c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index/scan-insert-inverted-index-ngram-correlated-secondary-index.2.update.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+load dataset DBLP using localfs ((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),(`format`=`delimited-text`),(`delimiter`=`:`)) pre-sorted;
+
+insert into DBLP1
+select element {'id':o.id,'dblpid':o.dblpid,'title':o.title,'authors':o.authors,'misc':o.misc}
+from DBLP as o
+where test.contains(o.title,'Multimedia')
+order by o.id
+;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index/scan-insert-inverted-index-ngram-correlated-secondary-index.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index/scan-insert-inverted-index-ngram-correlated-secondary-index.3.ddl.sqlpp
new file mode 100644
index 0000000..752107f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index/scan-insert-inverted-index-ngram-correlated-secondary-index.3.ddl.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+create index ngram_index on DBLP (title) type ngram (3);
+
+create index ngram_index1 on DBLP1 (title) type ngram (3);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index/scan-insert-inverted-index-ngram-correlated-secondary-index.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index/scan-insert-inverted-index-ngram-correlated-secondary-index.4.query.sqlpp
new file mode 100644
index 0000000..cfa4899
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-correlated-secondary-index/scan-insert-inverted-index-ngram-correlated-secondary-index.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element o
+from DBLP1 as o
+where test.contains(o.title,'Multimedia')
+order by o.id
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-nullable/scan-insert-inverted-index-word-correlated-secondary-index-nullable.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-nullable/scan-insert-inverted-index-word-correlated-secondary-index-nullable.1.ddl.sqlpp
new file mode 100644
index 0000000..a196e40
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-nullable/scan-insert-inverted-index-word-correlated-secondary-index-nullable.1.ddl.sqlpp
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-insert-inverted-index-word-correlated-secondary-index-nullable.sqlpp
+ * Description : This test is intended to test insertion from correlated secondary keyword inverted index
+ * that are built on nullable fields.
+ * Expected Result : Success
+ * Date : June 21 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : bigint,
+ dblpid : string,
+ title : string?,
+ authors : string,
+ misc : string
+}
+
+create dataset DBLP(DBLPType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset DBLP1(DBLPType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-nullable/scan-insert-inverted-index-word-correlated-secondary-index-nullable.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-nullable/scan-insert-inverted-index-word-correlated-secondary-index-nullable.2.update.sqlpp
new file mode 100644
index 0000000..e65cac3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-nullable/scan-insert-inverted-index-word-correlated-secondary-index-nullable.2.update.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+load dataset DBLP using localfs ((`path`=`asterix_nc1://data/dblp-small/dblp-small-nulls.adm`),(`format`=`adm`));
+
+insert into DBLP1
+select element {'id':o.id,'dblpid':o.dblpid,'title':o.title,'authors':o.authors,'misc':o.misc}
+from DBLP as o
+order by o.id
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-nullable/scan-insert-inverted-index-word-correlated-secondary-index-nullable.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-nullable/scan-insert-inverted-index-word-correlated-secondary-index-nullable.3.ddl.sqlpp
new file mode 100644
index 0000000..ab2be5c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-nullable/scan-insert-inverted-index-word-correlated-secondary-index-nullable.3.ddl.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index keyword_index on DBLP (title) type keyword;
+
+create index keyword_index1 on DBLP1 (title) type keyword;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-nullable/scan-insert-inverted-index-word-correlated-secondary-index-nullable.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-nullable/scan-insert-inverted-index-word-correlated-secondary-index-nullable.4.query.sqlpp
new file mode 100644
index 0000000..34f332c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-nullable/scan-insert-inverted-index-word-correlated-secondary-index-nullable.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element o
+from DBLP1 as o
+with jacc as test.`similarity-jaccard-check`(test.`word-tokens`(o.title),test.`word-tokens`('Transactions for Cooperative Environments'),0.500000f)
+where jacc[0]
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-open/scan-insert-inverted-index-word-correlated-secondary-index-open.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-open/scan-insert-inverted-index-word-correlated-secondary-index-open.1.ddl.sqlpp
new file mode 100644
index 0000000..0cf8927
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-open/scan-insert-inverted-index-word-correlated-secondary-index-open.1.ddl.sqlpp
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-insert-inverted-index-word-correlated-secondary-index-open.sqlpp
+ * Description : This test is intended to test insertion from correlated secondary keyword inverted index
+ * that is built on open field.
+ * Expected Result : Success
+ * Date : June 21 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : bigint,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.DBLPOpenType as
+{
+ id : bigint,
+ dblpid : string,
+ authors : string,
+ misc : string
+}
+
+create dataset DBLP(DBLPType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset DBLPOpen(DBLPOpenType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-open/scan-insert-inverted-index-word-correlated-secondary-index-open.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-open/scan-insert-inverted-index-word-correlated-secondary-index-open.2.update.sqlpp
new file mode 100644
index 0000000..62c8ab9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-open/scan-insert-inverted-index-word-correlated-secondary-index-open.2.update.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+use test;
+
+
+load dataset DBLP using localfs ((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),(`format`=`delimited-text`),(`delimiter`=`:`)) pre-sorted;
+
+insert into DBLPOpen
+select element {'id':o.id,'dblpid':o.dblpid,'title':o.title,'authors':o.authors,'misc':o.misc}
+from DBLP as o
+order by o.id
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-open/scan-insert-inverted-index-word-correlated-secondary-index-open.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-open/scan-insert-inverted-index-word-correlated-secondary-index-open.3.ddl.sqlpp
new file mode 100644
index 0000000..8ad7222
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-open/scan-insert-inverted-index-word-correlated-secondary-index-open.3.ddl.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index keyword_index on DBLP (title) type keyword;
+
+create index keyword_index1 on DBLPOpen (title:string?) type keyword enforced;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-open/scan-insert-inverted-index-word-correlated-secondary-index-open.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-open/scan-insert-inverted-index-word-correlated-secondary-index-open.4.query.sqlpp
new file mode 100644
index 0000000..414345c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-open/scan-insert-inverted-index-word-correlated-secondary-index-open.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element o
+from DBLPOpen as o
+with jacc as test.`similarity-jaccard-check`(test.`word-tokens`(o.title),test.`word-tokens`('Transactions for Cooperative Environments'),0.500000f)
+where jacc[0]
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index/scan-insert-inverted-index-word-correlated-secondary-index.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index/scan-insert-inverted-index-word-correlated-secondary-index.1.ddl.sqlpp
new file mode 100644
index 0000000..2085f30
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index/scan-insert-inverted-index-word-correlated-secondary-index.1.ddl.sqlpp
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-insert-inverted-index-word-correlated-secondary-index.sqlpp
+ * Description : This test is intended to test insertion from correlated secondary keyword inverted index.
+ * Expected Result : Success
+ * Date : June 21 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : bigint,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create dataset DBLP(DBLPType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset DBLP1(DBLPType) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index/scan-insert-inverted-index-word-correlated-secondary-index.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index/scan-insert-inverted-index-word-correlated-secondary-index.2.update.sqlpp
new file mode 100644
index 0000000..cf9a86a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index/scan-insert-inverted-index-word-correlated-secondary-index.2.update.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+use test;
+
+
+load dataset DBLP using localfs ((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),(`format`=`delimited-text`),(`delimiter`=`:`)) pre-sorted;
+
+insert into DBLP1
+select element {'id':o.id,'dblpid':o.dblpid,'title':o.title,'authors':o.authors,'misc':o.misc}
+from DBLP as o
+order by o.id
+;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index/scan-insert-inverted-index-word-correlated-secondary-index.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index/scan-insert-inverted-index-word-correlated-secondary-index.3.ddl.sqlpp
new file mode 100644
index 0000000..ab2be5c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index/scan-insert-inverted-index-word-correlated-secondary-index.3.ddl.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index keyword_index on DBLP (title) type keyword;
+
+create index keyword_index1 on DBLP1 (title) type keyword;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index/scan-insert-inverted-index-word-correlated-secondary-index.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index/scan-insert-inverted-index-word-correlated-secondary-index.4.query.sqlpp
new file mode 100644
index 0000000..34f332c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index/scan-insert-inverted-index-word-correlated-secondary-index.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element o
+from DBLP1 as o
+with jacc as test.`similarity-jaccard-check`(test.`word-tokens`(o.title),test.`word-tokens`('Transactions for Cooperative Environments'),0.500000f)
+where jacc[0]
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-nullable/scan-insert-rtree-correlated-secondary-index-nullable.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-nullable/scan-insert-rtree-correlated-secondary-index-nullable.1.ddl.sqlpp
new file mode 100644
index 0000000..7994440
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-nullable/scan-insert-rtree-correlated-secondary-index-nullable.1.ddl.sqlpp
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-insert-rtree-correlated-secondary-index-nullable.sqlpp
+ * Description : This test is intended to test insertion into correlated secondary rtree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date : June 20 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.MyRecord as
+ closed {
+ id : bigint,
+ point : point?,
+ kwds : string,
+ line1 : line,
+ line2 : line,
+ poly1 : polygon,
+ poly2 : polygon,
+ rec : rectangle
+}
+
+create type test.MyMiniRecord as
+ closed {
+ id : bigint,
+ point : point?
+}
+
+create dataset MyData(MyRecord) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+
+create dataset MyMiniData(MyMiniRecord) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create feed MyFeed
+using localfs
+((`path`=`asterix_nc1://data/spatial/spatialDataNulls.json`),
+(`format`=`adm`),
+(`type-name`=`MyRecord`),
+(`tuple-interval`=`10`));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-nullable/scan-insert-rtree-correlated-secondary-index-nullable.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-nullable/scan-insert-rtree-correlated-secondary-index-nullable.2.update.sqlpp
new file mode 100644
index 0000000..2844872
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-nullable/scan-insert-rtree-correlated-secondary-index-nullable.2.update.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+set `wait-for-completion-feed` "true";
+
+connect feed MyFeed to dataset MyData;
+
+start feed MyFeed;
+
+insert into MyMiniData
+select element {'id':m.id,'point':m.point}
+from MyData as m
+;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-nullable/scan-insert-rtree-correlated-secondary-index-nullable.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-nullable/scan-insert-rtree-correlated-secondary-index-nullable.3.ddl.sqlpp
new file mode 100644
index 0000000..c2af66c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-nullable/scan-insert-rtree-correlated-secondary-index-nullable.3.ddl.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+
+create index rtree_index_point on MyMiniData (point) type rtree;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-nullable/scan-insert-rtree-correlated-secondary-index-nullable.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-nullable/scan-insert-rtree-correlated-secondary-index-nullable.4.query.sqlpp
new file mode 100644
index 0000000..40290f3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-nullable/scan-insert-rtree-correlated-secondary-index-nullable.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element {'id':o.id}
+from MyMiniData as o
+where test.`spatial-intersect`(o.point,test.`create-polygon`([0.0,1.0,0.0,4.0,12.0,4.0,12.0,1.0]))
+order by o.id
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-open/scan-insert-rtree-correlated-secondary-index-open.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-open/scan-insert-rtree-correlated-secondary-index-open.1.ddl.sqlpp
new file mode 100644
index 0000000..051c378
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-open/scan-insert-rtree-correlated-secondary-index-open.1.ddl.sqlpp
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : scan-insert-rtree-correlated-secondary-index-open.sqlpp
+ * Description : This test is intended to test insertion from correlated secondary rtree index
+ * that is built on open field.
+ * Expected Result : Success
+ * Date : June 20 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.MyRecord as
+ closed {
+ id : bigint,
+ point : point,
+ kwds : string,
+ line1 : line,
+ line2 : line,
+ poly1 : polygon,
+ poly2 : polygon,
+ rec : rectangle,
+ circle : circle
+}
+
+create type test.MyOpenRecord as
+{
+ id : bigint
+}
+
+create dataset MyData(MyRecord) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset MyOpenData(MyOpenRecord) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create feed MyFeed
+using localfs
+((`path`=`asterix_nc1://data/spatial/spatialData.json`),
+(`format`=`adm`),
+(`type-name`=`MyRecord`),
+(`tuple-interval`=`10`));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-open/scan-insert-rtree-correlated-secondary-index-open.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-open/scan-insert-rtree-correlated-secondary-index-open.2.update.sqlpp
new file mode 100644
index 0000000..0bb80a0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-open/scan-insert-rtree-correlated-secondary-index-open.2.update.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+set `wait-for-completion-feed` "true";
+
+connect feed MyFeed to dataset MyData;
+
+start feed MyFeed;
+
+insert into MyOpenData
+select element {'id':m.id,'point':m.point}
+from MyData as m
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-open/scan-insert-rtree-correlated-secondary-index-open.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-open/scan-insert-rtree-correlated-secondary-index-open.3.ddl.sqlpp
new file mode 100644
index 0000000..cb2e330
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-open/scan-insert-rtree-correlated-secondary-index-open.3.ddl.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+create index rtree_index_point_0 on MyData (point) type rtree;
+
+create index rtree_index_point on MyOpenData (point:point?) type rtree enforced;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-open/scan-insert-rtree-correlated-secondary-index-open.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-open/scan-insert-rtree-correlated-secondary-index-open.5.query.sqlpp
new file mode 100644
index 0000000..f1edec6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index-open/scan-insert-rtree-correlated-secondary-index-open.5.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element {'id':o.id}
+from MyOpenData as o
+where test.`spatial-intersect`(o.point,test.`create-polygon`([0.0,1.0,0.0,4.0,12.0,4.0,12.0,1.0]))
+order by o.id
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index/scan-insert-rtree-correlated-secondary-index.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index/scan-insert-rtree-correlated-secondary-index.1.ddl.sqlpp
new file mode 100644
index 0000000..5e7bb43
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index/scan-insert-rtree-correlated-secondary-index.1.ddl.sqlpp
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type test.MyRecord as
+ closed {
+ id : bigint,
+ point : point,
+ kwds : string,
+ line1 : line,
+ line2 : line,
+ poly1 : polygon,
+ poly2 : polygon,
+ rec : rectangle,
+ circle : circle
+}
+
+create type test.MyMiniRecord as
+ closed {
+ id : bigint,
+ point : point
+}
+
+create dataset MyData(MyRecord) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset MyMiniData(MyMiniRecord) primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create feed MyFeed
+using localfs
+((`path`=`asterix_nc1://data/spatial/spatialData.json`),
+(`format`=`adm`),
+(`type-name`=`MyRecord`),
+(`tuple-interval`=`10`));
+
+create feed MyMiniFeed
+using localfs
+((`path`=`asterix_nc1://data/spatial/spatialData0.json`),
+(`format`=`adm`),
+(`type-name`=`MyMiniRecord`),
+(`tuple-interval`=`10`));
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index/scan-insert-rtree-correlated-secondary-index.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index/scan-insert-rtree-correlated-secondary-index.2.update.sqlpp
new file mode 100644
index 0000000..c7ce132
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index/scan-insert-rtree-correlated-secondary-index.2.update.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+set `wait-for-completion-feed` "true";
+
+connect feed MyFeed to dataset MyData;
+
+start feed MyFeed;
+
+connect feed MyMiniFeed to dataset MyMiniData;
+
+start feed MyMiniFeed;
+
+insert into MyMiniData
+select element {'id':m.id,'point':m.point}
+from MyData as m
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index/scan-insert-rtree-correlated-secondary-index.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index/scan-insert-rtree-correlated-secondary-index.3.ddl.sqlpp
new file mode 100644
index 0000000..ba72d94
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index/scan-insert-rtree-correlated-secondary-index.3.ddl.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index rtree_index_point_0 on MyData (point) type rtree;
+
+create index rtree_index_point on MyMiniData (point) type rtree;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index/scan-insert-rtree-correlated-secondary-index.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index/scan-insert-rtree-correlated-secondary-index.4.query.sqlpp
new file mode 100644
index 0000000..40290f3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-rtree-correlated-secondary-index/scan-insert-rtree-correlated-secondary-index.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element {'id':o.id}
+from MyMiniData as o
+where test.`spatial-intersect`(o.point,test.`create-polygon`([0.0,1.0,0.0,4.0,12.0,4.0,12.0,1.0]))
+order by o.id
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-ngram/insert-with-correlated-secondary-inverted-ngram.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-ngram/insert-with-correlated-secondary-inverted-ngram.1.ddl.sqlpp
new file mode 100644
index 0000000..d2bff51
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-ngram/insert-with-correlated-secondary-inverted-ngram.1.ddl.sqlpp
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test filters with insert pipeline in the existence of a secondary ngram index
+ * Expected Res : Success
+ * Date : 25th Jun 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.FacebookMessageType as
+ closed {
+ `message-id` : bigint,
+ `author-id` : bigint,
+ `in-response-to` : bigint?,
+ `sender-location` : point?,
+ message : string,
+ `send-time` : datetime
+}
+
+create dataset FacebookMessages(FacebookMessageType) primary key `message-id`
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset FacebookMessages2(FacebookMessageType) primary key `message-id`
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`))
+with filter on `send-time`;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-ngram/insert-with-correlated-secondary-inverted-ngram.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-ngram/insert-with-correlated-secondary-inverted-ngram.2.update.sqlpp
new file mode 100644
index 0000000..2449684
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-ngram/insert-with-correlated-secondary-inverted-ngram.2.update.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+load dataset FacebookMessages using localfs ((`path`=`asterix_nc1://data/fbm-with-send-time.adm`),(`format`=`adm`));
+
+insert into FacebookMessages2
+select element m
+from FacebookMessages as m
+;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-ngram/insert-with-correlated-secondary-inverted-ngram.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-ngram/insert-with-correlated-secondary-inverted-ngram.3.ddl.sqlpp
new file mode 100644
index 0000000..14b65f8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-ngram/insert-with-correlated-secondary-inverted-ngram.3.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index fbMessageIdx on FacebookMessages2 (message) type ngram (3);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-ngram/insert-with-correlated-secondary-inverted-ngram.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-ngram/insert-with-correlated-secondary-inverted-ngram.4.query.sqlpp
new file mode 100644
index 0000000..b6dd766
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-ngram/insert-with-correlated-secondary-inverted-ngram.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element m
+from FacebookMessages as m
+where (test.contains(m.message,'love') and (m.`send-time` < test.datetime('2012-12-20T10:10:00')))
+order by m.`send-time`
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-word/insert-with-correlated-secondary-inverted-word.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-word/insert-with-correlated-secondary-inverted-word.1.ddl.sqlpp
new file mode 100644
index 0000000..55940ba
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-word/insert-with-correlated-secondary-inverted-word.1.ddl.sqlpp
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test filters with insert pipeline in the existence of a correlated secondary word index
+ * Expected Res : Success
+ * Date : June 21 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.FacebookMessageType as
+ closed {
+ `message-id` : bigint,
+ `author-id` : bigint,
+ `in-response-to` : bigint?,
+ `sender-location` : point?,
+ message : string,
+ `send-time` : datetime
+}
+
+create dataset FacebookMessages(FacebookMessageType) primary key `message-id`
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset FacebookMessages2(FacebookMessageType) primary key `message-id`
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`))
+with filter on `send-time`;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-word/insert-with-correlated-secondary-inverted-word.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-word/insert-with-correlated-secondary-inverted-word.2.update.sqlpp
new file mode 100644
index 0000000..2449684
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-word/insert-with-correlated-secondary-inverted-word.2.update.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+load dataset FacebookMessages using localfs ((`path`=`asterix_nc1://data/fbm-with-send-time.adm`),(`format`=`adm`));
+
+insert into FacebookMessages2
+select element m
+from FacebookMessages as m
+;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-word/insert-with-correlated-secondary-inverted-word.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-word/insert-with-correlated-secondary-inverted-word.3.ddl.sqlpp
new file mode 100644
index 0000000..9fdd0df
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-word/insert-with-correlated-secondary-inverted-word.3.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index fbMessageIdx on FacebookMessages2 (message) type keyword;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-word/insert-with-correlated-secondary-inverted-word.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-word/insert-with-correlated-secondary-inverted-word.4.query.sqlpp
new file mode 100644
index 0000000..0bfb013
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-inverted-word/insert-with-correlated-secondary-inverted-word.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element m
+from FacebookMessages as m
+where ((test.`similarity-jaccard`(test.`word-tokens`(m.message),test.`word-tokens`('love sprint at&t verizon')) >= 0.200000f) and (m.`send-time` < test.datetime('2012-12-20T10:10:00')))
+order by m.`send-time`
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-rtree/insert-with-correlated-secondary-rtree.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-rtree/insert-with-correlated-secondary-rtree.1.ddl.sqlpp
new file mode 100644
index 0000000..4bbff39
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-rtree/insert-with-correlated-secondary-rtree.1.ddl.sqlpp
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test filters with insert pipeline in the existence of a secondary r-tree
+ * Expected Res : Success
+ * Date : June 20 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.FacebookMessageType as
+ closed {
+ `message-id` : bigint,
+ `author-id` : bigint,
+ `in-response-to` : bigint?,
+ `sender-location` : point?,
+ message : string,
+ `send-time` : datetime
+}
+
+create dataset FacebookMessages(FacebookMessageType) primary key `message-id`
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset FacebookMessages2(FacebookMessageType) primary key `message-id`
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`))
+with filter on `send-time`;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-rtree/insert-with-correlated-secondary-rtree.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-rtree/insert-with-correlated-secondary-rtree.2.update.sqlpp
new file mode 100644
index 0000000..2449684
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-rtree/insert-with-correlated-secondary-rtree.2.update.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+load dataset FacebookMessages using localfs ((`path`=`asterix_nc1://data/fbm-with-send-time.adm`),(`format`=`adm`));
+
+insert into FacebookMessages2
+select element m
+from FacebookMessages as m
+;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-rtree/insert-with-correlated-secondary-rtree.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-rtree/insert-with-correlated-secondary-rtree.3.ddl.sqlpp
new file mode 100644
index 0000000..f50c706
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-rtree/insert-with-correlated-secondary-rtree.3.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create index fbSenderLocIndex on FacebookMessages2 (`sender-location`) type rtree;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-rtree/insert-with-correlated-secondary-rtree.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-rtree/insert-with-correlated-secondary-rtree.4.query.sqlpp
new file mode 100644
index 0000000..7b03c24
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/filters/insert-with-correlated-secondary-rtree/insert-with-correlated-secondary-rtree.4.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+select element m
+from FacebookMessages as m
+where (test.`spatial-intersect`(m.`sender-location`,test.`create-polygon`([40.0,79.87,30.0,75.0,50.0,80.0,10.0,10.0])) and (m.`send-time` < test.datetime('2012-11-20T10:10:00.000Z')))
+order by m.`send-time`
+;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/multiple-correlated-secondaries/multiple-correlated-secondaries.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/multiple-correlated-secondaries/multiple-correlated-secondaries.1.ddl.sqlpp
new file mode 100644
index 0000000..7ac198f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/multiple-correlated-secondaries/multiple-correlated-secondaries.1.ddl.sqlpp
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type MyRecord as closed {
+ id: int64,
+ point: point,
+ kwds: string,
+ line1: line,
+ line2: line,
+ poly1: polygon,
+ poly2: polygon,
+ rec: rectangle,
+ circle: circle
+}
+
+
+create dataset UpsertTo(MyRecord)
+primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+
+create dataset UpsertFrom(MyRecord)
+primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/multiple-correlated-secondaries/multiple-correlated-secondaries.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/multiple-correlated-secondaries/multiple-correlated-secondaries.2.update.sqlpp
new file mode 100644
index 0000000..0c67457
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/multiple-correlated-secondaries/multiple-correlated-secondaries.2.update.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use test;
+
+load dataset UpsertTo
+using localfs
+(("path"="asterix_nc1://data/spatial/spatialData.json"),("format"="adm"));
+
+load dataset UpsertFrom
+using localfs
+(("path"="asterix_nc1://data/spatial/moreSpatialData.json"),("format"="adm"));
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/multiple-correlated-secondaries/multiple-correlated-secondaries.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/multiple-correlated-secondaries/multiple-correlated-secondaries.3.ddl.sqlpp
new file mode 100644
index 0000000..acf2ebd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/multiple-correlated-secondaries/multiple-correlated-secondaries.3.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use test;
+
+
+create index btree_index on UpsertTo(kwds);
+create index rtree_index on UpsertTo(point) type rtree;
+create index inverted_index on UpsertTo(kwds) type keyword;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/multiple-correlated-secondaries/multiple-correlated-secondaries.4.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/multiple-correlated-secondaries/multiple-correlated-secondaries.4.update.sqlpp
new file mode 100644
index 0000000..d7e5245
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/multiple-correlated-secondaries/multiple-correlated-secondaries.4.update.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use test;
+
+upsert into UpsertTo(
+ select value x
+ from UpsertFrom x
+);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/multiple-correlated-secondaries/multiple-correlated-secondaries.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/multiple-correlated-secondaries/multiple-correlated-secondaries.5.query.sqlpp
new file mode 100644
index 0000000..ad45ca2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/multiple-correlated-secondaries/multiple-correlated-secondaries.5.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+USE test;
+
+SELECT o.id id
+FROM UpsertTo o
+WHERE spatial_intersect(o.point, create_polygon([4.0,1.0,4.0,4.0,12.0,4.0,12.0,1.0]))
+ORDER BY o.id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-inverted/primary-correlated-secondary-inverted.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-inverted/primary-correlated-secondary-inverted.1.ddl.sqlpp
new file mode 100644
index 0000000..1480b78
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-inverted/primary-correlated-secondary-inverted.1.ddl.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type DBLPType as closed {
+ id: int64,
+ dblpid: string,
+ title: string,
+ authors: string,
+ misc: string
+}
+
+create dataset UpsertToDBLP(DBLPType)
+primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset UpsertFromDBLP(DBLPType)
+primary key id
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-inverted/primary-correlated-secondary-inverted.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-inverted/primary-correlated-secondary-inverted.2.update.sqlpp
new file mode 100644
index 0000000..29a7396
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-inverted/primary-correlated-secondary-inverted.2.update.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use test;
+
+load dataset UpsertToDBLP
+using localfs
+(("path"="asterix_nc1://data/dblp-small/dblp-small-id.txt"),("format"="delimited-text"),("delimiter"=":"));
+
+load dataset UpsertFromDBLP
+using localfs
+(("path"="asterix_nc1://data/dblp-small/more-dblp-small-id.txt"),("format"="delimited-text"),("delimiter"=":"));
+
+upsert into UpsertToDBLP(
+ select value x
+ from UpsertFromDBLP x
+);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-inverted/primary-correlated-secondary-inverted.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-inverted/primary-correlated-secondary-inverted.3.ddl.sqlpp
new file mode 100644
index 0000000..61d1674
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-inverted/primary-correlated-secondary-inverted.3.ddl.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use test;
+
+create index keyword_index on UpsertToDBLP(title) type keyword;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-inverted/primary-correlated-secondary-inverted.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-inverted/primary-correlated-secondary-inverted.4.query.sqlpp
new file mode 100644
index 0000000..31e091c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-inverted/primary-correlated-secondary-inverted.4.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+USE test;
+
+SELECT VALUE o
+FROM UpsertToDBLP o
+WHERE contains(o.title, "SQL")
+ORDER BY o.id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-rtree/primary-correlated-secondary-rtree.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-rtree/primary-correlated-secondary-rtree.1.ddl.sqlpp
new file mode 100644
index 0000000..1c1e480
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-rtree/primary-correlated-secondary-rtree.1.ddl.sqlpp
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type MyRecord as closed {
+ id: int64,
+ point: point,
+ kwds: string,
+ line1: line,
+ line2: line,
+ poly1: polygon,
+ poly2: polygon,
+ rec: rectangle,
+ circle: circle
+}
+
+create dataset UpsertTo(MyRecord)
+ primary key id
+ using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+create dataset UpsertFrom(MyRecord)
+ primary key id
+ using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-rtree/primary-correlated-secondary-rtree.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-rtree/primary-correlated-secondary-rtree.2.update.sqlpp
new file mode 100644
index 0000000..b0a3bbc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-rtree/primary-correlated-secondary-rtree.2.update.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use test;
+
+load dataset UpsertTo
+using localfs
+(("path"="asterix_nc1://data/spatial/spatialData.json"),("format"="adm"));
+
+load dataset UpsertFrom
+using localfs
+(("path"="asterix_nc1://data/spatial/moreSpatialData.json"),("format"="adm"));
+
+upsert into UpsertTo(
+ from UpsertFrom x
+ select value x
+);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-rtree/primary-correlated-secondary-rtree.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-rtree/primary-correlated-secondary-rtree.3.ddl.sqlpp
new file mode 100644
index 0000000..91ed33b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-rtree/primary-correlated-secondary-rtree.3.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+create index rtree_index_point on UpsertTo(point) type rtree;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-rtree/primary-correlated-secondary-rtree.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-rtree/primary-correlated-secondary-rtree.4.query.sqlpp
new file mode 100644
index 0000000..42d4040
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/primary-correlated-secondary-rtree/primary-correlated-secondary-rtree.4.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use test;
+
+select o.id
+from UpsertTo o
+where spatial_intersect(o.point, create_polygon([4.0,1.0,4.0,4.0,12.0,4.0,12.0,1.0]));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index df0c6b5..cd0355a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -1904,65 +1904,130 @@
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit name="scan-delete-rtree-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-rtree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="scan-delete-rtree-secondary-index">
<output-dir compare="Text">scan-delete-rtree-secondary-index</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit name="scan-delete-rtree-correlated-secondary-index">
+ <output-dir compare="Text">scan-delete-rtree-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="scan-insert-btree-secondary-index-nullable">
<output-dir compare="Text">scan-insert-btree-secondary-index-nullable</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit name="scan-insert-btree-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-btree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="scan-insert-rtree-secondary-index-nullable">
<output-dir compare="Text">scan-insert-rtree-secondary-index-nullable</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit name="scan-insert-rtree-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-insert-rtree-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="scan-insert-rtree-secondary-index">
<output-dir compare="Text">scan-insert-rtree-secondary-index</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-rtree-correlated-secondary-index">
+ <output-dir compare="Text">scan-insert-rtree-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="dml">
<compilation-unit name="scan-insert-inverted-index-ngram-secondary-index">
<output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-ngram-correlated-secondary-index">
+ <output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="scan-insert-inverted-index-word-secondary-index">
<output-dir compare="Text">scan-insert-inverted-index-word-secondary-index</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-word-correlated-secondary-index">
+ <output-dir compare="Text">scan-insert-inverted-index-word-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="scan-insert-inverted-index-ngram-secondary-index-nullable">
<output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index-nullable</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-ngram-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="scan-insert-inverted-index-word-secondary-index-nullable">
<output-dir compare="Text">scan-insert-inverted-index-word-secondary-index-nullable</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-word-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-insert-inverted-index-word-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="dml">
<compilation-unit name="scan-delete-inverted-index-ngram-secondary-index">
<output-dir compare="Text">scan-delete-inverted-index-ngram-secondary-index</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-ngram-correlated-secondary-index">
+ <output-dir compare="Text">scan-delete-inverted-index-ngram-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="dml">
<compilation-unit name="scan-delete-inverted-index-word-secondary-index">
<output-dir compare="Text">scan-delete-inverted-index-word-secondary-index</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-word-correlated-secondary-index">
+ <output-dir compare="Text">scan-delete-inverted-index-word-secondary-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="scan-delete-inverted-index-ngram-secondary-index-nullable">
<output-dir compare="Text">scan-delete-inverted-index-ngram-secondary-index-nullable</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-ngram-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-inverted-index-ngram-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="dml">
<compilation-unit name="scan-delete-inverted-index-word-secondary-index-nullable">
<output-dir compare="Text">scan-delete-inverted-index-word-secondary-index-nullable</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-word-correlated-secondary-index-nullable">
+ <output-dir compare="Text">scan-delete-inverted-index-word-secondary-index-nullable</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="dml">
<compilation-unit name="load-with-index-open">
<output-dir compare="Text">load-with-index-open</output-dir>
@@ -1999,16 +2064,31 @@
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-ngram-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-delete-inverted-index-ngram-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="scan-delete-inverted-index-word-secondary-index-open">
<output-dir compare="Text">scan-delete-inverted-index-word-secondary-index-open</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="scan-delete-inverted-index-word-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-delete-inverted-index-word-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="dml">
<compilation-unit name="scan-delete-rtree-secondary-index-open">
<output-dir compare="Text">scan-delete-rtree-secondary-index-open</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit name="scan-delete-rtree-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-delete-rtree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="scan-insert-btree-secondary-index-open">
<output-dir compare="Text">scan-insert-btree-secondary-index-open</output-dir>
</compilation-unit>
@@ -2024,16 +2104,31 @@
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-ngram-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-insert-inverted-index-ngram-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="scan-insert-inverted-index-word-secondary-index-open">
<output-dir compare="Text">scan-insert-inverted-index-word-secondary-index-open</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit name="scan-insert-inverted-index-word-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-insert-inverted-index-word-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="scan-insert-rtree-secondary-index-open">
<output-dir compare="Text">scan-insert-rtree-secondary-index-open</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit name="scan-insert-rtree-correlated-secondary-index-open">
+ <output-dir compare="Text">scan-insert-rtree-secondary-index-open</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="delete-multi-statement">
<output-dir compare="Text">delete-multi-statement</output-dir>
</compilation-unit>
@@ -4116,11 +4211,6 @@
<output-dir compare="Text">scan-insert-btree-secondary-index-nullable</output-dir>
</compilation-unit>
</test-case>
- <test-case FilePath="dml">
- <compilation-unit name="scan-insert-btree-correlated-secondary-index-nullable">
- <output-dir compare="Text">scan-delete-btree-secondary-index-nullable</output-dir>
- </compilation-unit>
- </test-case>
<test-case FilePath="nested-index-dml">
<compilation-unit name="scan-insert-rtree-secondary-index-nullable">
<output-dir compare="Text">scan-insert-rtree-secondary-index-nullable</output-dir>
@@ -8607,8 +8697,8 @@
<output-dir compare="Text">insert-with-secondary-btree</output-dir>
</compilation-unit>
</test-case>
- <test-case FilePath="filters">
- <compilation-unit name="insert-with-secondary-correlated-btree">
+ <test-case FilePath="filters">
+ <compilation-unit name="insert-with-correlated-secondary-btree">
<output-dir compare="Text">insert-with-secondary-btree</output-dir>
</compilation-unit>
</test-case>
@@ -8618,16 +8708,31 @@
</compilation-unit>
</test-case>
<test-case FilePath="filters">
+ <compilation-unit name="insert-with-correlated-secondary-inverted-ngram">
+ <output-dir compare="Text">insert-with-secondary-inverted-ngram</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
<compilation-unit name="insert-with-secondary-inverted-word">
<output-dir compare="Text">insert-with-secondary-inverted-word</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="filters">
+ <compilation-unit name="insert-with-correlated-secondary-inverted-word">
+ <output-dir compare="Text">insert-with-secondary-inverted-word</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="filters">
<compilation-unit name="insert-with-secondary-rtree">
<output-dir compare="Text">insert-with-secondary-rtree</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="filters">
+ <compilation-unit name="insert-with-correlated-secondary-rtree">
+ <output-dir compare="Text">insert-with-secondary-rtree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="filters">
<compilation-unit name="nested-filterequality-predicate">
<output-dir compare="Text">nested-filter-equality-predicate</output-dir>
</compilation-unit>
@@ -8875,6 +8980,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="upsert">
+ <compilation-unit name="primary-correlated-secondary-rtree">
+ <output-dir compare="Text">primary-secondary-rtree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
<compilation-unit name="upsert-with-self-read">
<output-dir compare="Text">upsert-with-self-read</output-dir>
</compilation-unit>
@@ -8910,9 +9020,19 @@
</compilation-unit>
</test-case>
<test-case FilePath="upsert">
+ <compilation-unit name="primary-correlated-secondary-inverted">
+ <output-dir compare="Text">primary-secondary-inverted</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="upsert">
<compilation-unit name="multiple-secondaries">
<output-dir compare="Text">multiple-secondaries</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="upsert">
+ <compilation-unit name="multiple-correlated-secondaries">
+ <output-dir compare="Text">multiple-secondaries</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
</test-suite>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 7c82ca3..9d889ea 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -113,6 +113,7 @@
public static final int DATASET_ID_EXHAUSTED = 1040;
public static final int INDEX_ILLEGAL_ENFORCED_NON_OPTIONAL = 1041;
public static final int INDEX_ILLEGAL_NON_ENFORCED_TYPED = 1042;
+ public static final int INDEX_RTREE_MULTIPLE_FIELDS_NOT_ALLOWED = 1043;
// Feed errors
public static final int DATAFLOW_ILLEGAL_STATE = 3001;
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index facf1a9..9d4db56 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -99,6 +99,7 @@
1040 = Dataset id space is exhausted
1041 = Cannot create enforced index on \"%1$s\" field with non-optional type
1042 = Cannot create non-enforced typed index of this kind: %1$s
+1043 = Cannot use %1$s fields as key for the R-tree index. There can be only one field as a key for the R-tree index.
# Feed Errors
3001 = Illegal state.
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.1.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.1.script.aql
new file mode 100644
index 0000000..7d441cd
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.1.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+create_and_start.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.10.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.10.script.aql
new file mode 100644
index 0000000..40df6fb
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.10.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+stop_and_delete.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.2.ddl.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.2.ddl.aql
new file mode 100644
index 0000000..be29328
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.2.ddl.aql
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : primary_plus_correlated_ngram_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : June 21 2017
+ */
+
+drop dataverse recovery if exists;
+create dataverse recovery;
+use dataverse recovery;
+
+/* For raw Fragile data */
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float,
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+};
+
+/* For cleaned Fragile data */
+create type FragileType as closed {
+ row_id: int32,
+ sid: int32,
+ date: date,
+ day: int32,
+ time: time,
+ bpm: int32,
+ RR: float,
+
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+};
+
+/* Create dataset for loading raw Fragile data */
+create dataset Fragile_raw (FragileTypeRaw)
+primary key row_id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+/* Create dataset for cleaned Fragile data */
+create dataset Fragile (FragileType)
+primary key row_id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.3.update.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.3.update.aql
new file mode 100644
index 0000000..5558005
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.3.update.aql
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse recovery;
+
+load dataset Fragile_raw using localfs
+(("path"="asterix_nc1://../../../../../../asterix-app/data/csv/fragile_02.adm"),("format"="adm")) pre-sorted;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.4.txneu.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.4.txneu.aql
new file mode 100644
index 0000000..1ebb2ef
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.4.txneu.aql
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse recovery;
+
+/* Load Fragile data from raw dataset into cleaned dataset */
+insert into dataset Fragile (
+ for $t in dataset Fragile_raw
+ return {
+ "row_id": $t.row_id % 28000,
+ "sid": $t.sid,
+ "date": date($t.date),
+ "day": $t.day,
+ "time": parse-time($t.time, "h:m:s"),
+ "bpm": $t.bpm,
+ "RR": $t.RR,
+ "text": $t.text,
+ "location": $t.location,
+ "text2": $t.text2
+ }
+);
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.5.ddl.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.5.ddl.aql
new file mode 100644
index 0000000..983ea7c
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.5.ddl.aql
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse recovery;
+
+/* Create default secondary index on dataset clean Fragile */
+create index cfText2Ix on Fragile(text2) type ngram(3);
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.6.txnqbc.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.6.txnqbc.aql
new file mode 100644
index 0000000..3c30ea8
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.6.txnqbc.aql
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile
+where contains($x.text2, "location") return $x);
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.7.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.7.script.aql
new file mode 100644
index 0000000..4583455
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.7.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.8.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.8.script.aql
new file mode 100644
index 0000000..7087cd3
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.8.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+stop_and_start.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.9.txnqar.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.9.txnqar.aql
new file mode 100644
index 0000000..3c30ea8
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_ngram_index/primary_plus_correlated_ngram_index.9.txnqar.aql
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile
+where contains($x.text2, "location") return $x);
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.1.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.1.script.aql
new file mode 100644
index 0000000..7d441cd
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.1.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+create_and_start.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.10.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.10.script.aql
new file mode 100644
index 0000000..40df6fb
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.10.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+stop_and_delete.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.2.ddl.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.2.ddl.aql
new file mode 100644
index 0000000..9443205
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.2.ddl.aql
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : primary_plus_keyword_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : June 20 2017
+ */
+
+drop dataverse recovery if exists;
+create dataverse recovery;
+use dataverse recovery;
+
+/* For raw Fragile data */
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float,
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+
+};
+
+/* For cleaned Fragile data */
+create type FragileType as closed {
+ row_id: int32,
+ sid: int32,
+ date: date,
+ day: int32,
+ time: time,
+ bpm: int32,
+ RR: float,
+
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+};
+
+/* Create dataset for loading raw Fragile data */
+create dataset Fragile_raw (FragileTypeRaw)
+primary key row_id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+/* Create dataset for cleaned Fragile data */
+create dataset Fragile (FragileType)
+primary key row_id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.3.update.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.3.update.aql
new file mode 100644
index 0000000..f48ccd5
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.3.update.aql
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse recovery;
+
+
+load dataset Fragile_raw using localfs
+(("path"="asterix_nc1://../../../../../../asterix-app/data/csv/fragile_02.adm"),("format"="adm")) pre-sorted;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.4.txneu.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.4.txneu.aql
new file mode 100644
index 0000000..ba93396
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.4.txneu.aql
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+use dataverse recovery;
+
+/* Load Fragile data from raw dataset into cleaned dataset */
+insert into dataset Fragile (
+ for $t in dataset Fragile_raw
+ return {
+ "row_id": $t.row_id % 28000,
+ "sid": $t.sid,
+ "date": date($t.date),
+ "day": $t.day,
+ "time": parse-time($t.time, "h:m:s"),
+ "bpm": $t.bpm,
+ "RR": $t.RR,
+ "text": $t.text,
+ "location": $t.location,
+ "text2": $t.text2
+ }
+);
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.5.ddl.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.5.ddl.aql
new file mode 100644
index 0000000..b8c4cfc
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.5.ddl.aql
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse recovery;
+
+/* Create rtree secondary index on dataset clean Fragile */
+create index cfLocation on Fragile(location) type rtree;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.6.txnqbc.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.6.txnqbc.aql
new file mode 100644
index 0000000..ddd0e61
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.6.txnqbc.aql
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse recovery;
+
+count (for $x in dataset Fragile where
+spatial-intersect($x.location, create-polygon([0.0,0.0, 2.0,2.0, 0.0,2.0, 2.0,0.0]))
+return $x);
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.7.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.7.script.aql
new file mode 100644
index 0000000..4583455
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.7.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.8.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.8.script.aql
new file mode 100644
index 0000000..7087cd3
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.8.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+stop_and_start.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.9.txnqar.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.9.txnqar.aql
new file mode 100644
index 0000000..ddd0e61
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_correlated_rtree_index/primary_plus_correlated_rtree_index.9.txnqar.aql
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse recovery;
+
+count (for $x in dataset Fragile where
+spatial-intersect($x.location, create-polygon([0.0,0.0, 2.0,2.0, 0.0,2.0, 2.0,0.0]))
+return $x);
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.1.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.1.script.aql
new file mode 100644
index 0000000..7d441cd
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.1.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+create_and_start.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.10.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.10.script.aql
new file mode 100644
index 0000000..40df6fb
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.10.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+stop_and_delete.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.2.ddl.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.2.ddl.aql
new file mode 100644
index 0000000..b1f61e2
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.2.ddl.aql
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : primary_plus_keyword_correlated_secondary_index.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : June 21 2017
+ */
+
+drop dataverse recovery if exists;
+create dataverse recovery;
+use dataverse recovery;
+
+/* For raw Fragile data */
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float,
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+
+};
+
+/* For cleaned Fragile data */
+create type FragileType as closed {
+ row_id: int32,
+ sid: int32,
+ date: date,
+ day: int32,
+ time: time,
+ bpm: int32,
+ RR: float,
+
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+};
+
+/* Create dataset for loading raw Fragile data */
+create dataset Fragile_raw (FragileTypeRaw)
+primary key row_id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+/* Create dataset for cleaned Fragile data */
+create dataset Fragile (FragileType)
+primary key row_id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.3.update.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.3.update.aql
new file mode 100644
index 0000000..f48ccd5
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.3.update.aql
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse recovery;
+
+
+load dataset Fragile_raw using localfs
+(("path"="asterix_nc1://../../../../../../asterix-app/data/csv/fragile_02.adm"),("format"="adm")) pre-sorted;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.4.txneu.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.4.txneu.aql
new file mode 100644
index 0000000..1ebb2ef
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.4.txneu.aql
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse recovery;
+
+/* Load Fragile data from raw dataset into cleaned dataset */
+insert into dataset Fragile (
+ for $t in dataset Fragile_raw
+ return {
+ "row_id": $t.row_id % 28000,
+ "sid": $t.sid,
+ "date": date($t.date),
+ "day": $t.day,
+ "time": parse-time($t.time, "h:m:s"),
+ "bpm": $t.bpm,
+ "RR": $t.RR,
+ "text": $t.text,
+ "location": $t.location,
+ "text2": $t.text2
+ }
+);
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.5.ddl.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.5.ddl.aql
new file mode 100644
index 0000000..bd9bace
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.5.ddl.aql
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse recovery;
+
+/* Create keyword secondary index on dataset clean Fragile */
+create index cfText on Fragile(text) type keyword;
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.6.txnqbc.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.6.txnqbc.aql
new file mode 100644
index 0000000..2b0eb36
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.6.txnqbc.aql
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse recovery;
+
+for $x in dataset Fragile
+where word-tokens($x.text) ~= word-tokens(" 1 20130417 1")
+return $x.text;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.7.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.7.script.aql
new file mode 100644
index 0000000..4583455
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.7.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.8.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.8.script.aql
new file mode 100644
index 0000000..7087cd3
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.8.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+stop_and_start.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.9.txnqar.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.9.txnqar.aql
new file mode 100644
index 0000000..2b0eb36
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_correlated_secondary_index/primary_plus_keyword_correlated_secondary_index.9.txnqar.aql
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse recovery;
+
+for $x in dataset Fragile
+where word-tokens($x.text) ~= word-tokens(" 1 20130417 1")
+return $x.text;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.1.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.1.script.aql
new file mode 100644
index 0000000..7d441cd
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.1.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+create_and_start.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.10.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.10.script.aql
new file mode 100644
index 0000000..40df6fb
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.10.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+stop_and_delete.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.2.ddl.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.2.ddl.aql
new file mode 100644
index 0000000..69a6c9d
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.2.ddl.aql
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : primary_plus_multiple_correlated_secondary_indices.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ * Date : June 21 2017
+ */
+
+drop dataverse recovery if exists;
+create dataverse recovery;
+use dataverse recovery;
+
+/* For raw Fragile data */
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float,
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+
+};
+
+/* For cleaned Fragile data */
+create type FragileType as closed {
+ row_id: int32,
+ sid: int32,
+ date: date,
+ day: int32,
+ time: time,
+ bpm: int32,
+ RR: float,
+
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+};
+
+/* Create dataset for loading raw Fragile data */
+create dataset Fragile_raw (FragileTypeRaw)
+primary key row_id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+/* Create dataset for cleaned Fragile data */
+create dataset Fragile (FragileType)
+primary key row_id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.3.update.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.3.update.aql
new file mode 100644
index 0000000..f48ccd5
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.3.update.aql
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse recovery;
+
+
+load dataset Fragile_raw using localfs
+(("path"="asterix_nc1://../../../../../../asterix-app/data/csv/fragile_02.adm"),("format"="adm")) pre-sorted;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.4.txneu.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.4.txneu.aql
new file mode 100644
index 0000000..1ebb2ef
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.4.txneu.aql
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse recovery;
+
+/* Load Fragile data from raw dataset into cleaned dataset */
+insert into dataset Fragile (
+ for $t in dataset Fragile_raw
+ return {
+ "row_id": $t.row_id % 28000,
+ "sid": $t.sid,
+ "date": date($t.date),
+ "day": $t.day,
+ "time": parse-time($t.time, "h:m:s"),
+ "bpm": $t.bpm,
+ "RR": $t.RR,
+ "text": $t.text,
+ "location": $t.location,
+ "text2": $t.text2
+ }
+);
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.5.ddl.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.5.ddl.aql
new file mode 100644
index 0000000..3a35fd4
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.5.ddl.aql
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse recovery;
+
+create index cfLocation on Fragile(location) type rtree;
+
+create index cfText on Fragile(text) type keyword;
+
+create index cfSidIdx on Fragile(sid);
+
+create index cfText2Ix on Fragile(text2) type ngram(3);
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.6.txnqbc.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.6.txnqbc.aql
new file mode 100644
index 0000000..3c30ea8
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.6.txnqbc.aql
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile
+where contains($x.text2, "location") return $x);
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.7.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.7.script.aql
new file mode 100644
index 0000000..4583455
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.7.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.8.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.8.script.aql
new file mode 100644
index 0000000..7087cd3
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.8.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+stop_and_start.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.9.txnqar.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.9.txnqar.aql
new file mode 100644
index 0000000..3c30ea8
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/primary_plus_multiple_correlated_secondary_indices.9.txnqar.aql
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile
+where contains($x.text2, "location") return $x);
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_ngram_index/create_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_ngram_index/create_and_start.sh
new file mode 100755
index 0000000..789914b
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_ngram_index/create_and_start.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix create -n asterix -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_ngram_index/kill_cc_and_nc.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_ngram_index/kill_cc_and_nc.sh
new file mode 100755
index 0000000..4b876be
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_ngram_index/kill_cc_and_nc.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+ps -ef | awk '/java.*org\.apache\.hyracks\.control\.[cn]c\.[CN]CDriver/ {print $2}' | xargs -n 1 kill -9
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_ngram_index/stop_and_delete.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_ngram_index/stop_and_delete.sh
new file mode 100755
index 0000000..eb1c01e
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_ngram_index/stop_and_delete.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix delete -n asterix;
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_ngram_index/stop_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_ngram_index/stop_and_start.sh
new file mode 100755
index 0000000..2fb80ce
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_ngram_index/stop_and_start.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix start -n asterix;
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_rtree_index/create_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_rtree_index/create_and_start.sh
new file mode 100755
index 0000000..789914b
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_rtree_index/create_and_start.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix create -n asterix -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_rtree_index/kill_cc_and_nc.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_rtree_index/kill_cc_and_nc.sh
new file mode 100755
index 0000000..4b876be
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_rtree_index/kill_cc_and_nc.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+ps -ef | awk '/java.*org\.apache\.hyracks\.control\.[cn]c\.[CN]CDriver/ {print $2}' | xargs -n 1 kill -9
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_rtree_index/stop_and_delete.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_rtree_index/stop_and_delete.sh
new file mode 100755
index 0000000..eb1c01e
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_rtree_index/stop_and_delete.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix delete -n asterix;
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_rtree_index/stop_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_rtree_index/stop_and_start.sh
new file mode 100755
index 0000000..2fb80ce
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_correlated_rtree_index/stop_and_start.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix start -n asterix;
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_keyword_correlated_secondary_index/create_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_keyword_correlated_secondary_index/create_and_start.sh
new file mode 100755
index 0000000..789914b
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_keyword_correlated_secondary_index/create_and_start.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix create -n asterix -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_keyword_correlated_secondary_index/kill_cc_and_nc.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_keyword_correlated_secondary_index/kill_cc_and_nc.sh
new file mode 100755
index 0000000..4b876be
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_keyword_correlated_secondary_index/kill_cc_and_nc.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+ps -ef | awk '/java.*org\.apache\.hyracks\.control\.[cn]c\.[CN]CDriver/ {print $2}' | xargs -n 1 kill -9
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_keyword_correlated_secondary_index/stop_and_delete.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_keyword_correlated_secondary_index/stop_and_delete.sh
new file mode 100755
index 0000000..eb1c01e
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_keyword_correlated_secondary_index/stop_and_delete.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix delete -n asterix;
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_keyword_correlated_secondary_index/stop_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_keyword_correlated_secondary_index/stop_and_start.sh
new file mode 100755
index 0000000..2fb80ce
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_keyword_correlated_secondary_index/stop_and_start.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix start -n asterix;
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/create_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/create_and_start.sh
new file mode 100755
index 0000000..789914b
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/create_and_start.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix create -n asterix -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/kill_cc_and_nc.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/kill_cc_and_nc.sh
new file mode 100755
index 0000000..4b876be
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/kill_cc_and_nc.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+ps -ef | awk '/java.*org\.apache\.hyracks\.control\.[cn]c\.[CN]CDriver/ {print $2}' | xargs -n 1 kill -9
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/stop_and_delete.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/stop_and_delete.sh
new file mode 100755
index 0000000..eb1c01e
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/stop_and_delete.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix delete -n asterix;
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/stop_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/stop_and_start.sh
new file mode 100755
index 0000000..2fb80ce
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_multiple_correlated_secondary_indices/stop_and_start.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix start -n asterix;
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
index 92ae1bd..bd68075 100644
--- a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
@@ -68,6 +68,12 @@
</test-case>
<test-case FilePath="recover_after_abort">
+ <compilation-unit name="primary_plus_correlated_rtree_index">
+ <output-dir compare="Text">primary_plus_rtree_index</output-dir>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="recover_after_abort">
<compilation-unit name="primary_plus_rtree_index_insert_and_delete">
<output-dir compare="Text">primary_plus_rtree_index_insert_and_delete</output-dir>
</compilation-unit>
@@ -80,18 +86,36 @@
</test-case>
<test-case FilePath="recover_after_abort">
+ <compilation-unit name="primary_plus_keyword_correlated_secondary_index">
+ <output-dir compare="Text">primary_plus_keyword_secondary_index</output-dir>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="recover_after_abort">
<compilation-unit name="primary_plus_ngram_index">
<output-dir compare="Text">primary_plus_ngram_index</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="recover_after_abort">
+ <compilation-unit name="primary_plus_correlated_ngram_index">
+ <output-dir compare="Text">primary_plus_ngram_index</output-dir>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="recover_after_abort">
<compilation-unit name="primary_plus_multiple_secondary_indices">
<output-dir compare="Text">primary_plus_multiple_secondary_indices</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="recover_after_abort">
+ <compilation-unit name="primary_plus_multiple_correlated_secondary_indices">
+ <output-dir compare="Text">primary_plus_multiple_secondary_indices</output-dir>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="recover_after_abort">
<compilation-unit name="primary_index_only_filtered">
<output-dir compare="Text">primary_index_only_filtered</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 3b70ea9..68a2448 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -37,6 +37,7 @@
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.transactions.JobId;
import org.apache.asterix.common.utils.StoragePathUtil;
+import org.apache.asterix.dataflow.data.nontagged.MissingWriterFactory;
import org.apache.asterix.external.adapter.factory.LookupAdapterFactory;
import org.apache.asterix.external.api.IAdapterFactory;
import org.apache.asterix.external.api.IDataSourceAdapter;
@@ -1534,7 +1535,7 @@
}
tokenizerOp = new BinaryTokenizerOperatorDescriptor(spec, tokenKeyPairRecDesc, tokenizerFactory, docField,
- keyFields, isPartitioned, true);
+ keyFields, isPartitioned, true, false, MissingWriterFactory.INSTANCE);
return new Pair<>(tokenizerOp, splitsAndConstraint.second);
} catch (Exception e) {
throw new AlgebricksException(e);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
index 7daa150..96fdf27 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
@@ -99,8 +99,9 @@
getTaggedSecondaryComparatorFactories(secondaryComparatorFactories), taggedSecondaryRecDesc);
// Create secondary BTree bulk load op.
- LSMSecondaryIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec,
- metadataProvider, taggedSecondaryRecDesc, getNumSecondaryKeys(), numPrimaryKeys, false);
+ LSMSecondaryIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp =
+ createTreeIndexBulkLoadOp(spec, metadataProvider, taggedSecondaryRecDesc,
+ createFieldPermutationForBulkLoadOp(), getNumSecondaryKeys(), numPrimaryKeys, false);
AlgebricksMetaOperatorDescriptor metaOp =
new AlgebricksMetaOperatorDescriptor(spec, 1, 0, new IPushRuntimeFactory[] { new SinkRuntimeFactory() },
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java
new file mode 100644
index 0000000..15f8a23
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.utils;
+
+import org.apache.asterix.common.config.DatasetConfig.IndexType;
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.dataflow.data.nontagged.MissingWriterFactory;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.asterix.om.utils.RecordUtil;
+import org.apache.asterix.runtime.formats.FormatUtils;
+import org.apache.asterix.runtime.utils.RuntimeUtils;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.core.jobgen.impl.ConnectorPolicyAssignmentPolicy;
+import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+import org.apache.hyracks.algebricks.data.ISerializerDeserializerProvider;
+import org.apache.hyracks.algebricks.data.ITypeTraitProvider;
+import org.apache.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.algebricks.runtime.operators.base.SinkRuntimeFactory;
+import org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
+import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import org.apache.hyracks.data.std.primitive.ShortPointable;
+import org.apache.hyracks.dataflow.common.data.marshalling.ShortSerializerDeserializer;
+import org.apache.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
+import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
+import org.apache.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
+import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.BinaryTokenizerOperatorDescriptor;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
+
+public class SecondaryCorrelatedInvertedIndexOperationsHelper extends SecondaryCorrelatedTreeIndexOperationsHelper {
+
+ private IAType secondaryKeyType;
+ private ITypeTraits[] invListsTypeTraits;
+ private IBinaryComparatorFactory[] tokenComparatorFactories;
+ private ITypeTraits[] tokenTypeTraits;
+ private IBinaryTokenizerFactory tokenizerFactory;
+ // For tokenization, sorting and loading. Represents <token, primary keys>.
+ private int numTokenKeyPairFields;
+ private IBinaryComparatorFactory[] tokenKeyPairComparatorFactories;
+ private RecordDescriptor tokenKeyPairRecDesc;
+ private boolean isPartitioned;
+ private int[] invertedIndexFields;
+ private int[] invertedIndexFieldsForNonBulkLoadOps;
+ private int[] secondaryFilterFieldsForNonBulkLoadOps;
+
+ protected SecondaryCorrelatedInvertedIndexOperationsHelper(Dataset dataset, Index index,
+ PhysicalOptimizationConfig physOptConf, MetadataProvider metadataProvider) throws AlgebricksException {
+ super(dataset, index, physOptConf, metadataProvider);
+ }
+
+ @Override
+ @SuppressWarnings("rawtypes")
+ protected void setSecondaryRecDescAndComparators() throws AlgebricksException {
+ int numSecondaryKeys = index.getKeyFieldNames().size();
+ IndexType indexType = index.getIndexType();
+ boolean isOverridingKeyFieldTypes = index.isOverridingKeyFieldTypes();
+ // Sanity checks.
+ if (numPrimaryKeys > 1) {
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_INDEX_FOR_DATASET_WITH_COMPOSITE_PRIMARY_INDEX,
+ indexType, RecordUtil.toFullyQualifiedName(dataset.getDataverseName(), dataset.getDatasetName()));
+ }
+ if (numSecondaryKeys > 1) {
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_INDEX_NUM_OF_FIELD, numSecondaryKeys,
+ indexType, 1);
+ }
+ if (indexType == IndexType.LENGTH_PARTITIONED_WORD_INVIX
+ || indexType == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX) {
+ isPartitioned = true;
+ } else {
+ isPartitioned = false;
+ }
+ // Prepare record descriptor used in the assign op, and the optional
+ // select op.
+ secondaryFieldAccessEvalFactories = new IScalarEvaluatorFactory[numSecondaryKeys + numFilterFields];
+ ISerializerDeserializer[] secondaryRecFields =
+ new ISerializerDeserializer[numPrimaryKeys + numSecondaryKeys + numFilterFields];
+ ISerializerDeserializer[] enforcedRecFields = new ISerializerDeserializer[1 + numPrimaryKeys + numFilterFields];
+ secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys];
+ ITypeTraits[] enforcedTypeTraits = new ITypeTraits[1 + numPrimaryKeys];
+ ISerializerDeserializerProvider serdeProvider = FormatUtils.getDefaultFormat().getSerdeProvider();
+ ITypeTraitProvider typeTraitProvider = FormatUtils.getDefaultFormat().getTypeTraitProvider();
+ int recordColumn = NUM_TAG_FIELDS + numPrimaryKeys;
+ if (numSecondaryKeys > 0) {
+ secondaryFieldAccessEvalFactories[0] = FormatUtils.getDefaultFormat().getFieldAccessEvaluatorFactory(
+ isOverridingKeyFieldTypes ? enforcedItemType : itemType, index.getKeyFieldNames().get(0),
+ recordColumn);
+ Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0),
+ index.getKeyFieldNames().get(0), itemType);
+ secondaryKeyType = keyTypePair.first;
+ anySecondaryKeyIsNullable = anySecondaryKeyIsNullable || keyTypePair.second;
+ ISerializerDeserializer keySerde = serdeProvider.getSerializerDeserializer(secondaryKeyType);
+ secondaryRecFields[0] = keySerde;
+ secondaryTypeTraits[0] = typeTraitProvider.getTypeTrait(secondaryKeyType);
+ }
+ if (numFilterFields > 0) {
+ secondaryFieldAccessEvalFactories[numSecondaryKeys] = FormatUtils.getDefaultFormat()
+ .getFieldAccessEvaluatorFactory(itemType, filterFieldName, recordColumn);
+ Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
+ IAType type = keyTypePair.first;
+ ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);
+ secondaryRecFields[numPrimaryKeys + numSecondaryKeys] = serde;
+ }
+ secondaryRecDesc = new RecordDescriptor(secondaryRecFields);
+ // Comparators and type traits for tokens.
+ int numTokenFields = (!isPartitioned) ? numSecondaryKeys : numSecondaryKeys + 1;
+ tokenComparatorFactories = new IBinaryComparatorFactory[numTokenFields];
+ tokenTypeTraits = new ITypeTraits[numTokenFields];
+ tokenComparatorFactories[0] = NonTaggedFormatUtil.getTokenBinaryComparatorFactory(secondaryKeyType);
+ tokenTypeTraits[0] = NonTaggedFormatUtil.getTokenTypeTrait(secondaryKeyType);
+ if (isPartitioned) {
+ // The partitioning field is hardcoded to be a short *without* an Asterix type tag.
+ tokenComparatorFactories[1] = PointableBinaryComparatorFactory.of(ShortPointable.FACTORY);
+ tokenTypeTraits[1] = ShortPointable.TYPE_TRAITS;
+ }
+ // Set tokenizer factory.
+ // TODO: We might want to expose the hashing option at the AQL level,
+ // and add the choice to the index metadata.
+ tokenizerFactory = NonTaggedFormatUtil.getBinaryTokenizerFactory(secondaryKeyType.getTypeTag(), indexType,
+ index.getGramLength());
+ // Type traits for inverted-list elements. Inverted lists contain
+ // primary keys.
+ invListsTypeTraits = new ITypeTraits[numPrimaryKeys];
+ if (numPrimaryKeys > 0) {
+ invListsTypeTraits[0] = primaryRecDesc.getTypeTraits()[0];
+ enforcedRecFields[0] = primaryRecDesc.getFields()[0];
+ enforcedTypeTraits[0] = primaryRecDesc.getTypeTraits()[0];
+ }
+ enforcedRecFields[numPrimaryKeys] = serdeProvider.getSerializerDeserializer(itemType);
+ enforcedRecDesc = new RecordDescriptor(enforcedRecFields, enforcedTypeTraits);
+ // For tokenization, sorting and loading.
+ // One token (+ optional partitioning field) + primary keys.
+ numTokenKeyPairFields = (!isPartitioned) ? 1 + numPrimaryKeys : 2 + numPrimaryKeys;
+ ISerializerDeserializer[] tokenKeyPairFields =
+ new ISerializerDeserializer[numTokenKeyPairFields + numFilterFields];
+ ITypeTraits[] tokenKeyPairTypeTraits = new ITypeTraits[numTokenKeyPairFields];
+ tokenKeyPairComparatorFactories = new IBinaryComparatorFactory[numTokenKeyPairFields];
+ tokenKeyPairFields[0] = serdeProvider.getSerializerDeserializer(secondaryKeyType);
+ tokenKeyPairTypeTraits[0] = tokenTypeTraits[0];
+ tokenKeyPairComparatorFactories[0] = NonTaggedFormatUtil.getTokenBinaryComparatorFactory(secondaryKeyType);
+ int pkOff = 1;
+ if (isPartitioned) {
+ tokenKeyPairFields[1] = ShortSerializerDeserializer.INSTANCE;
+ tokenKeyPairTypeTraits[1] = tokenTypeTraits[1];
+ tokenKeyPairComparatorFactories[1] = PointableBinaryComparatorFactory.of(ShortPointable.FACTORY);
+ pkOff = 2;
+ }
+ if (numPrimaryKeys > 0) {
+ tokenKeyPairFields[pkOff] = primaryRecDesc.getFields()[0];
+ tokenKeyPairTypeTraits[pkOff] = primaryRecDesc.getTypeTraits()[0];
+ tokenKeyPairComparatorFactories[pkOff] = primaryComparatorFactories[0];
+ }
+ if (numFilterFields > 0) {
+ tokenKeyPairFields[numPrimaryKeys + pkOff] = secondaryRecFields[numPrimaryKeys + numSecondaryKeys];
+ }
+ tokenKeyPairRecDesc = new RecordDescriptor(tokenKeyPairFields, tokenKeyPairTypeTraits);
+ if (filterFieldName != null) {
+ invertedIndexFields = new int[numTokenKeyPairFields];
+ for (int i = 0; i < invertedIndexFields.length; i++) {
+ invertedIndexFields[i] = i;
+ }
+ secondaryFilterFieldsForNonBulkLoadOps = new int[numFilterFields];
+ secondaryFilterFieldsForNonBulkLoadOps[0] = numSecondaryKeys + numPrimaryKeys;
+ invertedIndexFieldsForNonBulkLoadOps = new int[numSecondaryKeys + numPrimaryKeys];
+ for (int i = 0; i < invertedIndexFieldsForNonBulkLoadOps.length; i++) {
+ invertedIndexFieldsForNonBulkLoadOps[i] = i;
+ }
+ }
+ }
+
+ @Override
+ protected int getNumSecondaryKeys() {
+ return numTokenKeyPairFields - numPrimaryKeys;
+ }
+
+ @Override
+ public JobSpecification buildLoadingJobSpec() throws AlgebricksException {
+ JobSpecification spec = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext());
+ JobId jobId = IndexUtil.bindJobEventListener(spec, metadataProvider);
+
+ // Create dummy key provider for feeding the primary index scan.
+ IOperatorDescriptor keyProviderOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset, metadataProvider);
+
+ // Create primary index scan op.
+ IOperatorDescriptor primaryScanOp = createPrimaryIndexScanDiskComponentsOp(spec, metadataProvider,
+ getTaggedRecordDescriptor(dataset.getPrimaryRecordDescriptor(metadataProvider)), jobId);
+
+ IOperatorDescriptor sourceOp = primaryScanOp;
+ boolean isOverridingKeyFieldTypes = index.isOverridingKeyFieldTypes();
+ int numSecondaryKeys = index.getKeyFieldNames().size();
+ if (isOverridingKeyFieldTypes && !enforcedItemType.equals(itemType)) {
+ sourceOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced());
+ spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0);
+ }
+
+ RecordDescriptor taggedSecondaryRecDesc = getTaggedRecordDescriptor(secondaryRecDesc);
+
+ RecordDescriptor taggedTokenKeyPairRecDesc = getTaggedRecordDescriptor(tokenKeyPairRecDesc);
+
+ AlgebricksMetaOperatorDescriptor asterixAssignOp =
+ createAssignOp(spec, numSecondaryKeys, taggedSecondaryRecDesc);
+
+ // Generate compensate tuples for upsert
+ IOperatorDescriptor processorOp =
+ createTupleProcessorOp(spec, taggedSecondaryRecDesc, numSecondaryKeys, numPrimaryKeys, true);
+
+ // Create a tokenizer op.
+ AbstractOperatorDescriptor tokenizerOp = createTokenizerOp(spec);
+
+ // Create a sort op
+ ExternalSortOperatorDescriptor sortOp = createSortOp(spec,
+ getTaggedSecondaryComparatorFactories(tokenKeyPairComparatorFactories), taggedTokenKeyPairRecDesc);
+
+ // Create secondary inverted index bulk load op.
+ AbstractSingleActivityOperatorDescriptor invIndexBulkLoadOp =
+ createTreeIndexBulkLoadOp(spec, metadataProvider, taggedTokenKeyPairRecDesc,
+ createFieldPermutationForBulkLoadOp(), getNumSecondaryKeys(), numPrimaryKeys, true);
+
+ AlgebricksMetaOperatorDescriptor metaOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
+ new IPushRuntimeFactory[] { new SinkRuntimeFactory() }, new RecordDescriptor[] {});
+ // Connect the operators.
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, asterixAssignOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, processorOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), processorOp, 0, tokenizerOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), tokenizerOp, 0, sortOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, invIndexBulkLoadOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), invIndexBulkLoadOp, 0, metaOp, 0);
+ spec.addRoot(metaOp);
+ spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+ return spec;
+ }
+
+ private AbstractOperatorDescriptor createTokenizerOp(JobSpecification spec) throws AlgebricksException {
+ int docField = NUM_TAG_FIELDS;
+ int numSecondaryKeys = index.getKeyFieldNames().size();
+ int[] keyFields = new int[NUM_TAG_FIELDS + numPrimaryKeys + numFilterFields];
+ // set tag fields
+ for (int i = 0; i < NUM_TAG_FIELDS; i++) {
+ keyFields[i] = i;
+ }
+ // set primary key + filter fields
+ for (int i = NUM_TAG_FIELDS; i < keyFields.length; i++) {
+ keyFields[i] = i + numSecondaryKeys;
+ }
+ BinaryTokenizerOperatorDescriptor tokenizerOp = new BinaryTokenizerOperatorDescriptor(spec,
+ getTaggedRecordDescriptor(tokenKeyPairRecDesc), tokenizerFactory, docField, keyFields, isPartitioned,
+ false, true, MissingWriterFactory.INSTANCE);
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, tokenizerOp,
+ primaryPartitionConstraint);
+ return tokenizerOp;
+ }
+
+ @Override
+ protected ExternalSortOperatorDescriptor createSortOp(JobSpecification spec,
+ IBinaryComparatorFactory[] taggedSecondaryComparatorFactories, RecordDescriptor taggedSecondaryRecDesc) {
+ /**
+ * after tokenization, the field layout becomes
+ * [token, num?, tag, primary key, filter value]
+ * we need to sort on
+ * [tag, token, num?, primary key]
+ */
+ int[] taggedSortFields = new int[taggedSecondaryComparatorFactories.length];
+ int numSecondaryKeys = getNumSecondaryKeys();
+ int idx = 0;
+ // set component pos fields
+ taggedSortFields[idx++] = numSecondaryKeys;
+ // set secondary keys
+ for (int i = 0; i < numSecondaryKeys; i++) {
+ taggedSortFields[idx++] = i;
+ }
+ // set primary keys
+ for (int i = 0; i < numPrimaryKeys; i++) {
+ taggedSortFields[idx++] = i + numSecondaryKeys + NUM_TAG_FIELDS;
+ }
+ ExternalSortOperatorDescriptor sortOp =
+ new ExternalSortOperatorDescriptor(spec, physOptConf.getMaxFramesExternalSort(), taggedSortFields,
+ taggedSecondaryComparatorFactories, taggedSecondaryRecDesc);
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, sortOp, primaryPartitionConstraint);
+ return sortOp;
+ }
+
+ @Override
+ protected int[] createFieldPermutationForBulkLoadOp() {
+ /**
+ * after tokenization, the field layout becomes
+ * [token, num?, tag, primary key, filter value]
+ * we need to restore it back to
+ * [tag, token, num?, primary key, filter value]
+ */
+ int[] fieldPermutation = new int[NUM_TAG_FIELDS + numTokenKeyPairFields + numFilterFields];
+ int numSecondaryKeys = getNumSecondaryKeys();
+ int idx = 0;
+ // set tag fields
+ for (int i = 0; i < NUM_TAG_FIELDS; i++) {
+ fieldPermutation[idx++] = i + numSecondaryKeys;
+ }
+ // set secondary keys
+ for (int i = 0; i < numSecondaryKeys; i++) {
+ fieldPermutation[idx++] = i;
+ }
+ // set primary key + filter
+ for (int i = 0; i < numPrimaryKeys + numFilterFields; i++) {
+ fieldPermutation[idx++] = i + NUM_TAG_FIELDS + numSecondaryKeys;
+ }
+ return fieldPermutation;
+ }
+
+}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java
new file mode 100644
index 0000000..8fd8a7a
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.utils;
+
+import java.util.List;
+
+import org.apache.asterix.common.config.DatasetConfig.DatasetType;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.formats.nontagged.BinaryComparatorFactoryProvider;
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.formats.nontagged.TypeTraitProvider;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.asterix.runtime.operators.LSMSecondaryIndexBulkLoadOperatorDescriptor;
+import org.apache.asterix.runtime.utils.RuntimeUtils;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.core.jobgen.impl.ConnectorPolicyAssignmentPolicy;
+import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+import org.apache.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import org.apache.hyracks.algebricks.runtime.operators.base.SinkRuntimeFactory;
+import org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
+import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
+import org.apache.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+
+@SuppressWarnings("rawtypes")
+public class SecondaryCorrelatedRTreeOperationsHelper extends SecondaryCorrelatedTreeIndexOperationsHelper {
+
+ protected IPrimitiveValueProviderFactory[] valueProviderFactories;
+ protected int numNestedSecondaryKeyFields;
+ protected ATypeTag keyType;
+ protected int[] primaryKeyFields;
+ protected int[] rtreeFields;
+ protected boolean isPointMBR;
+ protected RecordDescriptor secondaryRecDescForPointMBR = null;
+
+ protected SecondaryCorrelatedRTreeOperationsHelper(Dataset dataset, Index index,
+ PhysicalOptimizationConfig physOptConf, MetadataProvider metadataProvider) throws AlgebricksException {
+ super(dataset, index, physOptConf, metadataProvider);
+ }
+
+ @Override
+ protected int getNumSecondaryKeys() {
+ return numNestedSecondaryKeyFields;
+ }
+
+ @Override
+ protected void setSecondaryRecDescAndComparators() throws AlgebricksException {
+ List<List<String>> secondaryKeyFields = index.getKeyFieldNames();
+ int numSecondaryKeys = secondaryKeyFields.size();
+ boolean isOverridingKeyFieldTypes = index.isOverridingKeyFieldTypes();
+ if (numSecondaryKeys != 1) {
+ throw AsterixException.create(ErrorCode.INDEX_RTREE_MULTIPLE_FIELDS_NOT_ALLOWED, numSecondaryKeys);
+ }
+ Pair<IAType, Boolean> spatialTypePair =
+ Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0), secondaryKeyFields.get(0), itemType);
+ IAType spatialType = spatialTypePair.first;
+ anySecondaryKeyIsNullable = spatialTypePair.second;
+ isPointMBR = spatialType.getTypeTag() == ATypeTag.POINT || spatialType.getTypeTag() == ATypeTag.POINT3D;
+ int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
+ numNestedSecondaryKeyFields = numDimensions * 2;
+ int recordColumn = NUM_TAG_FIELDS + numPrimaryKeys;
+ secondaryFieldAccessEvalFactories =
+ metadataProvider.getFormat().createMBRFactory(isOverridingKeyFieldTypes ? enforcedItemType : itemType,
+ secondaryKeyFields.get(0), recordColumn, numDimensions, filterFieldName, isPointMBR);
+ secondaryComparatorFactories = new IBinaryComparatorFactory[numNestedSecondaryKeyFields];
+ valueProviderFactories = new IPrimitiveValueProviderFactory[numNestedSecondaryKeyFields];
+ ISerializerDeserializer[] secondaryRecFields =
+ new ISerializerDeserializer[numPrimaryKeys + numNestedSecondaryKeyFields + numFilterFields];
+ ISerializerDeserializer[] enforcedRecFields = new ISerializerDeserializer[1 + numPrimaryKeys + numFilterFields];
+ secondaryTypeTraits = new ITypeTraits[numNestedSecondaryKeyFields + numPrimaryKeys];
+ ITypeTraits[] enforcedTypeTraits = new ITypeTraits[1 + numPrimaryKeys];
+ IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(spatialType.getTypeTag());
+ keyType = nestedKeyType.getTypeTag();
+ for (int i = 0; i < numNestedSecondaryKeyFields; i++) {
+ ISerializerDeserializer keySerde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(nestedKeyType);
+ secondaryRecFields[i] = keySerde;
+ secondaryComparatorFactories[i] =
+ BinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(nestedKeyType, true);
+ secondaryTypeTraits[i] = TypeTraitProvider.INSTANCE.getTypeTrait(nestedKeyType);
+ valueProviderFactories[i] =
+ metadataProvider.getStorageComponentProvider().getPrimitiveValueProviderFactory();
+
+ }
+ // Add serializers and comparators for primary index fields.
+ // only support internal datasets
+ for (int i = 0; i < numPrimaryKeys; i++) {
+ secondaryRecFields[numNestedSecondaryKeyFields + i] = primaryRecDesc.getFields()[i];
+ secondaryTypeTraits[numNestedSecondaryKeyFields + i] = primaryRecDesc.getTypeTraits()[i];
+ enforcedRecFields[i] = primaryRecDesc.getFields()[i];
+ enforcedTypeTraits[i] = primaryRecDesc.getTypeTraits()[i];
+ }
+
+ enforcedRecFields[numPrimaryKeys] = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
+ enforcedRecDesc = new RecordDescriptor(enforcedRecFields, enforcedTypeTraits);
+ if (numFilterFields > 0) {
+ rtreeFields = new int[numNestedSecondaryKeyFields + numPrimaryKeys];
+ for (int i = 0; i < rtreeFields.length; i++) {
+ rtreeFields[i] = i;
+ }
+
+ Pair<IAType, Boolean> typePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
+ IAType type = typePair.first;
+ ISerializerDeserializer serde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(type);
+ secondaryRecFields[numPrimaryKeys + numNestedSecondaryKeyFields] = serde;
+ }
+ secondaryRecDesc = new RecordDescriptor(secondaryRecFields);
+ primaryKeyFields = new int[numPrimaryKeys];
+ for (int i = 0; i < primaryKeyFields.length; i++) {
+ primaryKeyFields[i] = i + numNestedSecondaryKeyFields;
+ }
+ if (isPointMBR) {
+ int numNestedSecondaryKeyFieldForPointMBR = numNestedSecondaryKeyFields / 2;
+ ISerializerDeserializer[] recFieldsForPointMBR = new ISerializerDeserializer[numPrimaryKeys
+ + numNestedSecondaryKeyFieldForPointMBR + numFilterFields];
+ int idx = 0;
+ for (int i = 0; i < numNestedSecondaryKeyFieldForPointMBR; i++) {
+ recFieldsForPointMBR[idx++] = secondaryRecFields[i];
+ }
+ for (int i = 0; i < numPrimaryKeys + numFilterFields; i++) {
+ recFieldsForPointMBR[idx++] = secondaryRecFields[numNestedSecondaryKeyFields + i];
+ }
+ secondaryRecDescForPointMBR = new RecordDescriptor(recFieldsForPointMBR);
+ }
+ }
+
+ @Override
+ public JobSpecification buildLoadingJobSpec() throws AsterixException, AlgebricksException {
+ /***************************************************
+ * [ About PointMBR Optimization ]
+ * Instead of storing a MBR(4 doubles) for a point(2 doubles) in RTree leaf node,
+ * PointMBR concept is introduced.
+ * PointMBR is a way to store a point as 2 doubles in RTree leaf node.
+ * This reduces RTree index size roughly in half.
+ * In order to fully benefit from the PointMBR concept, besides RTree,
+ * external sort operator during bulk-loading (from either data loading or index creation)
+ * must deal with point as 2 doubles instead of 4 doubles. Otherwise, external sort will suffer from twice as
+ * many doubles as it actually requires. For this purpose,
+ * PointMBR specific optimization logic is added as follows:
+ * 1) CreateMBR function in assign operator generates 2 doubles, instead of 4 doubles.
+ * 2) External sort operator sorts points represented with 2 doubles.
+ * 3) Bulk-loading in RTree takes 4 doubles by reading 2 doubles twice and then,
+ * do the same work as non-point MBR cases.
+ ***************************************************/
+ JobSpecification spec = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext());
+ int numNestedSecondaryKeFieldsConsideringPointMBR =
+ isPointMBR ? numNestedSecondaryKeyFields / 2 : numNestedSecondaryKeyFields;
+ RecordDescriptor secondaryRecDescConsideringPointMBR = isPointMBR
+ ? getTaggedRecordDescriptor(secondaryRecDescForPointMBR) : getTaggedRecordDescriptor(secondaryRecDesc);
+
+ boolean isOverridingKeyFieldTypes = index.isOverridingKeyFieldTypes();
+
+ assert dataset.getDatasetType() == DatasetType.INTERNAL;
+
+ // Create dummy key provider for feeding the primary index scan.
+ IOperatorDescriptor keyProviderOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset, metadataProvider);
+ JobId jobId = IndexUtil.bindJobEventListener(spec, metadataProvider);
+
+ // Create primary index scan op.
+ IOperatorDescriptor primaryScanOp = createPrimaryIndexScanDiskComponentsOp(spec, metadataProvider,
+ getTaggedRecordDescriptor(dataset.getPrimaryRecordDescriptor(metadataProvider)), jobId);
+
+ // Assign op.
+ IOperatorDescriptor sourceOp = primaryScanOp;
+ if (isOverridingKeyFieldTypes && !enforcedItemType.equals(itemType)) {
+ sourceOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced());
+ spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0);
+ }
+
+ AlgebricksMetaOperatorDescriptor asterixAssignOp = createAssignOp(spec,
+ numNestedSecondaryKeFieldsConsideringPointMBR, secondaryRecDescConsideringPointMBR);
+
+ // Generate compensate tuples for upsert
+ IOperatorDescriptor processorOp = createTupleProcessorOp(spec, secondaryRecDescConsideringPointMBR,
+ numNestedSecondaryKeFieldsConsideringPointMBR, numPrimaryKeys, false);
+
+ ExternalSortOperatorDescriptor sortOp = createSortOp(spec,
+ getTaggedSecondaryComparatorFactories(new IBinaryComparatorFactory[] {
+ MetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length) }),
+ secondaryRecDescConsideringPointMBR);
+
+ // Create secondary RTree bulk load op.
+ LSMSecondaryIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec,
+ metadataProvider, secondaryRecDescConsideringPointMBR, createFieldPermutationForBulkLoadOp(),
+ numNestedSecondaryKeFieldsConsideringPointMBR, numPrimaryKeys, false);
+
+ AlgebricksMetaOperatorDescriptor metaOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
+ new IPushRuntimeFactory[] { new SinkRuntimeFactory() }, new RecordDescriptor[] {});
+
+ // Connect the operators.
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, asterixAssignOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, processorOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), processorOp, 0, sortOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), secondaryBulkLoadOp, 0, metaOp, 0);
+ spec.addRoot(metaOp);
+ spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+
+ return spec;
+ }
+
+ @Override
+ protected int[] createFieldPermutationForBulkLoadOp() {
+ if (isPointMBR) {
+ int[] fieldPermutation =
+ new int[NUM_TAG_FIELDS + numNestedSecondaryKeyFields + numPrimaryKeys + numFilterFields];
+ int idx = 0;
+ int numSecondaryKeyFieldsForPointMBR = numNestedSecondaryKeyFields / 2;
+ for (int i = 0; i < NUM_TAG_FIELDS + numSecondaryKeyFieldsForPointMBR; i++) {
+ fieldPermutation[idx++] = i;
+ }
+ //add the rest of the sk fields for pointMBR
+ for (int i = 0; i < numSecondaryKeyFieldsForPointMBR; i++) {
+ fieldPermutation[idx++] = NUM_TAG_FIELDS + i;
+ }
+ //add the pk and filter fields
+ int end = numSecondaryKeyFieldsForPointMBR + numPrimaryKeys + numFilterFields;
+ for (int i = numSecondaryKeyFieldsForPointMBR; i < end; i++) {
+ fieldPermutation[idx++] = NUM_TAG_FIELDS + i;
+ }
+ return fieldPermutation;
+ } else {
+ return super.createFieldPermutationForBulkLoadOp();
+ }
+
+ }
+}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java
index fa33790..171d72e 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java
@@ -263,8 +263,8 @@
}
protected LSMSecondaryIndexBulkLoadOperatorDescriptor createTreeIndexBulkLoadOp(JobSpecification spec,
- MetadataProvider metadataProvider, RecordDescriptor taggedSecondaryRecDesc, int numSecondaryKeys,
- int numPrimaryKeys, boolean hasBuddyBtree) throws AlgebricksException {
+ MetadataProvider metadataProvider, RecordDescriptor taggedSecondaryRecDesc, int[] fieldPermutation,
+ int numSecondaryKeys, int numPrimaryKeys, boolean hasBuddyBtree) throws AlgebricksException {
IndexDataflowHelperFactory primaryIndexHelperFactory = new IndexDataflowHelperFactory(
metadataProvider.getStorageComponentProvider().getStorageManager(), primaryFileSplitProvider);
@@ -273,7 +273,8 @@
LSMSecondaryIndexBulkLoadOperatorDescriptor treeIndexBulkLoadOp =
new LSMSecondaryIndexBulkLoadOperatorDescriptor(spec, taggedSecondaryRecDesc, primaryIndexHelperFactory,
- secondaryIndexHelperFactory, NUM_TAG_FIELDS, numSecondaryKeys, numPrimaryKeys, hasBuddyBtree);
+ secondaryIndexHelperFactory, fieldPermutation, NUM_TAG_FIELDS, numSecondaryKeys,
+ numPrimaryKeys, hasBuddyBtree);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, treeIndexBulkLoadOp,
secondaryPartitionConstraint);
return treeIndexBulkLoadOp;
@@ -306,14 +307,16 @@
new SecondaryCorrelatedBTreeOperationsHelper(dataset, index, physOptConf, metadataProvider);
break;
case RTREE:
- //TODO RTree
+ indexOperationsHelper =
+ new SecondaryCorrelatedRTreeOperationsHelper(dataset, index, physOptConf, metadataProvider);
+ break;
case SINGLE_PARTITION_WORD_INVIX:
case SINGLE_PARTITION_NGRAM_INVIX:
case LENGTH_PARTITIONED_WORD_INVIX:
case LENGTH_PARTITIONED_NGRAM_INVIX:
- //TODO Inverted Index
- //TODO This will be fixed soon
- throw new UnsupportedOperationException();
+ indexOperationsHelper = new SecondaryCorrelatedInvertedIndexOperationsHelper(dataset, index,
+ physOptConf, metadataProvider);
+ break;
default:
throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_INDEX_TYPE, index.getIndexType());
}
@@ -321,4 +324,12 @@
return indexOperationsHelper;
}
+ protected int[] createFieldPermutationForBulkLoadOp() {
+ int[] fieldPermutation = new int[NUM_TAG_FIELDS + getNumSecondaryKeys() + numPrimaryKeys + numFilterFields];
+ for (int i = 0; i < fieldPermutation.length; i++) {
+ fieldPermutation[i] = i;
+ }
+ return fieldPermutation;
+ }
+
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
index cb15b98..b4d8a22 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
@@ -23,6 +23,7 @@
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.dataflow.data.nontagged.MissingWriterFactory;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
@@ -213,8 +214,8 @@
IOperatorDescriptor keyProviderOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset, metadataProvider);
// Create primary index scan op.
- IOperatorDescriptor primaryScanOp = DatasetUtil.createPrimaryIndexScanOp(spec, metadataProvider, dataset,
- jobId);
+ IOperatorDescriptor primaryScanOp =
+ DatasetUtil.createPrimaryIndexScanOp(spec, metadataProvider, dataset, jobId);
IOperatorDescriptor sourceOp = primaryScanOp;
boolean isOverridingKeyFieldTypes = index.isOverridingKeyFieldTypes();
@@ -270,8 +271,9 @@
for (int i = 0; i < primaryKeyFields.length; i++) {
primaryKeyFields[i] = numSecondaryKeys + i;
}
- BinaryTokenizerOperatorDescriptor tokenizerOp = new BinaryTokenizerOperatorDescriptor(spec, tokenKeyPairRecDesc,
- tokenizerFactory, docField, primaryKeyFields, isPartitioned, false);
+ BinaryTokenizerOperatorDescriptor tokenizerOp =
+ new BinaryTokenizerOperatorDescriptor(spec, tokenKeyPairRecDesc, tokenizerFactory, docField,
+ primaryKeyFields, isPartitioned, false, false, MissingWriterFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, tokenizerOp,
primaryPartitionConstraint);
return tokenizerOp;
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AbstractLSMSecondaryIndexCreationNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AbstractLSMSecondaryIndexCreationNodePushable.java
index 6b050af..0d71f71 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AbstractLSMSecondaryIndexCreationNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AbstractLSMSecondaryIndexCreationNodePushable.java
@@ -24,7 +24,6 @@
import org.apache.hyracks.data.std.primitive.BooleanPointable;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
@@ -33,9 +32,6 @@
protected final IHyracksTaskContext ctx;
protected final RecordDescriptor inputRecDesc;
- // with tag fields
- protected final FrameTupleReference tuple;
-
protected final int partition;
protected final int numTagFields;
protected final int numSecondaryKeys;
@@ -50,7 +46,6 @@
boolean hasBuddyBTree) {
this.ctx = ctx;
this.inputRecDesc = inputRecDesc;
- this.tuple = new FrameTupleReference();
this.partition = partition;
this.numTagFields = numTagFields;
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadNodePushable.java
index f09bcd2..2eb84fb 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadNodePushable.java
@@ -29,6 +29,7 @@
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import org.apache.hyracks.storage.am.common.tuples.PermutingFrameTupleReference;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader;
@@ -45,7 +46,7 @@
*/
public class LSMSecondaryIndexBulkLoadNodePushable extends AbstractLSMSecondaryIndexCreationNodePushable {
// with tag fields
-
+ private final PermutingFrameTupleReference tuple;
private final PermutingTupleReference sourceTuple;
private final PermutingTupleReference deletedKeyTuple;
@@ -63,16 +64,17 @@
public LSMSecondaryIndexBulkLoadNodePushable(IHyracksTaskContext ctx, int partition, RecordDescriptor inputRecDesc,
IIndexDataflowHelperFactory primaryIndexHelperFactory,
- IIndexDataflowHelperFactory secondaryIndexHelperFactory, int numTagFields, int numSecondaryKeys,
- int numPrimaryKeys, boolean hasBuddyBTree) throws HyracksDataException {
+ IIndexDataflowHelperFactory secondaryIndexHelperFactory, int[] fieldPermutation, int numTagFields,
+ int numSecondaryKeys, int numPrimaryKeys, boolean hasBuddyBTree) throws HyracksDataException {
super(ctx, partition, inputRecDesc, numTagFields, numSecondaryKeys, numPrimaryKeys, hasBuddyBTree);
this.primaryIndexHelper =
primaryIndexHelperFactory.create(ctx.getJobletContext().getServiceContext(), partition);
this.secondaryIndexHelper =
secondaryIndexHelperFactory.create(ctx.getJobletContext().getServiceContext(), partition);
+ this.tuple = new PermutingFrameTupleReference(fieldPermutation);
- int[] sourcePermutation = new int[inputRecDesc.getFieldCount() - numTagFields];
+ int[] sourcePermutation = new int[fieldPermutation.length - numTagFields];
for (int i = 0; i < sourcePermutation.length; i++) {
sourcePermutation[i] = i + numTagFields;
}
@@ -170,7 +172,6 @@
loadNewComponent(componentPos);
currentComponentPos = componentPos;
}
-
if (isAntiMatterTuple(tuple)) {
addAntiMatterTuple(tuple);
} else {
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadOperatorDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadOperatorDescriptor.java
index 804b700..4f1dfd7 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadOperatorDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadOperatorDescriptor.java
@@ -35,6 +35,7 @@
private final IIndexDataflowHelperFactory primaryIndexHelperFactory;
private final IIndexDataflowHelperFactory secondaryIndexHelperFactory;
+ private final int[] fieldPermutation;
private final int numTagFields;
private final int numSecondaryKeys;
private final int numPrimaryKeys;
@@ -43,12 +44,13 @@
public LSMSecondaryIndexBulkLoadOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor outRecDesc,
IIndexDataflowHelperFactory primaryIndexHelperFactory,
- IIndexDataflowHelperFactory secondaryIndexHelperFactory, int numTagFields, int numSecondaryKeys,
- int numPrimaryKeys, boolean hasBuddyBTree) {
+ IIndexDataflowHelperFactory secondaryIndexHelperFactory, int[] fieldPermutation, int numTagFields,
+ int numSecondaryKeys, int numPrimaryKeys, boolean hasBuddyBTree) {
super(spec, 1, 1);
this.outRecDescs[0] = outRecDesc;
this.primaryIndexHelperFactory = primaryIndexHelperFactory;
this.secondaryIndexHelperFactory = secondaryIndexHelperFactory;
+ this.fieldPermutation = fieldPermutation;
this.numTagFields = numTagFields;
this.numSecondaryKeys = numSecondaryKeys;
this.numPrimaryKeys = numPrimaryKeys;
@@ -61,6 +63,7 @@
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new LSMSecondaryIndexBulkLoadNodePushable(ctx, partition,
recordDescProvider.getInputRecordDescriptor(getActivityId(), 0), primaryIndexHelperFactory,
- secondaryIndexHelperFactory, numTagFields, numSecondaryKeys, numPrimaryKeys, hasBuddyBTree);
+ secondaryIndexHelperFactory, fieldPermutation, numTagFields, numSecondaryKeys, numPrimaryKeys,
+ hasBuddyBTree);
}
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexCreationTupleProcessorNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexCreationTupleProcessorNodePushable.java
index b2a2fa4..9376d1b 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexCreationTupleProcessorNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexCreationTupleProcessorNodePushable.java
@@ -36,6 +36,7 @@
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
+import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.dataflow.common.data.marshalling.BooleanSerializerDeserializer;
import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
@@ -69,6 +70,7 @@
* [component pos, anti-matter flag, secondary keys, primary keys, filter values]
*/
public class LSMSecondaryIndexCreationTupleProcessorNodePushable extends AbstractLSMSecondaryIndexCreationNodePushable {
+ private final FrameTupleReference tuple = new FrameTupleReference();
// prevSourceTuple stores the previous matter tuple
private final ArrayTupleBuilder prevMatterTupleBuilder;
private final ArrayTupleReference prevMatterTuple = new ArrayTupleReference();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/BinaryTokenizerOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/BinaryTokenizerOperatorDescriptor.java
index fe9decd..3ed47e6 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/BinaryTokenizerOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/BinaryTokenizerOperatorDescriptor.java
@@ -21,6 +21,7 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
+import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -47,8 +48,13 @@
// False: [token, number of token(if a partitioned index), keyfield1, keyfield2 ...]
private final boolean writeKeyFieldsFirst;
+ private final boolean writeMissing;
+
+ private final IMissingWriterFactory missingWriterFactory;
+
public BinaryTokenizerOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recDesc,
- IBinaryTokenizerFactory tokenizerFactory, int docField, int[] keyFields, boolean addNumTokensKey, boolean writeKeyFieldsFirst) {
+ IBinaryTokenizerFactory tokenizerFactory, int docField, int[] keyFields, boolean addNumTokensKey,
+ boolean writeKeyFieldsFirst, boolean writeMissing, IMissingWriterFactory missingWriterFactory) {
super(spec, 1, 1);
this.tokenizerFactory = tokenizerFactory;
this.docField = docField;
@@ -56,13 +62,16 @@
this.addNumTokensKey = addNumTokensKey;
outRecDescs[0] = recDesc;
this.writeKeyFieldsFirst = writeKeyFieldsFirst;
+ this.writeMissing = writeMissing;
+ this.missingWriterFactory = missingWriterFactory;
}
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
- return new BinaryTokenizerOperatorNodePushable(ctx, recordDescProvider.getInputRecordDescriptor(
- getActivityId(), 0), outRecDescs[0], tokenizerFactory.createTokenizer(), docField, keyFields,
- addNumTokensKey, writeKeyFieldsFirst);
+ return new BinaryTokenizerOperatorNodePushable(ctx,
+ recordDescProvider.getInputRecordDescriptor(getActivityId(), 0), outRecDescs[0],
+ tokenizerFactory.createTokenizer(), docField, keyFields, addNumTokensKey, writeKeyFieldsFirst,
+ writeMissing, missingWriterFactory);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/BinaryTokenizerOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/BinaryTokenizerOperatorNodePushable.java
index 002457b..3df185a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/BinaryTokenizerOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/BinaryTokenizerOperatorNodePushable.java
@@ -24,6 +24,8 @@
import org.apache.hyracks.api.comm.VSizeFrame;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.IMissingWriter;
+import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.util.GrowableArray;
@@ -31,6 +33,8 @@
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
+import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IToken;
@@ -45,6 +49,9 @@
private final boolean writeKeyFieldsFirst;
private final RecordDescriptor inputRecDesc;
private final RecordDescriptor outputRecDesc;
+ private final boolean writeMissing;
+ private final IMissingWriter missingWriter;
+ private final FrameTupleReference tuple = new FrameTupleReference();
private FrameTupleAccessor accessor;
private ArrayTupleBuilder builder;
@@ -53,7 +60,8 @@
public BinaryTokenizerOperatorNodePushable(IHyracksTaskContext ctx, RecordDescriptor inputRecDesc,
RecordDescriptor outputRecDesc, IBinaryTokenizer tokenizer, int docField, int[] keyFields,
- boolean addNumTokensKey, boolean writeKeyFieldsFirst) {
+ boolean addNumTokensKey, boolean writeKeyFieldsFirst, boolean writeMissing,
+ IMissingWriterFactory missingWriterFactory) {
this.ctx = ctx;
this.tokenizer = tokenizer;
this.docField = docField;
@@ -62,6 +70,8 @@
this.inputRecDesc = inputRecDesc;
this.outputRecDesc = outputRecDesc;
this.writeKeyFieldsFirst = writeKeyFieldsFirst;
+ this.writeMissing = writeMissing;
+ this.missingWriter = missingWriterFactory.createMissingWriter();
}
@Override
@@ -79,76 +89,85 @@
int tupleCount = accessor.getTupleCount();
for (int i = 0; i < tupleCount; i++) {
+ tuple.reset(accessor, i);
+
short numTokens = 0;
- tokenizer.reset(accessor.getBuffer().array(), accessor.getTupleStartOffset(i)
- + accessor.getFieldSlotsLength() + accessor.getFieldStartOffset(i, docField),
- accessor.getFieldLength(i, docField));
+ if (!isDocFieldMissing(tuple)) {
+ tokenizer.reset(tuple.getFieldData(docField), tuple.getFieldStart(docField),
+ tuple.getFieldLength(docField));
+ if (addNumTokensKey) {
+ // Get the total number of tokens.
+ numTokens = tokenizer.getTokensCount();
+ }
+ // Write token and data into frame by following the order specified
+ // in the writeKeyFieldsFirst field.
+ while (tokenizer.hasNext()) {
+ tokenizer.next();
+ IToken token = tokenizer.getToken();
+ writeTuple(token, numTokens, i);
+ }
+ } else if (writeMissing) {
+ writeTuple(null, 0, i);
+ }
+ }
- if (addNumTokensKey) {
- // Get the total number of tokens.
- numTokens = tokenizer.getTokensCount();
+ }
+
+ private void writeTuple(IToken token, int numTokens, int fieldIdx) throws HyracksDataException {
+ builder.reset();
+
+ // Writing Order: token, number of token, keyfield1 ... n
+ if (!writeKeyFieldsFirst) {
+ try {
+ if (token != null) {
+ token.serializeToken(builderData);
+ builder.addFieldEndOffset();
+ } else {
+ missingWriter.writeMissing(builder.getDataOutput());
+ builder.addFieldEndOffset();
+ }
+
+ // Add number of tokens if requested.
+ if (addNumTokensKey) {
+ builder.getDataOutput().writeShort(numTokens);
+ builder.addFieldEndOffset();
+ }
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
}
- // Write token and data into frame by following the order specified
- // in the writeKeyFieldsFirst field.
- while (tokenizer.hasNext()) {
-
- tokenizer.next();
-
- builder.reset();
-
- // Writing Order: token, number of token, keyfield1 ... n
- if (!writeKeyFieldsFirst) {
- try {
- IToken token = tokenizer.getToken();
- token.serializeToken(builderData);
-
- builder.addFieldEndOffset();
- // Add number of tokens if requested.
- if (addNumTokensKey) {
- builder.getDataOutput().writeShort(numTokens);
- builder.addFieldEndOffset();
- }
- } catch (IOException e) {
- throw new HyracksDataException(e.getMessage());
- }
-
- for (int k = 0; k < keyFields.length; k++) {
- builder.addField(accessor, i, keyFields[k]);
- }
-
- }
- // Writing Order: keyfield1 ... n, token, number of token
- else {
-
- for (int k = 0; k < keyFields.length; k++) {
- builder.addField(accessor, i, keyFields[k]);
- }
-
- try {
- IToken token = tokenizer.getToken();
- token.serializeToken(builderData);
-
- builder.addFieldEndOffset();
- // Add number of tokens if requested.
- if (addNumTokensKey) {
- builder.getDataOutput().writeShort(numTokens);
- builder.addFieldEndOffset();
- }
- } catch (IOException e) {
- throw new HyracksDataException(e.getMessage());
- }
-
- }
-
- FrameUtils.appendToWriter(writer, appender, builder.getFieldEndOffsets(), builder.getByteArray(), 0,
- builder.getSize());
-
+ for (int k = 0; k < keyFields.length; k++) {
+ builder.addField(accessor, fieldIdx, keyFields[k]);
}
}
+ // Writing Order: keyfield1 ... n, token, number of token
+ else {
+ for (int k = 0; k < keyFields.length; k++) {
+ builder.addField(accessor, fieldIdx, keyFields[k]);
+ }
+ try {
+ if (token != null) {
+ token.serializeToken(builderData);
+ builder.addFieldEndOffset();
+ } else {
+ missingWriter.writeMissing(builder.getDataOutput());
+ builder.addFieldEndOffset();
+ }
+ // Add number of tokens if requested.
+ if (addNumTokensKey) {
+ builder.getDataOutput().writeShort(numTokens);
+ builder.addFieldEndOffset();
+ }
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ FrameUtils.appendToWriter(writer, appender, builder.getFieldEndOffsets(), builder.getByteArray(), 0,
+ builder.getSize());
}
@Override
@@ -169,4 +188,14 @@
public void flush() throws HyracksDataException {
appender.flush(writer);
}
+
+ /**
+ * Returns whether the doc field is missing (only with a type tag)
+ *
+ * @param tuple
+ * @return
+ */
+ private boolean isDocFieldMissing(ITupleReference tuple) {
+ return tuple.getFieldLength(docField) <= 1;
+ }
}