ASTERIXDB-1538: fix an issue where LSMRTree index instance is created during the recovery process.
- LSMRTreeWithAntiMatterTuples index instance is properly created during the recovery process.
Change-Id: I701a7ad748ff8982534a132a01f5a1ae77c952fd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1010
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Young-Seok Kim <kisskys@gmail.com>
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.1.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.1.script.aql
new file mode 100644
index 0000000..323b1cf
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.2.ddl.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.2.ddl.aql
new file mode 100644
index 0000000..efd95c1
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.2.ddl.aql
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : primary_plus_rtree_index_insert_and_delete.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ */
+
+drop dataverse recovery if exists;
+create dataverse recovery;
+use dataverse recovery;
+
+/* For raw Fragile data */
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float,
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+
+};
+
+/* For cleaned Fragile data */
+create type FragileType as closed {
+ row_id: int32,
+ sid: int32,
+ date: date,
+ day: int32,
+ time: time,
+ bpm: int32,
+ RR: float,
+
+ /* new string field and location field*/
+ text: string,
+ location: point,
+ text2: string
+};
+
+/* Create dataset for loading raw Fragile data */
+create dataset Fragile_raw (FragileTypeRaw)
+primary key row_id;
+
+/* Create dataset for cleaned Fragile data */
+create dataset Fragile (FragileType)
+primary key row_id;
+
+
+/* Create rtree secondary index on dataset clean Fragile */
+create index cfLocation on Fragile(location) type rtree;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.3.update.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.3.update.aql
new file mode 100644
index 0000000..1d24079
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.3.update.aql
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : primary_plus_rtree_index_insert_and_delete.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ */
+
+use dataverse recovery;
+
+
+load dataset Fragile_raw using localfs
+(("path"="asterix_nc1://../../../../../../asterix-app/data/csv/fragile_02.adm"),("format"="adm")) pre-sorted;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.4.txneu.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.4.txneu.aql
new file mode 100644
index 0000000..2fd81ab
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.4.txneu.aql
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : primary_plus_rtree_index_insert_and_delete.aql
+ * Description : Check that abort from duplicate key exception works and crash recovery works after the abort.
+ * Expected Result : Success
+ */
+
+use dataverse recovery;
+
+/* insert Fragile data from raw dataset into cleaned dataset (only the first 8000 rows) */
+insert into dataset Fragile (
+ for $t in dataset Fragile_raw
+ where $t.row_id <= 8000
+ return {
+ "row_id": $t.row_id,
+ "sid": $t.sid,
+ "date": date($t.date),
+ "day": $t.day,
+ "time": parse-time($t.time, "h:m:s"),
+ "bpm": $t.bpm,
+ "RR": $t.RR,
+ "text": $t.text,
+ "location": $t.location,
+ "text2": $t.text2
+ }
+);
+
+/* delete the first 3000 rows */
+delete $o from dataset Fragile
+where $o.row_id <= 3000;
+
+/* insert Fragile data from raw dataset into cleaned dataset (only the rowid between 2000 and 3500):
+ this time, the exception should occur since the rows with the rowid between 3001 and 3500 already exist.
+*/
+insert into dataset Fragile (
+ for $t in dataset Fragile_raw
+ where $t.row_id >= 2000 and $t.row_id <= 3500
+ return {
+ "row_id": $t.row_id,
+ "sid": $t.sid,
+ "date": date($t.date),
+ "day": $t.day,
+ "time": parse-time($t.time, "h:m:s"),
+ "bpm": $t.bpm,
+ "RR": $t.RR,
+ "text": $t.text,
+ "location": $t.location,
+ "text2": $t.text2
+ }
+);
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.5.txnqbc.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.5.txnqbc.aql
new file mode 100644
index 0000000..a256c53
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.5.txnqbc.aql
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse recovery;
+
+count (for $x in dataset Fragile where
+spatial-intersect($x.location, create-polygon([0.0,0.0, 180.0,180.0, 0.0,180.0, 180.0,0.0]))
+return $x);
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.6.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.6.script.aql
new file mode 100644
index 0000000..31d37ae
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.6.script.aql
@@ -0,0 +1 @@
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.7.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.7.script.aql
new file mode 100644
index 0000000..37ef6c0
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.7.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.8.txnqar.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.8.txnqar.aql
new file mode 100644
index 0000000..a256c53
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.8.txnqar.aql
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse recovery;
+
+count (for $x in dataset Fragile where
+spatial-intersect($x.location, create-polygon([0.0,0.0, 180.0,180.0, 0.0,180.0, 180.0,0.0]))
+return $x);
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.9.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.9.script.aql
new file mode 100644
index 0000000..f75dfc9
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index_insert_and_delete/primary_plus_rtree_index_insert_and_delete.9.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_rtree_index_insert_and_delete/create_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_rtree_index_insert_and_delete/create_and_start.sh
new file mode 100755
index 0000000..789914b
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_rtree_index_insert_and_delete/create_and_start.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix create -n asterix -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_rtree_index_insert_and_delete/kill_cc_and_nc.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_rtree_index_insert_and_delete/kill_cc_and_nc.sh
new file mode 100755
index 0000000..2582713
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_rtree_index_insert_and_delete/kill_cc_and_nc.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_rtree_index_insert_and_delete/stop_and_delete.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_rtree_index_insert_and_delete/stop_and_delete.sh
new file mode 100755
index 0000000..eb1c01e
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_rtree_index_insert_and_delete/stop_and_delete.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix delete -n asterix;
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_rtree_index_insert_and_delete/stop_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_rtree_index_insert_and_delete/stop_and_start.sh
new file mode 100755
index 0000000..2fb80ce
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/recover_after_abort/primary_plus_rtree_index_insert_and_delete/stop_and_start.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix start -n asterix;
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
index 75b2457..a9644b0 100644
--- a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
@@ -55,6 +55,12 @@
</test-case>
<test-case FilePath="recover_after_abort">
+ <compilation-unit name="primary_plus_rtree_index_insert_and_delete">
+ <output-dir compare="Text">primary_plus_rtree_index_insert_and_delete</output-dir>
+ </compilation-unit>
+ </test-case>
+
+ <test-case FilePath="recover_after_abort">
<compilation-unit name="primary_plus_keyword_secondary_index">
<output-dir compare="Text">primary_plus_keyword_secondary_index</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
index afe9cd2..535448c 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
@@ -81,16 +81,16 @@
List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getDatasetLifecycleManager()
.getVirtualBufferCaches(datasetID, ioDeviceNum);
try {
- return LSMRTreeUtils.createLSMTree(virtualBufferCaches, file, runtimeContextProvider.getBufferCache(),
- runtimeContextProvider.getFileMapManager(), typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, rtreePolicyType, runtimeContextProvider.getBloomFilterFalsePositiveRate(),
+ return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(virtualBufferCaches, file,
+ runtimeContextProvider.getBufferCache(), runtimeContextProvider.getFileMapManager(), typeTraits,
+ rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
runtimeContextProvider.getDatasetLifecycleManager()),
new BaseOperationTracker(datasetID,
runtimeContextProvider.getDatasetLifecycleManager().getDatasetInfo(datasetID)),
runtimeContextProvider.getLSMIOScheduler(),
LSMRTreeIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), linearizeCmpFactory,
- rtreeFields, btreeFields, filterTypeTraits, filterCmpFactories, filterFields, true, isPointMBR);
+ rtreeFields, filterTypeTraits, filterCmpFactories, filterFields, true, isPointMBR);
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
}