Changes to prevent loading a dataset more than once
Secondary indexes will always bulkload the primary index when they are created and they can also bulkload again when we use a load statement.
Removed test cases that does not make sense anymore which used to load a dataset more than once.
Fixed couple of test cases that was loading a dataset twice by mistake.
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
index fde3165..5fdc646 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
@@ -268,8 +268,8 @@
TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
splitsAndConstraint.first, typeTraits, comparatorFactories, blooFilterKeyFields, fieldPermutation,
- GlobalConfig.DEFAULT_BTREE_FILL_FACTOR, false, numElementsHint, new LSMBTreeDataflowHelperFactory(
- new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
+ GlobalConfig.DEFAULT_BTREE_FILL_FACTOR, false, numElementsHint, true,
+ new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.LSMBTREE_PRIMARY_PROVIDER,
new PrimaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.LSMBTREE_PRIMARY_PROVIDER,
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java
index 46f5b1a..07eb1c9 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java
@@ -343,7 +343,7 @@
AsterixRuntimeComponentsProvider.LSMBTREE_SECONDARY_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_SECONDARY_PROVIDER, secondaryFileSplitProvider,
secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields,
- fieldPermutation, fillFactor, false, numElementsHint, dataflowHelperFactory,
+ fieldPermutation, fillFactor, false, numElementsHint, false, dataflowHelperFactory,
NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, treeIndexBulkLoadOp,
secondaryPartitionConstraint);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
index 9a54847..435b4a7 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
@@ -266,10 +266,11 @@
}
IIndexDataflowHelperFactory dataflowHelperFactory = createDataflowHelperFactory();
LSMInvertedIndexBulkLoadOperatorDescriptor invIndexBulkLoadOp = new LSMInvertedIndexBulkLoadOperatorDescriptor(
- spec, fieldPermutation, false, numElementsHint, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
- secondaryFileSplitProvider, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, tokenTypeTraits,
- tokenComparatorFactories, invListsTypeTraits, primaryComparatorFactories, tokenizerFactory,
- dataflowHelperFactory, NoOpOperationCallbackFactory.INSTANCE);
+ spec, fieldPermutation, false, numElementsHint, false,
+ AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, secondaryFileSplitProvider,
+ AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, tokenTypeTraits, tokenComparatorFactories,
+ invListsTypeTraits, primaryComparatorFactories, tokenizerFactory, dataflowHelperFactory,
+ NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, invIndexBulkLoadOp,
secondaryPartitionConstraint);
return invIndexBulkLoadOp;
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/empty-load-with-index/empty-load-with-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/empty-load-with-index/empty-load-with-index.1.ddl.aql
deleted file mode 100644
index 356aafb..0000000
--- a/asterix-app/src/test/resources/runtimets/queries/dml/empty-load-with-index/empty-load-with-index.1.ddl.aql
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Test case Name : empty-load-with-index.aql
- * Description : Check that an empty load doesn't preclude a future non-empty load on secondary index
- * Expected Result : Success
- * Date : May 2 2012
- */
-
-drop dataverse test if exists;
-create dataverse test;
-
-use dataverse test;
-
-create type LineItemType as closed {
- l_orderkey: int32,
- l_partkey: int32,
- l_suppkey: int32,
- l_linenumber: int32,
- l_quantity: double,
- l_extendedprice: double,
- l_discount: double,
- l_tax: double,
- l_returnflag: string,
- l_linestatus: string,
- l_shipdate: string,
- l_commitdate: string,
- l_receiptdate: string,
- l_shipinstruct: string,
- l_shipmode: string,
- l_comment: string
-}
-
-create dataset LineItem(LineItemType)
- primary key l_orderkey, l_linenumber;
-
-create index part_index on LineItem(l_partkey);
-
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/empty-load-with-index/empty-load-with-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/empty-load-with-index/empty-load-with-index.2.update.aql
deleted file mode 100644
index 821c6a3..0000000
--- a/asterix-app/src/test/resources/runtimets/queries/dml/empty-load-with-index/empty-load-with-index.2.update.aql
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
- * Test case Name : empty-load-with-index.aql
- * Description : Check that an empty load doesn't preclude a future non-empty load on secondary index
- * Expected Result : Success
- * Date : May 2 2012
- */
-
-use dataverse test;
-
-load dataset LineItem
-using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
-(("path"="nc1://data/empty.adm"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
-
-load dataset LineItem
-using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
-(("path"="nc1://data/tpch0.001/lineitem.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
-
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/empty-load-with-index/empty-load-with-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/empty-load-with-index/empty-load-with-index.3.query.aql
deleted file mode 100644
index 5111256..0000000
--- a/asterix-app/src/test/resources/runtimets/queries/dml/empty-load-with-index/empty-load-with-index.3.query.aql
+++ /dev/null
@@ -1,13 +0,0 @@
-/*
- * Test case Name : empty-load-with-index.aql
- * Description : Check that an empty load doesn't preclude a future non-empty load on secondary index
- * Expected Result : Success
- * Date : May 2 2012
- */
-
-use dataverse test;
-
-for $c in dataset('LineItem')
-order by $c.l_orderkey, $c.l_linenumber
-limit 1
-return $c
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/empty-load/empty-load.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/empty-load/empty-load.1.ddl.aql
deleted file mode 100644
index 54da1a2..0000000
--- a/asterix-app/src/test/resources/runtimets/queries/dml/empty-load/empty-load.1.ddl.aql
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Test case Name : empty-load-with-index.aql
- * Description : Check that an empty load doesn't preclude a future non-empty load on primary index
- * Expected Result : Success
- * Date : May 2 2012
- */
-
-drop dataverse test if exists;
-create dataverse test;
-
-use dataverse test;
-
-create type LineItemType as closed {
- l_orderkey: int32,
- l_partkey: int32,
- l_suppkey: int32,
- l_linenumber: int32,
- l_quantity: double,
- l_extendedprice: double,
- l_discount: double,
- l_tax: double,
- l_returnflag: string,
- l_linestatus: string,
- l_shipdate: string,
- l_commitdate: string,
- l_receiptdate: string,
- l_shipinstruct: string,
- l_shipmode: string,
- l_comment: string
-}
-
-create dataset LineItem(LineItemType)
- primary key l_orderkey, l_linenumber;
-
-create index part_index on LineItem(l_partkey);
-
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/empty-load/empty-load.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/empty-load/empty-load.2.update.aql
deleted file mode 100644
index e9072d1..0000000
--- a/asterix-app/src/test/resources/runtimets/queries/dml/empty-load/empty-load.2.update.aql
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
- * Test case Name : empty-load-with-index.aql
- * Description : Check that an empty load doesn't preclude a future non-empty load on primary index
- * Expected Result : Success
- * Date : May 2 2012
- */
-
-use dataverse test;
-
-load dataset LineItem
-using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
-(("path"="nc1://data/empty.adm"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
-
-load dataset LineItem
-using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
-(("path"="nc1://data/tpch0.001/lineitem.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
-
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/empty-load/empty-load.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/empty-load/empty-load.3.query.aql
deleted file mode 100644
index bb4cac6..0000000
--- a/asterix-app/src/test/resources/runtimets/queries/dml/empty-load/empty-load.3.query.aql
+++ /dev/null
@@ -1,13 +0,0 @@
-/*
- * Test case Name : empty-load-with-index.aql
- * Description : Check that an empty load doesn't preclude a future non-empty load on primary index
- * Expected Result : Success
- * Date : May 2 2012
- */
-
-use dataverse test;
-
-for $c in dataset('LineItem')
-order by $c.l_orderkey, $c.l_linenumber
-limit 1
-return $c
diff --git a/asterix-app/src/test/resources/runtimets/queries/fuzzyjoin/dblp-3_1.1/dblp-3_1.1.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/fuzzyjoin/dblp-3_1.1/dblp-3_1.1.1.ddl.aql
index d68b466e..5af25b2 100644
--- a/asterix-app/src/test/resources/runtimets/queries/fuzzyjoin/dblp-3_1.1/dblp-3_1.1.1.ddl.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/fuzzyjoin/dblp-3_1.1/dblp-3_1.1.1.ddl.aql
@@ -12,9 +12,4 @@
misc: string
}
-create dataset DBLP(DBLPType) primary key id;
-
-load dataset DBLP
-using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
-(("path"="nc1://data/pub-small/dblp-small-id.txt"),("format"="delimited-text"),("delimiter"=":"));
-
+create dataset DBLP(DBLPType) primary key id;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index 7571664..f37ae3f 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -960,16 +960,6 @@
</compilation-unit>
</test-case>
<test-case FilePath="dml">
- <compilation-unit name="empty-load-with-index">
- <output-dir compare="Text">empty-load-with-index</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="dml">
- <compilation-unit name="empty-load">
- <output-dir compare="Text">empty-load</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="dml">
<compilation-unit name="insert-into-empty-dataset-with-index">
<output-dir compare="Text">insert-into-empty-dataset-with-index</output-dir>
</compilation-unit>
diff --git a/asterix-installer/ittest/asterix-lifecycle_backupRestore.adm b/asterix-installer/ittest/asterix-lifecycle_backupRestore.adm
index b74416d..139a6c5 100644
--- a/asterix-installer/ittest/asterix-lifecycle_backupRestore.adm
+++ b/asterix-installer/ittest/asterix-lifecycle_backupRestore.adm
@@ -1 +1 @@
-{ "DataverseName": "backupDataverse", "DataFormat": "edu.uci.ics.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Thu Jun 06 10:29:23 PDT 2013", "PendingOp": 0 }
+{ "DataverseName": "backupDataverse", "DataFormat": "edu.uci.ics.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Sat Jun 22 00:33:55 PDT 2013", "PendingOp": 0 }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
index 51ab213..836039e 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
@@ -804,8 +804,8 @@
TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
appContext.getStorageManagerInterface(), appContext.getIndexLifecycleManagerProvider(),
splitsAndConstraint.first, typeTraits, comparatorFactories, bloomFilterKeyFields, fieldPermutation,
- GlobalConfig.DEFAULT_BTREE_FILL_FACTOR, false, numElementsHint, new LSMBTreeDataflowHelperFactory(
- new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
+ GlobalConfig.DEFAULT_BTREE_FILL_FACTOR, false, numElementsHint, true,
+ new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.LSMBTREE_PRIMARY_PROVIDER,
new PrimaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.LSMBTREE_PRIMARY_PROVIDER,