ASTERIXDB-1841: Fix Inverted Index Drop Operator
Change-Id: I2c0f52770fd791ab9293c661de810c0b3b9ba8fb
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1600
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index bd5c214..74efb6c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -1050,7 +1050,7 @@
bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
try {
- JobSpecification jobSpec = IndexUtil.buildDropSecondaryIndexJobSpec(index, metadataProvider, ds);
+ JobSpecification jobSpec = IndexUtil.buildDropIndexJobSpec(index, metadataProvider, ds);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
JobUtils.runJob(hcc, jobSpec, true);
@@ -1199,7 +1199,7 @@
MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
for (int k = 0; k < indexes.size(); k++) {
if (indexes.get(k).isSecondaryIndex()) {
- jobsToExecute.add(IndexUtil.buildDropSecondaryIndexJobSpec(indexes.get(k), metadataProvider,
+ jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(k), metadataProvider,
datasets.get(j)));
}
}
@@ -1215,7 +1215,7 @@
jobsToExecute.add(ExternalIndexingOperations.buildDropFilesIndexJobSpec(metadataProvider,
datasets.get(j)));
} else {
- jobsToExecute.add(IndexUtil.buildDropSecondaryIndexJobSpec(indexes.get(k), metadataProvider,
+ jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(k), metadataProvider,
datasets.get(j)));
}
}
@@ -1423,7 +1423,7 @@
}
}
// #. prepare a job to drop the index in NC.
- jobsToExecute.add(IndexUtil.buildDropSecondaryIndexJobSpec(index, metadataProvider, ds));
+ jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(index, metadataProvider, ds));
// #. mark PendingDropOp on the existing index
MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName, indexName);
@@ -1463,7 +1463,7 @@
throw new AlgebricksException("Dropping a dataset's files index is not allowed.");
}
// #. prepare a job to drop the index in NC.
- jobsToExecute.add(IndexUtil.buildDropSecondaryIndexJobSpec(index, metadataProvider, ds));
+ jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(index, metadataProvider, ds));
List<Index> datasetIndexes =
MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
if (datasetIndexes.size() == 2) {
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.1.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.1.script.aql
new file mode 100644
index 0000000..cc46136
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.2.ddl.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.2.ddl.aql
new file mode 100644
index 0000000..60deb2d
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.2.ddl.aql
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+* Description : Create different types of secondary indexes then drop them after restart.
+* Expected Res : Success
+* Date : 20th March 2017
+*/
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+create type Name as open {
+first : string,
+last : string
+}
+
+create type Person as open {
+name : Name
+}
+
+create type TestType as open {
+id : int32,
+name : string,
+locn : point,
+zip : string,
+person : Person
+}
+
+create dataset t1(TestType) primary key id;
+create index rtree_index_point on t1(locn) type rtree;
+create index rtree_index_point_open on t1(open_locn:point?) type rtree enforced;
+create index keyWD_indx on t1(name) type keyword;
+create index keyWD_indx_open on t1(nickname:string?) type keyword enforced;
+create index secndIndx on t1(zip);
+create index nested on t1(person.name.first);
+create index secndIndx_open on t1(address:string?) enforced;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.3.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.3.script.aql
new file mode 100644
index 0000000..3ba1dc0
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.3.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.4.ddl.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.4.ddl.aql
new file mode 100644
index 0000000..42a361e
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.4.ddl.aql
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+* Description : Create different types of secondary indexes then drop them after restart.
+* Expected Res : Success
+* Date : 20th March 2017
+*/
+use dataverse test;
+drop index t1.rtree_index_point;
+drop index t1.rtree_index_point_open;
+drop index t1.keyWD_indx;
+drop index t1.keyWD_indx_open;
+drop index t1.secndIndx;
+drop index t1.nested;
+drop index t1.secndIndx_open;
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.5.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.5.script.aql
new file mode 100644
index 0000000..10e1a51
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.5.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/dml_after_restart/multiple_secondary_indices/create_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/dml_after_restart/multiple_secondary_indices/create_and_start.sh
new file mode 100755
index 0000000..2f6e71c
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/dml_after_restart/multiple_secondary_indices/create_and_start.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n asterix 1>/dev/null 2>&1;
+$MANAGIX_HOME/bin/managix delete -n asterix 1>/dev/null 2>&1;
+$MANAGIX_HOME/bin/managix create -n asterix -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/dml_after_restart/multiple_secondary_indices/stop_and_delete.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/dml_after_restart/multiple_secondary_indices/stop_and_delete.sh
new file mode 100755
index 0000000..5f247db
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/dml_after_restart/multiple_secondary_indices/stop_and_delete.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix delete -n asterix;
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/dml_after_restart/multiple_secondary_indices/stop_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/dml_after_restart/multiple_secondary_indices/stop_and_start.sh
new file mode 100755
index 0000000..e2b0583
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/dml_after_restart/multiple_secondary_indices/stop_and_start.sh
@@ -0,0 +1,2 @@
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix start -n asterix;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
index a9644b0..07e2d6d 100644
--- a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
@@ -34,6 +34,13 @@
</compilation-unit>
</test-case>
</test-group>
+ <test-group name="dml_after_restart">
+ <test-case FilePath="dml_after_restart">
+ <compilation-unit name="multiple_secondary_indices">
+ <output-dir compare="Text">multiple_secondary_indices</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
<test-group name="recover_after_abort">
<test-case FilePath="recover_after_abort">
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index cd8cf3b..82fe036 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -295,7 +295,7 @@
MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx.getValue(), dataverseName, datasetName);
for (int j = 0; j < indexes.size(); j++) {
if (indexes.get(j).isSecondaryIndex()) {
- jobsToExecute.add(IndexUtil.dropJob(indexes.get(j), metadataProvider, this));
+ jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(j), metadataProvider, this));
}
}
Index primaryIndex =
@@ -334,7 +334,7 @@
MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx.getValue(), dataverseName, datasetName);
for (int j = 0; j < indexes.size(); j++) {
if (ExternalIndexingOperations.isFileIndex(indexes.get(j))) {
- jobsToExecute.add(IndexUtil.dropJob(indexes.get(j), metadataProvider, this));
+ jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(j), metadataProvider, this));
} else {
jobsToExecute.add(DatasetUtil.buildDropFilesIndexJobSpec(metadataProvider, this));
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
index 701d0d6..83403d5 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
@@ -18,11 +18,7 @@
*/
package org.apache.asterix.metadata.utils;
-import java.util.List;
-import java.util.Map;
-
import org.apache.asterix.common.config.OptimizationConfUtil;
-import org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.external.indexing.ExternalFile;
@@ -30,18 +26,14 @@
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.runtime.utils.RuntimeUtils;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
-import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
-import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+
+import java.util.Collections;
+import java.util.List;
public class IndexUtil {
@@ -97,27 +89,24 @@
return empty;
}
- public static JobSpecification dropJob(Index index, MetadataProvider metadataProvider, Dataset dataset)
+ public static JobSpecification buildDropIndexJobSpec(Index index, MetadataProvider metadataProvider,
+ Dataset dataset)
throws AlgebricksException {
- JobSpecification spec = RuntimeUtils.createJobSpecification();
- IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider();
- Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
- metadataProvider.getSplitProviderAndConstraints(dataset, index.getIndexName());
- Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
- DatasetUtil.getMergePolicyFactory(dataset, metadataProvider.getMetadataTxnContext());
ARecordType recordType =
(ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
ARecordType metaType = DatasetUtil.getMetaType(metadataProvider, dataset);
- IIndexDataflowHelperFactory dataflowHelperFactory = dataset.getIndexDataflowHelperFactory(metadataProvider,
- index, recordType, metaType, compactionInfo.first, compactionInfo.second);
- IndexDropOperatorDescriptor btreeDrop =
- new IndexDropOperatorDescriptor(spec, storageComponentProvider.getStorageManager(),
- storageComponentProvider.getIndexLifecycleManagerProvider(), splitsAndConstraint.first,
- dataflowHelperFactory, storageComponentProvider.getMetadataPageManagerFactory());
- AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, btreeDrop,
- splitsAndConstraint.second);
- spec.addRoot(btreeDrop);
- return spec;
+ ARecordType enforcedType = null;
+ ARecordType enforcedMetaType = null;
+ if (index.isEnforcingKeyFileds()) {
+ Pair<ARecordType, ARecordType> enforcedTypes =
+ TypeUtil.createEnforcedType(recordType, metaType, Collections.singletonList(index));
+ enforcedType = enforcedTypes.first;
+ enforcedMetaType = enforcedTypes.second;
+ }
+ SecondaryIndexOperationsHelper secondaryIndexHelper =
+ SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider,
+ physicalOptimizationConfig, recordType, metaType, enforcedType, enforcedMetaType);
+ return secondaryIndexHelper.buildDropJobSpec();
}
public static JobSpecification buildSecondaryIndexCreationJobSpec(Dataset dataset, Index index,
@@ -148,31 +137,6 @@
return secondaryIndexHelper.buildLoadingJobSpec();
}
- public static JobSpecification buildDropSecondaryIndexJobSpec(Index index, MetadataProvider metadataProvider,
- Dataset dataset) throws AlgebricksException {
- JobSpecification spec = RuntimeUtils.createJobSpecification();
- IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider();
- Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
- metadataProvider.getSplitProviderAndConstraints(dataset, index.getIndexName());
- Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
- DatasetUtil.getMergePolicyFactory(dataset, metadataProvider.getMetadataTxnContext());
- ARecordType recordType =
- (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
- ARecordType metaType = DatasetUtil.getMetaType(metadataProvider, dataset);
- IIndexDataflowHelperFactory dataflowHelperFactory = dataset.getIndexDataflowHelperFactory(metadataProvider,
- index, recordType, metaType, compactionInfo.first, compactionInfo.second);
- // The index drop operation should be persistent regardless of temp datasets or permanent dataset.
- IndexDropOperatorDescriptor btreeDrop =
- new IndexDropOperatorDescriptor(spec, storageComponentProvider.getStorageManager(),
- storageComponentProvider.getIndexLifecycleManagerProvider(), splitsAndConstraint.first,
- dataflowHelperFactory, storageComponentProvider.getMetadataPageManagerFactory());
- AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, btreeDrop,
- splitsAndConstraint.second);
- spec.addRoot(btreeDrop);
-
- return spec;
- }
-
public static JobSpecification buildSecondaryIndexCompactJobSpec(Dataset dataset, Index index, ARecordType recType,
ARecordType metaType, ARecordType enforcedType, ARecordType enforcedMetaType,
MetadataProvider metadataProvider) throws AlgebricksException {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
index 63368c7..59068c8 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
@@ -66,7 +66,7 @@
import org.apache.hyracks.storage.common.file.ILocalResourceFactoryProvider;
import org.apache.hyracks.storage.common.file.LocalResource;
-public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelper {
+public class SecondaryBTreeOperationsHelper extends SecondaryTreeIndexOperationsHelper {
protected SecondaryBTreeOperationsHelper(Dataset dataset, Index index, PhysicalOptimizationConfig physOptConf,
IPropertiesProvider propertiesProvider, MetadataProvider metadataProvider, ARecordType recType,
@@ -237,28 +237,6 @@
}
@Override
- public JobSpecification buildCompactJobSpec() throws AlgebricksException {
- JobSpecification spec = RuntimeUtils.createJobSpecification();
- LSMTreeIndexCompactOperatorDescriptor compactOp;
- IIndexDataflowHelperFactory dataflowHelperFactory = dataset.getIndexDataflowHelperFactory(metadataProvider,
- index, itemType, metaType, mergePolicyFactory, mergePolicyFactoryProperties);
- IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider();
- compactOp =
- new LSMTreeIndexCompactOperatorDescriptor(spec, storageComponentProvider.getStorageManager(),
- storageComponentProvider.getIndexLifecycleManagerProvider(), secondaryFileSplitProvider,
- secondaryTypeTraits, secondaryComparatorFactories, secondaryBloomFilterKeyFields,
- dataflowHelperFactory,
- dataset.getModificationCallbackFactory(storageComponentProvider, index, null,
- IndexOperation.FULL_MERGE, null),
- storageComponentProvider.getMetadataPageManagerFactory());
- AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
- secondaryPartitionConstraint);
- spec.addRoot(compactOp);
- spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
- return spec;
- }
-
- @Override
@SuppressWarnings("rawtypes")
protected void setSecondaryRecDescAndComparators() throws AlgebricksException {
int numSecondaryKeys = index.getKeyFieldNames().size();
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index d731603..efb3315 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -185,6 +185,8 @@
public abstract JobSpecification buildCompactJobSpec() throws AlgebricksException;
+ public abstract JobSpecification buildDropJobSpec() throws AlgebricksException;
+
protected void init() throws AlgebricksException {
payloadSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
metaSerde =
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
index b86004a..5ab36c1 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
@@ -36,6 +36,7 @@
import org.apache.asterix.runtime.utils.RuntimeUtils;
import org.apache.asterix.transaction.management.resource.LSMInvertedIndexLocalResourceMetadataFactory;
import org.apache.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -57,18 +58,23 @@
import org.apache.hyracks.dataflow.common.data.marshalling.ShortSerializerDeserializer;
import org.apache.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
+import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.BinaryTokenizerOperatorDescriptor;
import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexBulkLoadOperatorDescriptor;
import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexCompactOperator;
import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexCreateOperatorDescriptor;
+import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexDropOperatorDescriptor;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
import org.apache.hyracks.storage.common.file.ILocalResourceFactoryProvider;
import org.apache.hyracks.storage.common.file.LocalResource;
+import java.util.Map;
+
public class SecondaryInvertedIndexOperationsHelper extends SecondaryIndexOperationsHelper {
private IAType secondaryKeyType;
@@ -376,4 +382,27 @@
spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
return spec;
}
+
+ @Override
+ public JobSpecification buildDropJobSpec() throws AlgebricksException {
+ JobSpecification spec = RuntimeUtils.createJobSpecification();
+ IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider();
+ Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
+ metadataProvider.getSplitProviderAndConstraints(dataset, index.getIndexName());
+ Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
+ DatasetUtil.getMergePolicyFactory(dataset, metadataProvider.getMetadataTxnContext());
+ ARecordType recordType =
+ (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
+ ARecordType metaType = DatasetUtil.getMetaType(metadataProvider, dataset);
+ IIndexDataflowHelperFactory dataflowHelperFactory = dataset.getIndexDataflowHelperFactory(metadataProvider,
+ index, recordType, metaType, compactionInfo.first, compactionInfo.second);
+ LSMInvertedIndexDropOperatorDescriptor invIdxDrop =
+ new LSMInvertedIndexDropOperatorDescriptor(spec, storageComponentProvider.getStorageManager(),
+ storageComponentProvider.getIndexLifecycleManagerProvider(), splitsAndConstraint.first,
+ dataflowHelperFactory, storageComponentProvider.getMetadataPageManagerFactory());
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, invIdxDrop,
+ splitsAndConstraint.second);
+ spec.addRoot(invIdxDrop);
+ return spec;
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
index 460b635..93a88e2 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
@@ -72,7 +72,7 @@
import org.apache.hyracks.storage.common.file.LocalResource;
@SuppressWarnings("rawtypes")
-public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelper {
+public class SecondaryRTreeOperationsHelper extends SecondaryTreeIndexOperationsHelper {
protected IPrimitiveValueProviderFactory[] valueProviderFactories;
protected int numNestedSecondaryKeyFields;
@@ -430,25 +430,4 @@
}
return fieldPermutation;
}
-
- @Override
- public JobSpecification buildCompactJobSpec() throws AsterixException, AlgebricksException {
- JobSpecification spec = RuntimeUtils.createJobSpecification();
- IIndexDataflowHelperFactory indexDataflowHelperFactory = dataset.getIndexDataflowHelperFactory(
- metadataProvider, index, itemType, metaType, mergePolicyFactory, mergePolicyFactoryProperties);
- LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
- metadataProvider.getStorageComponentProvider().getStorageManager(),
- metadataProvider.getStorageComponentProvider().getIndexLifecycleManagerProvider(),
- secondaryFileSplitProvider, secondaryTypeTraits, secondaryComparatorFactories,
- secondaryBloomFilterKeyFields, indexDataflowHelperFactory,
- dataset.getModificationCallbackFactory(metadataProvider.getStorageComponentProvider(), index, null,
- IndexOperation.FULL_MERGE, null),
- metadataProvider.getStorageComponentProvider().getMetadataPageManagerFactory());
-
- AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
- secondaryPartitionConstraint);
- spec.addRoot(compactOp);
- spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
- return spec;
- }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java
new file mode 100644
index 0000000..2b025c9
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.metadata.utils;
+
+import org.apache.asterix.common.config.IPropertiesProvider;
+import org.apache.asterix.common.context.IStorageComponentProvider;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.runtime.utils.RuntimeUtils;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.core.jobgen.impl.ConnectorPolicyAssignmentPolicy;
+import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
+import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import org.apache.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexCompactOperatorDescriptor;
+
+import java.util.Map;
+
+public abstract class SecondaryTreeIndexOperationsHelper extends SecondaryIndexOperationsHelper {
+
+ protected SecondaryTreeIndexOperationsHelper(Dataset dataset,
+ Index index,
+ PhysicalOptimizationConfig physOptConf,
+ IPropertiesProvider propertiesProvider,
+ MetadataProvider metadataProvider,
+ ARecordType recType, ARecordType metaType,
+ ARecordType enforcedType,
+ ARecordType enforcedMetaType) {
+ super(dataset, index, physOptConf, propertiesProvider, metadataProvider, recType, metaType, enforcedType,
+ enforcedMetaType);
+ }
+
+ @Override
+ public JobSpecification buildDropJobSpec() throws AlgebricksException {
+ JobSpecification spec = RuntimeUtils.createJobSpecification();
+ IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider();
+ Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
+ metadataProvider.getSplitProviderAndConstraints(dataset, index.getIndexName());
+ Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
+ DatasetUtil.getMergePolicyFactory(dataset, metadataProvider.getMetadataTxnContext());
+ ARecordType recordType =
+ (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
+ ARecordType metaType = DatasetUtil.getMetaType(metadataProvider, dataset);
+ IIndexDataflowHelperFactory dataflowHelperFactory = dataset.getIndexDataflowHelperFactory(metadataProvider,
+ index, recordType, metaType, compactionInfo.first, compactionInfo.second);
+ // The index drop operation should be persistent regardless of temp datasets or permanent dataset.
+ IndexDropOperatorDescriptor btreeDrop =
+ new IndexDropOperatorDescriptor(spec, storageComponentProvider.getStorageManager(),
+ storageComponentProvider.getIndexLifecycleManagerProvider(), splitsAndConstraint.first,
+ dataflowHelperFactory, storageComponentProvider.getMetadataPageManagerFactory());
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, btreeDrop,
+ splitsAndConstraint.second);
+ spec.addRoot(btreeDrop);
+ return spec;
+ }
+
+ @Override
+ public JobSpecification buildCompactJobSpec() throws AlgebricksException {
+ JobSpecification spec = RuntimeUtils.createJobSpecification();
+ IIndexDataflowHelperFactory indexDataflowHelperFactory = dataset.getIndexDataflowHelperFactory(
+ metadataProvider, index, itemType, metaType, mergePolicyFactory, mergePolicyFactoryProperties);
+ LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
+ metadataProvider.getStorageComponentProvider().getStorageManager(),
+ metadataProvider.getStorageComponentProvider().getIndexLifecycleManagerProvider(),
+ secondaryFileSplitProvider, secondaryTypeTraits, secondaryComparatorFactories,
+ secondaryBloomFilterKeyFields, indexDataflowHelperFactory,
+ dataset.getModificationCallbackFactory(metadataProvider.getStorageComponentProvider(), index, null,
+ IndexOperation.FULL_MERGE, null),
+ metadataProvider.getStorageComponentProvider().getMetadataPageManagerFactory());
+
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
+ secondaryPartitionConstraint);
+ spec.addRoot(compactOp);
+ spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+ return spec;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDropOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDropOperatorDescriptor.java
new file mode 100644
index 0000000..fbbd088
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDropOperatorDescriptor.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.invertedindex.dataflow;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
+import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
+import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
+import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
+import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorNodePushable;
+import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
+import org.apache.hyracks.storage.common.IStorageManager;
+import org.apache.hyracks.storage.common.file.NoOpLocalResourceFactoryProvider;
+
+public class LSMInvertedIndexDropOperatorDescriptor extends AbstractLSMInvertedIndexOperatorDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ public LSMInvertedIndexDropOperatorDescriptor(IOperatorDescriptorRegistry spec, IStorageManager storageManager,
+ IIndexLifecycleManagerProvider lifecycleManagerProvider, IFileSplitProvider fileSplitProvider,
+ IIndexDataflowHelperFactory dataflowHelperFactory, IPageManagerFactory pageManagerFactory) {
+ super(spec, 0, 0, null, storageManager, fileSplitProvider, lifecycleManagerProvider, new ITypeTraits[] {
+ null },
+ new IBinaryComparatorFactory[] { null }, new ITypeTraits[] {
+ null }, new IBinaryComparatorFactory[] { null }, null,
+ dataflowHelperFactory, null, false, false, null, NoOpLocalResourceFactoryProvider.INSTANCE,
+ NoOpOperationCallbackFactory.INSTANCE, NoOpOperationCallbackFactory.INSTANCE, pageManagerFactory);
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
+ return new IndexDropOperatorNodePushable(this, ctx, partition);
+ }
+}