[NO ISSUE][MD] Ignore index not exists on compensating ops
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When executing dataset drop compensating operations, ignore
failures if the dataset files were already deleted.
Change-Id: Ia4b3ff6b3c3cd1a3327f193c7398f983bd891064
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9144
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Reviewed-by: Till Westmann <tillw@apache.org>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 607e23a..45f0a82 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -227,6 +227,7 @@
import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.control.cc.ClusterControllerService;
import org.apache.hyracks.control.common.controllers.CCConfig;
+import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor.DropOption;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
@@ -871,7 +872,8 @@
bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
try {
- JobSpecification jobSpec = DatasetUtil.dropDatasetJobSpec(dataset, metadataProvider);
+ JobSpecification jobSpec =
+ DatasetUtil.dropDatasetJobSpec(dataset, metadataProvider, EnumSet.of(DropOption.IF_EXISTS));
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
runJob(hcc, jobSpec);
@@ -1654,6 +1656,7 @@
MutableBoolean bActiveTxn = new MutableBoolean(true);
metadataProvider.setMetadataTxnContext(mdTxnCtx.getValue());
List<JobSpecification> jobsToExecute = new ArrayList<>();
+ Dataset ds = null;
try {
// Check if the dataverse exists
Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx.getValue(), dataverseName);
@@ -1669,7 +1672,7 @@
throw new CompilationException(ErrorCode.UNKNOWN_DATAVERSE, sourceLoc, dataverseName);
}
}
- Dataset ds = metadataProvider.findDataset(dataverseName, datasetName);
+ ds = metadataProvider.findDataset(dataverseName, datasetName);
if (ds == null) {
if (ifExists) {
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
@@ -1682,11 +1685,12 @@
validateDatasetState(metadataProvider, ds, sourceLoc);
ds.drop(metadataProvider, mdTxnCtx, jobsToExecute, bActiveTxn, progress, hcc, dropCorrespondingNodeGroup,
- sourceLoc);
+ sourceLoc, Collections.emptySet());
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
return true;
} catch (Exception e) {
+ LOGGER.error("failed to drop dataset; executing compensating operations", e);
if (bActiveTxn.booleanValue()) {
abort(e, e, mdTxnCtx.getValue());
}
@@ -1695,6 +1699,11 @@
// #. execute compensation operations
// remove the all indexes in NC
try {
+ if (ds != null) {
+ jobsToExecute.clear();
+ ds.drop(metadataProvider, mdTxnCtx, jobsToExecute, bActiveTxn, progress, hcc,
+ dropCorrespondingNodeGroup, sourceLoc, EnumSet.of(DropOption.IF_EXISTS));
+ }
for (JobSpecification jobSpec : jobsToExecute) {
JobUtils.runJob(hcc, jobSpec, true);
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index 2171974..4e556a2 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -23,6 +23,7 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import java.util.stream.IntStream;
import org.apache.asterix.common.api.IDatasetInfoProvider;
@@ -104,6 +105,7 @@
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor.DropOption;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.common.api.IFrameOperationCallbackFactory;
@@ -344,8 +346,8 @@
*/
public void drop(MetadataProvider metadataProvider, MutableObject<MetadataTransactionContext> mdTxnCtx,
List<JobSpecification> jobsToExecute, MutableBoolean bActiveTxn, MutableObject<ProgressState> progress,
- IHyracksClientConnection hcc, boolean dropCorrespondingNodeGroup, SourceLocation sourceLoc)
- throws Exception {
+ IHyracksClientConnection hcc, boolean dropCorrespondingNodeGroup, SourceLocation sourceLoc,
+ Set<DropOption> options) throws Exception {
Map<FeedConnectionId, Pair<JobSpecification, Boolean>> disconnectJobList = new HashMap<>();
if (getDatasetType() == DatasetType.INTERNAL) {
// #. prepare jobs to drop the datatset and the indexes in NC
@@ -353,11 +355,11 @@
MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx.getValue(), dataverseName, datasetName);
for (int j = 0; j < indexes.size(); j++) {
if (indexes.get(j).isSecondaryIndex()) {
- jobsToExecute
- .add(IndexUtil.buildDropIndexJobSpec(indexes.get(j), metadataProvider, this, sourceLoc));
+ jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(j), metadataProvider, this, options,
+ sourceLoc));
}
}
- jobsToExecute.add(DatasetUtil.dropDatasetJobSpec(this, metadataProvider));
+ jobsToExecute.add(DatasetUtil.dropDatasetJobSpec(this, metadataProvider, options));
// #. mark the existing dataset as PendingDropOp
MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName);
MetadataManager.INSTANCE.addDataset(mdTxnCtx.getValue(),
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index bf3f8cf..52a133d 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -20,6 +20,7 @@
import java.io.DataOutput;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -289,6 +290,11 @@
public static JobSpecification dropDatasetJobSpec(Dataset dataset, MetadataProvider metadataProvider)
throws AlgebricksException, ACIDException {
+ return dropDatasetJobSpec(dataset, metadataProvider, Collections.emptySet());
+ }
+
+ public static JobSpecification dropDatasetJobSpec(Dataset dataset, MetadataProvider metadataProvider,
+ Set<IndexDropOperatorDescriptor.DropOption> options) throws AlgebricksException, ACIDException {
LOGGER.info("DROP DATASET: " + dataset);
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
return RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext());
@@ -298,7 +304,8 @@
metadataProvider.getSplitProviderAndConstraints(dataset);
IIndexDataflowHelperFactory indexHelperFactory = new IndexDataflowHelperFactory(
metadataProvider.getStorageComponentProvider().getStorageManager(), splitsAndConstraint.first);
- IndexDropOperatorDescriptor primaryBtreeDrop = new IndexDropOperatorDescriptor(specPrimary, indexHelperFactory);
+ IndexDropOperatorDescriptor primaryBtreeDrop =
+ new IndexDropOperatorDescriptor(specPrimary, indexHelperFactory, options);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(specPrimary, primaryBtreeDrop,
splitsAndConstraint.second);
specPrimary.addRoot(primaryBtreeDrop);