[NO ISSUE] Force flag for dataset drop & query service servlet refactoring
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Expose the force flag for dataset drop to extensions.
Change-Id: I0cf4afa65752e3faa4351d51098f9f78412ba88c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9325
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Reviewed-by: Dmitry Lychagin <dmitry.lychagin@couchbase.com>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
index 417a130..4ad1040 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
@@ -67,4 +67,10 @@
* {@code 0} if all categories are allowed
*/
int getStatementCategoryRestrictionMask();
+
+ /**
+ * @return true if DROP DATASET statements in the request should force drop datasets which could make the metadata
+ * inconsistent.
+ */
+ boolean isForceDropDataset();
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index d43e611..6de91d8 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -90,6 +90,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.api.exceptions.Warning;
+import org.apache.hyracks.api.result.IResultSet;
import org.apache.hyracks.control.common.controllers.CCConfig;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
@@ -269,7 +270,7 @@
SessionOutput sessionOutput = createSessionOutput(httpWriter);
ResponsePrinter responsePrinter = new ResponsePrinter(sessionOutput);
ResultDelivery delivery = ResultDelivery.IMMEDIATE;
- QueryServiceRequestParameters param = newRequestParameters();
+ QueryServiceRequestParameters param = newQueryRequestParameters();
RequestExecutionState executionState = newRequestExecutionState();
try {
// buffer the output until we are ready to set the status of the response message correctly
@@ -411,9 +412,8 @@
int stmtCategoryRestriction = org.apache.asterix.app.translator.RequestParameters
.getStatementCategoryRestrictionMask(param.isReadOnly());
IRequestParameters requestParameters =
- new org.apache.asterix.app.translator.RequestParameters(requestReference, statementsText,
- getResultSet(), resultProperties, stats, statementProperties, null, param.getClientContextID(),
- optionalParameters, stmtParams, param.isMultiStatement(), stmtCategoryRestriction);
+ newRequestParameters(param, requestReference, statementsText, getResultSet(), resultProperties, stats,
+ statementProperties, optionalParameters, stmtParams, stmtCategoryRestriction);
translator.compileAndExecute(getHyracksClientConnection(), requestParameters);
executionState.end();
translator.getWarnings(warnings, maxWarnings - warnings.size());
@@ -492,10 +492,19 @@
responsePrinter.addResultPrinter(new ErrorsPrinter(Collections.singletonList(executionError)));
}
- protected QueryServiceRequestParameters newRequestParameters() {
+ protected QueryServiceRequestParameters newQueryRequestParameters() {
return new QueryServiceRequestParameters();
}
+ protected IRequestParameters newRequestParameters(QueryServiceRequestParameters param,
+ IRequestReference requestReference, String statementsText, IResultSet resultSet,
+ ResultProperties resultProperties, Stats stats, IStatementExecutor.StatementProperties statementProperties,
+ Map<String, String> optionalParameters, Map<String, IAObject> stmtParams, int stmtCategoryRestriction) {
+ return new RequestParameters(requestReference, statementsText, resultSet, resultProperties, stats,
+ statementProperties, null, param.getClientContextID(), optionalParameters, stmtParams,
+ param.isMultiStatement(), stmtCategoryRestriction);
+ }
+
protected static boolean isPrintingProfile(IStatementExecutor.Stats stats) {
return stats.getProfileType() == Stats.ProfileType.FULL && stats.getJobProfile() != null;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 2ba5c0e..c000d8b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -857,7 +857,8 @@
// #. add a new dataset with PendingNoOp after deleting the dataset with
// PendingAddOp
- MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(), dataverseName, datasetName);
+ MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(), dataverseName, datasetName,
+ requestParameters.isForceDropDataset());
dataset.setPendingOp(MetadataUtil.PENDING_NO_OP);
MetadataManager.INSTANCE.addDataset(metadataProvider.getMetadataTxnContext(), dataset);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -894,7 +895,8 @@
mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
metadataProvider.setMetadataTxnContext(mdTxnCtx);
try {
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataverseName, datasetName);
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataverseName, datasetName,
+ requestParameters.isForceDropDataset());
if (itemTypeAdded) {
MetadataManager.INSTANCE.dropDatatype(mdTxnCtx, itemTypeEntity.getDataverseName(),
itemTypeEntity.getDatatypeName());
@@ -1692,7 +1694,7 @@
validateDatasetState(metadataProvider, ds, sourceLoc);
ds.drop(metadataProvider, mdTxnCtx, jobsToExecute, bActiveTxn, progress, hcc, dropCorrespondingNodeGroup,
- sourceLoc, Collections.emptySet());
+ sourceLoc, Collections.emptySet(), requestParameters.isForceDropDataset());
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
return true;
@@ -1709,7 +1711,8 @@
if (ds != null) {
jobsToExecute.clear();
ds.drop(metadataProvider, mdTxnCtx, jobsToExecute, bActiveTxn, progress, hcc,
- dropCorrespondingNodeGroup, sourceLoc, EnumSet.of(DropOption.IF_EXISTS));
+ dropCorrespondingNodeGroup, sourceLoc, EnumSet.of(DropOption.IF_EXISTS),
+ requestParameters.isForceDropDataset());
}
for (JobSpecification jobSpec : jobsToExecute) {
JobUtils.runJob(hcc, jobSpec, true);
@@ -1724,7 +1727,7 @@
metadataProvider.setMetadataTxnContext(mdTxnCtx.getValue());
try {
MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(), dataverseName,
- datasetName);
+ datasetName, requestParameters.isForceDropDataset());
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
} catch (Exception e2) {
e.addSuppressed(e2);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
index 5ebc9ba..d5ea685 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
@@ -58,6 +58,7 @@
private final boolean multiStatement;
private final int statementCategoryRestrictionMask;
private final String statement;
+ private final boolean forceDropDataset;
public RequestParameters(IRequestReference requestReference, String statement, IResultSet resultSet,
ResultProperties resultProperties, Stats stats, StatementProperties statementProperties,
@@ -72,6 +73,16 @@
IStatementExecutor.ResultMetadata outMetadata, String clientContextId,
Map<String, String> optionalParameters, Map<String, IAObject> statementParameters, boolean multiStatement,
int statementCategoryRestrictionMask) {
+ this(requestReference, statement, resultSet, resultProperties, stats, statementProperties, outMetadata,
+ clientContextId, optionalParameters, statementParameters, multiStatement,
+ statementCategoryRestrictionMask, false);
+ }
+
+ public RequestParameters(IRequestReference requestReference, String statement, IResultSet resultSet,
+ ResultProperties resultProperties, Stats stats, StatementProperties statementProperties,
+ IStatementExecutor.ResultMetadata outMetadata, String clientContextId,
+ Map<String, String> optionalParameters, Map<String, IAObject> statementParameters, boolean multiStatement,
+ int statementCategoryRestrictionMask, boolean forceDropDataset) {
this.requestReference = requestReference;
this.statement = statement;
this.resultSet = resultSet;
@@ -84,6 +95,7 @@
this.statementParameters = statementParameters;
this.multiStatement = multiStatement;
this.statementCategoryRestrictionMask = statementCategoryRestrictionMask;
+ this.forceDropDataset = forceDropDataset;
}
@Override
@@ -127,6 +139,11 @@
}
@Override
+ public boolean isForceDropDataset() {
+ return forceDropDataset;
+ }
+
+ @Override
public int getStatementCategoryRestrictionMask() {
return statementCategoryRestrictionMask;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java
index d32bfb2..7ad90ad 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java
@@ -419,7 +419,7 @@
// drop dataset entry from metadata
runMetadataTransaction(metadataProvider,
() -> MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(),
- dataset.getDataverseName(), dataset.getDatasetName()));
+ dataset.getDataverseName(), dataset.getDatasetName(), false));
MetadataManager.INSTANCE.commitTransaction(metadataProvider.getMetadataTxnContext());
// try to drop the dataset's node group
runMetadataTransaction(metadataProvider, () -> tryDropDatasetNodegroup(dataset, metadataProvider));
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
index 268a9c0..a481f99 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
@@ -260,10 +260,10 @@
}
@Override
- public void dropDataset(MetadataTransactionContext ctx, DataverseName dataverseName, String datasetName)
- throws AlgebricksException {
+ public void dropDataset(MetadataTransactionContext ctx, DataverseName dataverseName, String datasetName,
+ boolean force) throws AlgebricksException {
try {
- metadataNode.dropDataset(ctx.getTxnId(), dataverseName, datasetName);
+ metadataNode.dropDataset(ctx.getTxnId(), dataverseName, datasetName, force);
} catch (RemoteException e) {
throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e);
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index 88098f8..c80f7ad 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -605,10 +605,6 @@
}
@Override
- public void dropDataset(TxnId txnId, DataverseName dataverseName, String datasetName) throws AlgebricksException {
- dropDataset(txnId, dataverseName, datasetName, false);
- }
-
public void dropDataset(TxnId txnId, DataverseName dataverseName, String datasetName, boolean force)
throws AlgebricksException {
if (!force) {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
index 1646a93..139216e 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
@@ -212,10 +212,12 @@
* Name of dataverse which holds the given dataset.
* @param datasetName
* Name of dataset to delete.
+ * @param force
+ * If true, forces drop the dataset. Setting it to true could make the metadata inconsistent.
* @throws AlgebricksException
* For example, if the dataset and/or dataverse does not exist.
*/
- void dropDataset(MetadataTransactionContext ctx, DataverseName dataverseName, String datasetName)
+ void dropDataset(MetadataTransactionContext ctx, DataverseName dataverseName, String datasetName, boolean force)
throws AlgebricksException;
/**
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
index bca4171..08ed0a3 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
@@ -217,11 +217,13 @@
* Name of dataverse which holds the given dataset.
* @param datasetName
* Name of dataset to delete.
+ * @param force
+ * If true, forces drop the dataset. Setting it to true could make the metadata inconsistent.
* @throws AlgebricksException
* For example, if the dataset and/or dataverse does not exist.
* @throws RemoteException
*/
- void dropDataset(TxnId txnId, DataverseName dataverseName, String datasetName)
+ void dropDataset(TxnId txnId, DataverseName dataverseName, String datasetName, boolean force)
throws AlgebricksException, RemoteException;
/**
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index 2aca994..8cfd146 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -471,7 +471,7 @@
throws AlgebricksException {
if (dataset.getPendingOp() != MetadataUtil.PENDING_NO_OP) {
// drop pending dataset
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataset.getDataverseName(), dataset.getDatasetName());
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataset.getDataverseName(), dataset.getDatasetName(), true);
if (LOGGER.isInfoEnabled()) {
LOGGER.info(
"Dropped a pending dataset: " + dataset.getDataverseName() + "." + dataset.getDatasetName());
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index 4e556a2..dd3f67d 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -347,7 +347,7 @@
public void drop(MetadataProvider metadataProvider, MutableObject<MetadataTransactionContext> mdTxnCtx,
List<JobSpecification> jobsToExecute, MutableBoolean bActiveTxn, MutableObject<ProgressState> progress,
IHyracksClientConnection hcc, boolean dropCorrespondingNodeGroup, SourceLocation sourceLoc,
- Set<DropOption> options) throws Exception {
+ Set<DropOption> options, boolean force) throws Exception {
Map<FeedConnectionId, Pair<JobSpecification, Boolean>> disconnectJobList = new HashMap<>();
if (getDatasetType() == DatasetType.INTERNAL) {
// #. prepare jobs to drop the datatset and the indexes in NC
@@ -361,7 +361,7 @@
}
jobsToExecute.add(DatasetUtil.dropDatasetJobSpec(this, metadataProvider, options));
// #. mark the existing dataset as PendingDropOp
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName);
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName, force);
MetadataManager.INSTANCE.addDataset(mdTxnCtx.getValue(),
new Dataset(dataverseName, datasetName, getItemTypeDataverseName(), getItemTypeName(),
getMetaItemTypeDataverseName(), getMetaItemTypeName(), getNodeGroupName(),
@@ -401,7 +401,7 @@
}
// #. mark the existing dataset as PendingDropOp
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName);
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName, force);
MetadataManager.INSTANCE.addDataset(mdTxnCtx.getValue(),
new Dataset(dataverseName, datasetName, getItemTypeDataverseName(), getItemTypeName(),
getNodeGroupName(), getCompactionPolicy(), getCompactionPolicyProperties(),
@@ -425,7 +425,7 @@
}
// #. finally, delete the dataset.
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName);
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName, force);
// drop inline types
if (TypeUtil.isDatasetInlineTypeName(this, recordTypeDataverseName, recordTypeName)) {