Merge branch 'gerrit/cheshire-cat'
Change-Id: I3bb823f8b37e6825de7c7ef94cd760e8fc3b5718
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
index 417a130..4ad1040 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
@@ -67,4 +67,10 @@
* {@code 0} if all categories are allowed
*/
int getStatementCategoryRestrictionMask();
+
+ /**
+ * @return true if DROP DATASET statements in the request should force drop datasets which could make the metadata
+ * inconsistent.
+ */
+ boolean isForceDropDataset();
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index d43e611..6de91d8 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -90,6 +90,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.api.exceptions.Warning;
+import org.apache.hyracks.api.result.IResultSet;
import org.apache.hyracks.control.common.controllers.CCConfig;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
@@ -269,7 +270,7 @@
SessionOutput sessionOutput = createSessionOutput(httpWriter);
ResponsePrinter responsePrinter = new ResponsePrinter(sessionOutput);
ResultDelivery delivery = ResultDelivery.IMMEDIATE;
- QueryServiceRequestParameters param = newRequestParameters();
+ QueryServiceRequestParameters param = newQueryRequestParameters();
RequestExecutionState executionState = newRequestExecutionState();
try {
// buffer the output until we are ready to set the status of the response message correctly
@@ -411,9 +412,8 @@
int stmtCategoryRestriction = org.apache.asterix.app.translator.RequestParameters
.getStatementCategoryRestrictionMask(param.isReadOnly());
IRequestParameters requestParameters =
- new org.apache.asterix.app.translator.RequestParameters(requestReference, statementsText,
- getResultSet(), resultProperties, stats, statementProperties, null, param.getClientContextID(),
- optionalParameters, stmtParams, param.isMultiStatement(), stmtCategoryRestriction);
+ newRequestParameters(param, requestReference, statementsText, getResultSet(), resultProperties, stats,
+ statementProperties, optionalParameters, stmtParams, stmtCategoryRestriction);
translator.compileAndExecute(getHyracksClientConnection(), requestParameters);
executionState.end();
translator.getWarnings(warnings, maxWarnings - warnings.size());
@@ -492,10 +492,19 @@
responsePrinter.addResultPrinter(new ErrorsPrinter(Collections.singletonList(executionError)));
}
- protected QueryServiceRequestParameters newRequestParameters() {
+ protected QueryServiceRequestParameters newQueryRequestParameters() {
return new QueryServiceRequestParameters();
}
+ protected IRequestParameters newRequestParameters(QueryServiceRequestParameters param,
+ IRequestReference requestReference, String statementsText, IResultSet resultSet,
+ ResultProperties resultProperties, Stats stats, IStatementExecutor.StatementProperties statementProperties,
+ Map<String, String> optionalParameters, Map<String, IAObject> stmtParams, int stmtCategoryRestriction) {
+ return new RequestParameters(requestReference, statementsText, resultSet, resultProperties, stats,
+ statementProperties, null, param.getClientContextID(), optionalParameters, stmtParams,
+ param.isMultiStatement(), stmtCategoryRestriction);
+ }
+
protected static boolean isPrintingProfile(IStatementExecutor.Stats stats) {
return stats.getProfileType() == Stats.ProfileType.FULL && stats.getJobProfile() != null;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 3e079af..69507bc 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -885,7 +885,8 @@
// #. add a new dataset with PendingNoOp after deleting the dataset with
// PendingAddOp
- MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(), dataverseName, datasetName);
+ MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(), dataverseName, datasetName,
+ requestParameters.isForceDropDataset());
dataset.setPendingOp(MetadataUtil.PENDING_NO_OP);
MetadataManager.INSTANCE.addDataset(metadataProvider.getMetadataTxnContext(), dataset);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -922,7 +923,8 @@
mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
metadataProvider.setMetadataTxnContext(mdTxnCtx);
try {
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataverseName, datasetName);
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataverseName, datasetName,
+ requestParameters.isForceDropDataset());
if (itemTypeAdded) {
MetadataManager.INSTANCE.dropDatatype(mdTxnCtx, itemTypeEntity.getDataverseName(),
itemTypeEntity.getDatatypeName());
@@ -1852,7 +1854,7 @@
validateDatasetState(metadataProvider, ds, sourceLoc);
ds.drop(metadataProvider, mdTxnCtx, jobsToExecute, bActiveTxn, progress, hcc, dropCorrespondingNodeGroup,
- sourceLoc, Collections.emptySet());
+ sourceLoc, Collections.emptySet(), requestParameters.isForceDropDataset());
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
return true;
@@ -1869,7 +1871,8 @@
if (ds != null) {
jobsToExecute.clear();
ds.drop(metadataProvider, mdTxnCtx, jobsToExecute, bActiveTxn, progress, hcc,
- dropCorrespondingNodeGroup, sourceLoc, EnumSet.of(DropOption.IF_EXISTS));
+ dropCorrespondingNodeGroup, sourceLoc, EnumSet.of(DropOption.IF_EXISTS),
+ requestParameters.isForceDropDataset());
}
for (JobSpecification jobSpec : jobsToExecute) {
JobUtils.runJob(hcc, jobSpec, true);
@@ -1884,7 +1887,7 @@
metadataProvider.setMetadataTxnContext(mdTxnCtx.getValue());
try {
MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(), dataverseName,
- datasetName);
+ datasetName, requestParameters.isForceDropDataset());
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
} catch (Exception e2) {
e.addSuppressed(e2);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
index 5ebc9ba..d5ea685 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
@@ -58,6 +58,7 @@
private final boolean multiStatement;
private final int statementCategoryRestrictionMask;
private final String statement;
+ private final boolean forceDropDataset;
public RequestParameters(IRequestReference requestReference, String statement, IResultSet resultSet,
ResultProperties resultProperties, Stats stats, StatementProperties statementProperties,
@@ -72,6 +73,16 @@
IStatementExecutor.ResultMetadata outMetadata, String clientContextId,
Map<String, String> optionalParameters, Map<String, IAObject> statementParameters, boolean multiStatement,
int statementCategoryRestrictionMask) {
+ this(requestReference, statement, resultSet, resultProperties, stats, statementProperties, outMetadata,
+ clientContextId, optionalParameters, statementParameters, multiStatement,
+ statementCategoryRestrictionMask, false);
+ }
+
+ public RequestParameters(IRequestReference requestReference, String statement, IResultSet resultSet,
+ ResultProperties resultProperties, Stats stats, StatementProperties statementProperties,
+ IStatementExecutor.ResultMetadata outMetadata, String clientContextId,
+ Map<String, String> optionalParameters, Map<String, IAObject> statementParameters, boolean multiStatement,
+ int statementCategoryRestrictionMask, boolean forceDropDataset) {
this.requestReference = requestReference;
this.statement = statement;
this.resultSet = resultSet;
@@ -84,6 +95,7 @@
this.statementParameters = statementParameters;
this.multiStatement = multiStatement;
this.statementCategoryRestrictionMask = statementCategoryRestrictionMask;
+ this.forceDropDataset = forceDropDataset;
}
@Override
@@ -127,6 +139,11 @@
}
@Override
+ public boolean isForceDropDataset() {
+ return forceDropDataset;
+ }
+
+ @Override
public int getStatementCategoryRestrictionMask() {
return statementCategoryRestrictionMask;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java
index d32bfb2..7ad90ad 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java
@@ -419,7 +419,7 @@
// drop dataset entry from metadata
runMetadataTransaction(metadataProvider,
() -> MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(),
- dataset.getDataverseName(), dataset.getDatasetName()));
+ dataset.getDataverseName(), dataset.getDatasetName(), false));
MetadataManager.INSTANCE.commitTransaction(metadataProvider.getMetadataTxnContext());
// try to drop the dataset's node group
runMetadataTransaction(metadataProvider, () -> tryDropDatasetNodegroup(dataset, metadataProvider));
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 614b0a9..00d1034 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -215,6 +215,7 @@
protected IExternalUDFLibrarian librarian;
private Map<File, TestLoop> testLoops = new HashMap<>();
private double timeoutMultiplier = 1;
+ protected int loopIteration;
public TestExecutor() {
this(Inet4Address.getLoopbackAddress().getHostAddress(), 19002);
@@ -1272,6 +1273,9 @@
throw new IllegalArgumentException("duration cannot be exceed 1d");
}
break;
+ case "":
+ // ignore blank lines;
+ break;
default:
throw new IllegalArgumentException("unknown directive: " + command[0]);
}
@@ -1756,7 +1760,7 @@
String name = m.group("name");
param.setName(name);
String value = m.group("value");
- param.setValue(value);
+ param.setValue(value.replace("\\n", "\n"));
String type = m.group("type");
if (type != null) {
try {
@@ -1896,6 +1900,7 @@
}
List<TestFileContext> expectedResultFileCtxs = testCaseCtx.getExpectedResultFiles(cUnit);
int[] savedQueryCounts = new int[numOfFiles + testFileCtxs.size()];
+ loopIteration = 0;
for (ListIterator<TestFileContext> iter = testFileCtxs.listIterator(); iter.hasNext();) {
TestFileContext ctx = iter.next();
savedQueryCounts[numOfFiles] = queryCount.getValue();
@@ -1903,10 +1908,15 @@
final File testFile = ctx.getFile();
final String statement = readTestFile(testFile);
try {
+ boolean loopCmd = testFile.getName().endsWith(".loop.cmd");
if (!testFile.getName().startsWith(DIAGNOSE)) {
executeTestFile(testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit,
queryCount, expectedResultFileCtxs, testFile, actualPath, expectedWarnings);
}
+ if (loopCmd) {
+ // this was a loop file and we have exited the loop; reset the loop iteration
+ loopIteration = 0;
+ }
} catch (TestLoop loop) {
// rewind the iterator until we find our target
while (!ctx.getFile().getName().equals(loop.getTarget())) {
@@ -1917,6 +1927,7 @@
numOfFiles--;
queryCount.setValue(savedQueryCounts[numOfFiles]);
}
+ loopIteration++;
} catch (Exception e) {
numOfErrors++;
boolean unexpected = isUnExpected(e, expectedErrors, numOfErrors, queryCount,
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerTest.java
index 9045680..164ea7f 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerTest.java
@@ -18,11 +18,20 @@
*/
package org.apache.asterix.test.metadata;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.test.common.TestExecutor;
import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.commons.lang3.StringUtils;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -61,4 +70,94 @@
testExecutor.executeSqlppUpdateOrDdl("drop dataverse test;", cleanJson);
testExecutor.executeSqlppUpdateOrDdl(sql.toString(), cleanJson);
}
+
+ @Test
+ public void testDataverseNameLimits() throws Exception {
+ TestCaseContext.OutputFormat cleanJson = TestCaseContext.OutputFormat.CLEAN_JSON;
+
+ // at max dataverse name limits
+
+ char auml = 228, euml = 235;
+
+ List<DataverseName> dvNameOkList =
+ Arrays.asList(
+ // #1. max single-part name
+ DataverseName.createSinglePartName(
+ StringUtils.repeat('a', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8)),
+ // #2. max single-part name (2-byte characters)
+ DataverseName.createSinglePartName(
+ StringUtils.repeat(auml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2)),
+ // #3. 4 max parts
+ DataverseName.create(Arrays.asList(
+ StringUtils.repeat('a', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8),
+ StringUtils.repeat('b', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8),
+ StringUtils.repeat('c', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8),
+ StringUtils.repeat('d', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8))),
+ // #3. 4 max parts (2-byte characters)
+ DataverseName.create(Arrays.asList(
+ StringUtils.repeat(auml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2),
+ StringUtils.repeat(euml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2),
+ StringUtils.repeat(auml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2),
+ StringUtils.repeat(euml,
+ MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2))));
+
+ for (DataverseName dvNameOk : dvNameOkList) {
+ String sql = String.format("create dataverse %s;", dvNameOk);
+ testExecutor.executeSqlppUpdateOrDdl(sql, cleanJson);
+ }
+
+ // exceeding dataverse name limits
+
+ char iuml = 239, ouml = 246;
+
+ List<DataverseName> dvNameErrList =
+ Arrays.asList(
+ // #1. single-part name exceeds part length limit
+ DataverseName.createSinglePartName(
+ StringUtils.repeat('A', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 + 1)),
+ // #2 single-part name exceeds part length limit (2-byte characters)
+ DataverseName.createSinglePartName(StringUtils.repeat(iuml,
+ MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2 + 1)),
+ // #3. 2-part name, 2nd part exceed part length limit
+ DataverseName.create(Arrays.asList("A",
+ StringUtils.repeat('B', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 + 1))),
+ // #4. 2-part name, 2nd part exceed part length limit (2-byte characters)
+ DataverseName.create(Arrays.asList("A",
+ StringUtils.repeat(ouml,
+ MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2 + 1))),
+ // #5. 5-part name, each part at the part length limit, total length limit is exceeded
+ DataverseName.create(Arrays.asList(
+ StringUtils.repeat('A', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8),
+ StringUtils.repeat('B', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8),
+ StringUtils.repeat('C', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8),
+ StringUtils.repeat('D', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8),
+ StringUtils.repeat('E', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8))),
+ // #6. 5-part name, each part at the part length limit, total length limit is exceeded (2-byte characters)
+ DataverseName.create(Arrays.asList(
+ StringUtils.repeat(iuml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2),
+ StringUtils.repeat(ouml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2),
+ StringUtils.repeat(iuml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2),
+ StringUtils.repeat(ouml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2),
+ StringUtils.repeat(iuml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2))),
+ // #7. Multi-part name, each part at the part length limit, total length limit is exceeded
+ DataverseName.create(
+ Collections.nCopies(MetadataConstants.DATAVERSE_NAME_TOTAL_LENGTH_LIMIT_UTF8 + 1, "A")),
+ // #8. Multi-part name, each part at the part length limit, total length limit is exceeded (2-bytes characters)
+ DataverseName.create(
+ Collections.nCopies(MetadataConstants.DATAVERSE_NAME_TOTAL_LENGTH_LIMIT_UTF8 / 2 + 1,
+ String.valueOf(iuml))));
+
+ String invalidNameErrCode = ErrorCode.ASTERIX + ErrorCode.INVALID_DATABASE_OBJECT_NAME;
+ for (DataverseName dvNameErr : dvNameErrList) {
+ String sql = String.format("create dataverse %s;", dvNameErr);
+ try {
+ testExecutor.executeSqlppUpdateOrDdl(sql, cleanJson);
+ Assert.fail("Expected failure: " + invalidNameErrCode);
+ } catch (Exception e) {
+
+ Assert.assertTrue("Unexpected error message: " + e.getMessage(),
+ e.getMessage().contains(invalidNameErrCode));
+ }
+ }
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
index 798c604..20c852a 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
@@ -265,10 +265,10 @@
}
@Override
- public void dropDataset(MetadataTransactionContext ctx, DataverseName dataverseName, String datasetName)
- throws AlgebricksException {
+ public void dropDataset(MetadataTransactionContext ctx, DataverseName dataverseName, String datasetName,
+ boolean force) throws AlgebricksException {
try {
- metadataNode.dropDataset(ctx.getTxnId(), dataverseName, datasetName);
+ metadataNode.dropDataset(ctx.getTxnId(), dataverseName, datasetName, force);
} catch (RemoteException e) {
throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e);
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index c078e8b..449e8d2 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -765,10 +765,6 @@
}
@Override
- public void dropDataset(TxnId txnId, DataverseName dataverseName, String datasetName) throws AlgebricksException {
- dropDataset(txnId, dataverseName, datasetName, false);
- }
-
public void dropDataset(TxnId txnId, DataverseName dataverseName, String datasetName, boolean force)
throws AlgebricksException {
if (!force) {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
index 970a4e0..5ff8a03 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
@@ -214,10 +214,12 @@
* Name of dataverse which holds the given dataset.
* @param datasetName
* Name of dataset to delete.
+ * @param force
+ * If true, forces drop the dataset. Setting it to true could make the metadata inconsistent.
* @throws AlgebricksException
* For example, if the dataset and/or dataverse does not exist.
*/
- void dropDataset(MetadataTransactionContext ctx, DataverseName dataverseName, String datasetName)
+ void dropDataset(MetadataTransactionContext ctx, DataverseName dataverseName, String datasetName, boolean force)
throws AlgebricksException;
/**
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
index ef1f45d..561a4fa 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
@@ -219,11 +219,13 @@
* Name of dataverse which holds the given dataset.
* @param datasetName
* Name of dataset to delete.
+ * @param force
+ * If true, forces drop the dataset. Setting it to true could make the metadata inconsistent.
* @throws AlgebricksException
* For example, if the dataset and/or dataverse does not exist.
* @throws RemoteException
*/
- void dropDataset(TxnId txnId, DataverseName dataverseName, String datasetName)
+ void dropDataset(TxnId txnId, DataverseName dataverseName, String datasetName, boolean force)
throws AlgebricksException, RemoteException;
/**
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index 84da2e2..047b823 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -60,6 +60,7 @@
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.metadata.entities.InternalDatasetDetails.FileStructure;
import org.apache.asterix.metadata.entities.InternalDatasetDetails.PartitioningStrategy;
+import org.apache.asterix.metadata.entities.Library;
import org.apache.asterix.metadata.entities.Node;
import org.apache.asterix.metadata.entities.NodeGroup;
import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies;
@@ -491,6 +492,12 @@
for (Dataset dataset : datasets) {
recoverDataset(mdTxnCtx, dataset);
}
+
+ List<Library> libraries =
+ MetadataManager.INSTANCE.getDataverseLibraries(mdTxnCtx, dataverse.getDataverseName());
+ for (Library library : libraries) {
+ recoverLibrary(mdTxnCtx, library);
+ }
}
}
@@ -498,7 +505,7 @@
throws AlgebricksException {
if (dataset.getPendingOp() != MetadataUtil.PENDING_NO_OP) {
// drop pending dataset
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataset.getDataverseName(), dataset.getDatasetName());
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataset.getDataverseName(), dataset.getDatasetName(), true);
if (LOGGER.isInfoEnabled()) {
LOGGER.info(
"Dropped a pending dataset: " + dataset.getDataverseName() + "." + dataset.getDatasetName());
@@ -535,6 +542,17 @@
}
}
+ private static void recoverLibrary(MetadataTransactionContext mdTxnCtx, Library library)
+ throws AlgebricksException {
+ if (library.getPendingOp() != MetadataUtil.PENDING_NO_OP) {
+ // drop pending library
+ MetadataManager.INSTANCE.dropLibrary(mdTxnCtx, library.getDataverseName(), library.getName());
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info("Dropped a pending library: " + library.getDataverseName() + "." + library.getName());
+ }
+ }
+ }
+
public static boolean isNewUniverse() {
return isNewUniverse;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 47623fc..f87831d 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -20,6 +20,7 @@
import java.io.File;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -1777,8 +1778,17 @@
public void validateDataverseName(DataverseName dataverseName, SourceLocation sourceLoc)
throws AlgebricksException {
+ int totalLengthUTF8 = 0;
for (String dvNamePart : dataverseName.getParts()) {
validateDatabaseObjectNameImpl(dvNamePart, sourceLoc);
+ int lengthUTF8 = dvNamePart.getBytes(StandardCharsets.UTF_8).length;
+ if (lengthUTF8 > MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8) {
+ throw new AsterixException(ErrorCode.INVALID_DATABASE_OBJECT_NAME, sourceLoc, dvNamePart);
+ }
+ totalLengthUTF8 += lengthUTF8;
+ }
+ if (totalLengthUTF8 > MetadataConstants.DATAVERSE_NAME_TOTAL_LENGTH_LIMIT_UTF8) {
+ throw new AsterixException(ErrorCode.INVALID_DATABASE_OBJECT_NAME, sourceLoc, dataverseName.toString());
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index 4e556a2..dd3f67d 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -347,7 +347,7 @@
public void drop(MetadataProvider metadataProvider, MutableObject<MetadataTransactionContext> mdTxnCtx,
List<JobSpecification> jobsToExecute, MutableBoolean bActiveTxn, MutableObject<ProgressState> progress,
IHyracksClientConnection hcc, boolean dropCorrespondingNodeGroup, SourceLocation sourceLoc,
- Set<DropOption> options) throws Exception {
+ Set<DropOption> options, boolean force) throws Exception {
Map<FeedConnectionId, Pair<JobSpecification, Boolean>> disconnectJobList = new HashMap<>();
if (getDatasetType() == DatasetType.INTERNAL) {
// #. prepare jobs to drop the datatset and the indexes in NC
@@ -361,7 +361,7 @@
}
jobsToExecute.add(DatasetUtil.dropDatasetJobSpec(this, metadataProvider, options));
// #. mark the existing dataset as PendingDropOp
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName);
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName, force);
MetadataManager.INSTANCE.addDataset(mdTxnCtx.getValue(),
new Dataset(dataverseName, datasetName, getItemTypeDataverseName(), getItemTypeName(),
getMetaItemTypeDataverseName(), getMetaItemTypeName(), getNodeGroupName(),
@@ -401,7 +401,7 @@
}
// #. mark the existing dataset as PendingDropOp
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName);
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName, force);
MetadataManager.INSTANCE.addDataset(mdTxnCtx.getValue(),
new Dataset(dataverseName, datasetName, getItemTypeDataverseName(), getItemTypeName(),
getNodeGroupName(), getCompactionPolicy(), getCompactionPolicyProperties(),
@@ -425,7 +425,7 @@
}
// #. finally, delete the dataset.
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName);
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName, force);
// drop inline types
if (TypeUtil.isDatasetInlineTypeName(this, recordTypeDataverseName, recordTypeName)) {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java
index 2875297..90a0d6b 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java
@@ -26,6 +26,9 @@
*/
public class MetadataConstants {
+ public static final int DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 = 255;
+ public static final int DATAVERSE_NAME_TOTAL_LENGTH_LIMIT_UTF8 = 1023;
+
// Name of the dataverse the metadata lives in.
public static final DataverseName METADATA_DATAVERSE_NAME = DataverseName.createBuiltinDataverseName("Metadata");
// Name of the node group where metadata is stored on.
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/NetworkUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/NetworkUtil.java
index 1bab960..958310d 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/NetworkUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/NetworkUtil.java
@@ -66,10 +66,23 @@
return builderFrom(host).build();
}
+ /**
+ * @param host the host for this uri
+ * @param path the path for this uri: this value is expected to be unescaped and may contain non ASCII characters
+ * @return
+ * @throws URISyntaxException
+ */
public static URI toUri(HttpHost host, String path) throws URISyntaxException {
return builderFrom(host).setPath(path).build();
}
+ /**
+ * @param uri the uri to append to
+ * @param pathSegments the path segments to append to the supplied uri: the segments are expected to be
+ * unescaped and may contain non ASCII characters
+ * @return the new uri i.e. original uri with appended path segment(s)
+ * @throws URISyntaxException
+ */
public static URI appendUriPath(URI uri, String... pathSegments) throws URISyntaxException {
URIBuilder builder = new URIBuilder(uri);
List<String> path = builder.getPathSegments();
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
index 7d162ee..35a3629 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
@@ -43,6 +43,14 @@
return joinPath(File.separatorChar, elements);
}
+ public static String joinPathUnix(String... elements) {
+ return joinPath('/', elements);
+ }
+
+ public static String joinPathWindows(String... elements) {
+ return joinPath('\\', elements);
+ }
+
public static void forceMkdirs(File dir) throws IOException {
File canonicalDir = dir.getCanonicalFile();
try {