[ASTERIXDB-3259][MTD] Enable using database level for cloud deployment
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
For cloud deployment, the 'database' level will be used. The metadata
catalog will contain System.Metadata.Database collection that has
2 predefined databases, 'System' and 'Default'.
- Pass MetadataIndexesProvider to IExtensionMetadataSearchKey to
allow extensions to create search keys based on whether database
is used or not.
- Change all entities to return the database-aware type when
requested.
Change-Id: I25956982977e2e73c52265e8a6c43c25b5504f78
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17816
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index e7d3a2f..9d32c61 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -338,7 +338,7 @@
throws AlgebricksException {
ExtensionMetadataDataset<T> index = getExtensionMetadataDataset(searchKey.getDatasetId());
IMetadataEntityTupleTranslator<T> tupleTranslator = index.getTupleTranslator(false);
- return getEntities(txnId, searchKey.getSearchKey(), tupleTranslator, index);
+ return getEntities(txnId, searchKey.getSearchKey(mdIndexesProvider), tupleTranslator, index);
}
private <T extends IExtensionMetadataEntity> ExtensionMetadataDataset<T> getExtensionMetadataDataset(
@@ -1314,14 +1314,16 @@
}
private void confirmFunctionIsUnusedByViews(TxnId txnId, FunctionSignature signature) throws AlgebricksException {
- confirmObjectIsUnusedByViews(txnId, "function", DependencyKind.FUNCTION, null, signature.getDataverseName(),
- signature.getName(), Integer.toString(signature.getArity()));
+ String functionDatabase = MetadataUtil.resolveDatabase(null, signature.getDataverseName());
+ confirmObjectIsUnusedByViews(txnId, "function", DependencyKind.FUNCTION, functionDatabase,
+ signature.getDataverseName(), signature.getName(), Integer.toString(signature.getArity()));
}
private void confirmFunctionIsUnusedByFunctions(TxnId txnId, FunctionSignature signature)
throws AlgebricksException {
- confirmObjectIsUnusedByFunctions(txnId, "function", DependencyKind.FUNCTION, null, signature.getDataverseName(),
- signature.getName(), Integer.toString(signature.getArity()));
+ String functionDatabase = MetadataUtil.resolveDatabase(null, signature.getDataverseName());
+ confirmObjectIsUnusedByFunctions(txnId, "function", DependencyKind.FUNCTION, functionDatabase,
+ signature.getDataverseName(), signature.getName(), Integer.toString(signature.getArity()));
}
private void confirmObjectIsUnusedByFunctions(TxnId txnId, String objectKindDisplayName,
@@ -1678,8 +1680,10 @@
@Override
public Function getFunction(TxnId txnId, FunctionSignature functionSignature) throws AlgebricksException {
- List<Function> functions = getFunctionsImpl(txnId, createTuple(null, functionSignature.getDataverseName(),
- functionSignature.getName(), Integer.toString(functionSignature.getArity())));
+ String functionDatabase = MetadataUtil.resolveDatabase(null, functionSignature.getDataverseName());
+ List<Function> functions =
+ getFunctionsImpl(txnId, createTuple(functionDatabase, functionSignature.getDataverseName(),
+ functionSignature.getName(), Integer.toString(functionSignature.getArity())));
return functions.isEmpty() ? null : functions.get(0);
}
@@ -1714,7 +1718,8 @@
}
try {
// Delete entry from the 'function' dataset.
- ITupleReference searchKey = createTuple(null, functionSignature.getDataverseName(),
+ String functionDatabase = MetadataUtil.resolveDatabase(null, functionSignature.getDataverseName());
+ ITupleReference searchKey = createTuple(functionDatabase, functionSignature.getDataverseName(),
functionSignature.getName(), Integer.toString(functionSignature.getArity()));
// Searches the index for the tuple to be deleted. Acquires an S lock on the 'function' dataset.
ITupleReference functionTuple =
@@ -1940,7 +1945,7 @@
}
}
- private static ITupleReference createDatabaseTuple(String databaseName, DataverseName dataverseName,
+ public static ITupleReference createDatabaseTuple(String databaseName, DataverseName dataverseName,
String... rest) {
try {
ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(2 + rest.length);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IExtensionMetadataSearchKey.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IExtensionMetadataSearchKey.java
index e8e39e3..ab554ea 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IExtensionMetadataSearchKey.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IExtensionMetadataSearchKey.java
@@ -20,6 +20,7 @@
import java.io.Serializable;
+import org.apache.asterix.metadata.bootstrap.MetadataIndexesProvider;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
/**
@@ -39,5 +40,5 @@
*
* @return the search key as a tuple reference
*/
- ITupleReference getSearchKey();
+ ITupleReference getSearchKey(MetadataIndexesProvider mdIndexesProvider);
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/CompactionPolicyEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/CompactionPolicyEntity.java
index b0d335e..58d1c5a 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/CompactionPolicyEntity.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/CompactionPolicyEntity.java
@@ -64,8 +64,8 @@
this.classNameIndex = startIndex++;
}
- public static CompactionPolicyEntity of(boolean cloudDeployment) {
- return COMPACTION_POLICY;
+ public static CompactionPolicyEntity of(boolean usingDatabase) {
+ return usingDatabase ? DB_COMPACTION_POLICY : COMPACTION_POLICY;
}
public MetadataIndex getIndex() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/DatasetEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/DatasetEntity.java
index 0dd4cde..546f66c 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/DatasetEntity.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/DatasetEntity.java
@@ -105,8 +105,8 @@
this.pendingOpIndex = startIndex++;
}
- public static DatasetEntity of(boolean cloudDeployment) {
- return DATASET;
+ public static DatasetEntity of(boolean usingDatabase) {
+ return usingDatabase ? DB_DATASET : DATASET;
}
public MetadataIndex getIndex() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/DatasourceAdapterEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/DatasourceAdapterEntity.java
index 25791a0..aa3024c 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/DatasourceAdapterEntity.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/DatasourceAdapterEntity.java
@@ -71,8 +71,8 @@
this.timestampIndex = startIndex++;
}
- public static DatasourceAdapterEntity of(boolean cloudDeployment) {
- return DATASOURCE_ADAPTER;
+ public static DatasourceAdapterEntity of(boolean usingDatabase) {
+ return usingDatabase ? DB_DATASOURCE_ADAPTER : DATASOURCE_ADAPTER;
}
public MetadataIndex getIndex() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/DatatypeEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/DatatypeEntity.java
index 605eef6..bb85db6 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/DatatypeEntity.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/DatatypeEntity.java
@@ -70,8 +70,8 @@
this.timestampIndex = startIndex++;
}
- public static DatatypeEntity of(boolean cloudDeployment) {
- return DATATYPE;
+ public static DatatypeEntity of(boolean usingDatabase) {
+ return usingDatabase ? DB_DATATYPE : DATATYPE;
}
public MetadataIndex getIndex() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/DataverseEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/DataverseEntity.java
index ca658d2..b6c6364 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/DataverseEntity.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/DataverseEntity.java
@@ -65,8 +65,8 @@
this.pendingOpIndex = startIndex++;
}
- public static DataverseEntity of(boolean cloudDeployment) {
- return DATAVERSE;
+ public static DataverseEntity of(boolean usingDatabase) {
+ return usingDatabase ? DB_DATAVERSE : DATAVERSE;
}
public MetadataIndex getIndex() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/ExternalFileEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/ExternalFileEntity.java
index 47ae4d1..c93c4d7 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/ExternalFileEntity.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/ExternalFileEntity.java
@@ -78,8 +78,8 @@
this.pendingOpIndex = startIndex++;
}
- public static ExternalFileEntity of(boolean cloudDeployment) {
- return EXTERNAL_FILE;
+ public static ExternalFileEntity of(boolean usingDatabase) {
+ return usingDatabase ? DB_EXTERNAL_FILE : EXTERNAL_FILE;
}
public MetadataIndex getIndex() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FeedConnectionEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FeedConnectionEntity.java
index 710afca..f675869 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FeedConnectionEntity.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FeedConnectionEntity.java
@@ -76,8 +76,8 @@
this.policyIndex = startIndex++;
}
- public static FeedConnectionEntity of(boolean cloudDeployment) {
- return FEED_CONNECTION;
+ public static FeedConnectionEntity of(boolean usingDatabase) {
+ return usingDatabase ? DB_FEED_CONNECTION : FEED_CONNECTION;
}
public MetadataIndex getIndex() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FeedEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FeedEntity.java
index d82c298..c690bb3 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FeedEntity.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FeedEntity.java
@@ -71,8 +71,8 @@
this.timestampIndex = startIndex++;
}
- public static FeedEntity of(boolean cloudDeployment) {
- return FEED;
+ public static FeedEntity of(boolean usingDatabase) {
+ return usingDatabase ? DB_FEED : FEED;
}
public MetadataIndex getIndex() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FeedPolicyEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FeedPolicyEntity.java
index c22977c..c9ddb93 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FeedPolicyEntity.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FeedPolicyEntity.java
@@ -70,8 +70,8 @@
this.propertiesIndex = startIndex++;
}
- public static FeedPolicyEntity of(boolean cloudDeployment) {
- return FEED_POLICY;
+ public static FeedPolicyEntity of(boolean usingDatabase) {
+ return usingDatabase ? DB_FEED_POLICY : FEED_POLICY;
}
public MetadataIndex getIndex() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FullTextConfigEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FullTextConfigEntity.java
index 6d4e331..ac1a2eb 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FullTextConfigEntity.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FullTextConfigEntity.java
@@ -69,8 +69,8 @@
this.filterPipelineIndex = startIndex++;;
}
- public static FullTextConfigEntity of(boolean cloudDeployment) {
- return FULL_TEXT_CONFIG;
+ public static FullTextConfigEntity of(boolean usingDatabase) {
+ return usingDatabase ? DB_FULL_TEXT_CONFIG : FULL_TEXT_CONFIG;
}
public MetadataIndex getIndex() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FullTextFilterEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FullTextFilterEntity.java
index 49b3334..7942da6 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FullTextFilterEntity.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FullTextFilterEntity.java
@@ -64,8 +64,8 @@
this.filterTypeIndex = startIndex++;
}
- public static FullTextFilterEntity of(boolean cloudDeployment) {
- return FULL_TEXT_CONFIG;
+ public static FullTextFilterEntity of(boolean usingDatabase) {
+ return usingDatabase ? DB_FULL_TEXT_CONFIG : FULL_TEXT_CONFIG;
}
public MetadataIndex getIndex() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FunctionEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FunctionEntity.java
index a665f35..45387d5 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FunctionEntity.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/FunctionEntity.java
@@ -81,8 +81,8 @@
this.functionDependenciesIndex = startIndex++;
}
- public static FunctionEntity of(boolean cloudDeployment) {
- return FUNCTION;
+ public static FunctionEntity of(boolean usingDatabase) {
+ return usingDatabase ? DB_FUNCTION : FUNCTION;
}
public MetadataIndex getIndex() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/IndexEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/IndexEntity.java
index fd23dd0..bd2c4c0 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/IndexEntity.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/IndexEntity.java
@@ -82,8 +82,8 @@
this.pendingOpIndex = startIndex++;
}
- public static IndexEntity of(boolean cloudDeployment) {
- return INDEX;
+ public static IndexEntity of(boolean usingDatabase) {
+ return usingDatabase ? DB_INDEX : INDEX;
}
public MetadataIndex getIndex() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/LibraryEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/LibraryEntity.java
index 692d35e..98b8a63 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/LibraryEntity.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/LibraryEntity.java
@@ -66,8 +66,8 @@
this.timestampIndex = startIndex++;
}
- public static LibraryEntity of(boolean cloudDeployment) {
- return LIBRARY;
+ public static LibraryEntity of(boolean usingDatabase) {
+ return usingDatabase ? DB_LIBRARY : LIBRARY;
}
public MetadataIndex getIndex() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndexesProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndexesProvider.java
index 8c19505..7dadde3 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndexesProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndexesProvider.java
@@ -25,82 +25,82 @@
public class MetadataIndexesProvider {
- protected final boolean cloudDeployment;
+ protected final boolean usingDatabase;
public MetadataIndexesProvider(INCServiceContext ncServiceCtx) {
- cloudDeployment = ncServiceCtx.getAppConfig().getBoolean(CLOUD_DEPLOYMENT);
+ usingDatabase = ncServiceCtx.getAppConfig().getBoolean(CLOUD_DEPLOYMENT);
}
public DatabaseEntity getDatabaseEntity() {
- return DatabaseEntity.of(cloudDeployment);
+ return DatabaseEntity.of(usingDatabase);
}
public DataverseEntity getDataverseEntity() {
- return DataverseEntity.of(cloudDeployment);
+ return DataverseEntity.of(usingDatabase);
}
public DatasetEntity getDatasetEntity() {
- return DatasetEntity.of(cloudDeployment);
+ return DatasetEntity.of(usingDatabase);
}
public DatatypeEntity getDatatypeEntity() {
- return DatatypeEntity.of(cloudDeployment);
+ return DatatypeEntity.of(usingDatabase);
}
public IndexEntity getIndexEntity() {
- return IndexEntity.of(cloudDeployment);
+ return IndexEntity.of(usingDatabase);
}
public SynonymEntity getSynonymEntity() {
- return SynonymEntity.of(cloudDeployment);
+ return SynonymEntity.of(usingDatabase);
}
public NodeEntity getNodeEntity() {
- return NodeEntity.of(cloudDeployment);
+ return NodeEntity.of(usingDatabase);
}
public NodeGroupEntity getNodeGroupEntity() {
- return NodeGroupEntity.of(cloudDeployment);
+ return NodeGroupEntity.of(usingDatabase);
}
public FunctionEntity getFunctionEntity() {
- return FunctionEntity.of(cloudDeployment);
+ return FunctionEntity.of(usingDatabase);
}
public DatasourceAdapterEntity getDatasourceAdapterEntity() {
- return DatasourceAdapterEntity.of(cloudDeployment);
+ return DatasourceAdapterEntity.of(usingDatabase);
}
public FeedEntity getFeedEntity() {
- return FeedEntity.of(cloudDeployment);
+ return FeedEntity.of(usingDatabase);
}
public FeedPolicyEntity getFeedPolicyEntity() {
- return FeedPolicyEntity.of(cloudDeployment);
+ return FeedPolicyEntity.of(usingDatabase);
}
public LibraryEntity getLibraryEntity() {
- return LibraryEntity.of(cloudDeployment);
+ return LibraryEntity.of(usingDatabase);
}
public CompactionPolicyEntity getCompactionPolicyEntity() {
- return CompactionPolicyEntity.of(cloudDeployment);
+ return CompactionPolicyEntity.of(usingDatabase);
}
public ExternalFileEntity getExternalFileEntity() {
- return ExternalFileEntity.of(cloudDeployment);
+ return ExternalFileEntity.of(usingDatabase);
}
public FeedConnectionEntity getFeedConnectionEntity() {
- return FeedConnectionEntity.of(cloudDeployment);
+ return FeedConnectionEntity.of(usingDatabase);
}
public FullTextConfigEntity getFullTextConfigEntity() {
- return FullTextConfigEntity.of(cloudDeployment);
+ return FullTextConfigEntity.of(usingDatabase);
}
public FullTextFilterEntity getFullTextFilterEntity() {
- return FullTextFilterEntity.of(cloudDeployment);
+ return FullTextFilterEntity.of(usingDatabase);
}
public IMetadataIndex[] getMetadataIndexes() {
@@ -126,6 +126,6 @@
}
public boolean isUsingDatabase() {
- return false;
+ return usingDatabase;
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/SynonymEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/SynonymEntity.java
index ba44f44..34ecc59 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/SynonymEntity.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/SynonymEntity.java
@@ -68,8 +68,8 @@
this.objectNameIndex = startIndex++;;
}
- public static SynonymEntity of(boolean cloudDeployment) {
- return SYNONYM;
+ public static SynonymEntity of(boolean usingDatabase) {
+ return usingDatabase ? DB_SYNONYM : SYNONYM;
}
public MetadataIndex getIndex() {