[ASTERIXDB-3259][MTD] Add namespace path resolver
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
Add namespace path resolver which given a database name
and a dataverse name will resolve the storage path to
be used depending on whether database is enabled or not.
- Pass the namespace path resolver to the data partitioning
provider.
Change-Id: I285be236fd634a8e036447dbecfd0e3039613907
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17846
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
index bb7be73..a79eb0d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
@@ -29,6 +29,8 @@
import org.apache.asterix.common.api.IConfigValidatorFactory;
import org.apache.asterix.common.api.ICoordinationService;
import org.apache.asterix.common.api.IMetadataLockManager;
+import org.apache.asterix.common.api.INamespacePathResolver;
+import org.apache.asterix.common.api.INamespaceResolver;
import org.apache.asterix.common.api.INodeJobTracker;
import org.apache.asterix.common.api.IReceptionist;
import org.apache.asterix.common.api.IReceptionistFactory;
@@ -55,6 +57,8 @@
import org.apache.asterix.common.external.IAdapterFactoryService;
import org.apache.asterix.common.metadata.IMetadataBootstrap;
import org.apache.asterix.common.metadata.IMetadataLockUtil;
+import org.apache.asterix.common.metadata.NamespacePathResolver;
+import org.apache.asterix.common.metadata.NamespaceResolver;
import org.apache.asterix.common.replication.INcLifecycleCoordinator;
import org.apache.asterix.common.storage.ICompressionManager;
import org.apache.asterix.common.transactions.IResourceIdManager;
@@ -123,6 +127,8 @@
private final IDataPartitioningProvider dataPartitioningProvider;
private final IGlobalTxManager globalTxManager;
private final IOManager ioManager;
+ private final NamespacePathResolver namespacePathResolver;
+ private final NamespaceResolver namespaceResolver;
public CcApplicationContext(ICCServiceContext ccServiceCtx, HyracksConnection hcc,
Supplier<IMetadataBootstrap> metadataBootstrapSupplier, IGlobalRecoveryManager globalRecoveryManager,
@@ -167,9 +173,11 @@
requestTracker = new RequestTracker(this);
configValidator = configValidatorFactory.create();
this.adapterFactoryService = adapterFactoryService;
- dataPartitioningProvider = DataPartitioningProvider.create(this);
+ this.namespacePathResolver = new NamespacePathResolver(isCloudDeployment());
+ this.namespaceResolver = new NamespaceResolver(isCloudDeployment());
this.globalTxManager = globalTxManager;
this.ioManager = ioManager;
+ dataPartitioningProvider = DataPartitioningProvider.create(this);
}
@Override
@@ -380,6 +388,16 @@
}
@Override
+ public INamespaceResolver getNamespaceResolver() {
+ return namespaceResolver;
+ }
+
+ @Override
+ public INamespacePathResolver getNamespacePathResolver() {
+ return namespacePathResolver;
+ }
+
+ @Override
public boolean isCloudDeployment() {
return ccServiceCtx.getAppConfig().getBoolean(CLOUD_DEPLOYMENT);
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index 9e796f2..2b869fd 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -38,6 +38,8 @@
import org.apache.asterix.common.api.ICoordinationService;
import org.apache.asterix.common.api.IDatasetLifecycleManager;
import org.apache.asterix.common.api.IDiskWriteRateLimiterProvider;
+import org.apache.asterix.common.api.INamespacePathResolver;
+import org.apache.asterix.common.api.INamespaceResolver;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.api.IPropertiesFactory;
import org.apache.asterix.common.api.IReceptionist;
@@ -59,6 +61,8 @@
import org.apache.asterix.common.context.GlobalVirtualBufferCache;
import org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.common.library.ILibraryManager;
+import org.apache.asterix.common.metadata.NamespacePathResolver;
+import org.apache.asterix.common.metadata.NamespaceResolver;
import org.apache.asterix.common.replication.IReplicationChannel;
import org.apache.asterix.common.replication.IReplicationManager;
import org.apache.asterix.common.replication.IReplicationStrategyFactory;
@@ -169,6 +173,8 @@
private IDiskWriteRateLimiterProvider diskWriteRateLimiterProvider;
private final CloudProperties cloudProperties;
private IPartitionBootstrapper partitionBootstrapper;
+ private final NamespacePathResolver namespacePathResolver;
+ private final NamespaceResolver namespaceResolver;
public NCAppRuntimeContext(INCServiceContext ncServiceContext, NCExtensionManager extensionManager,
IPropertiesFactory propertiesFactory) {
@@ -190,6 +196,8 @@
.createResourceIdFactory();
persistedResourceRegistry = ncServiceContext.getPersistedResourceRegistry();
cacheManager = new CacheManager();
+ namespacePathResolver = new NamespacePathResolver(isCloudDeployment());
+ namespaceResolver = new NamespaceResolver(isCloudDeployment());
}
@Override
@@ -466,6 +474,16 @@
}
@Override
+ public INamespaceResolver getNamespaceResolver() {
+ return namespaceResolver;
+ }
+
+ @Override
+ public INamespacePathResolver getNamespacePathResolver() {
+ return namespacePathResolver;
+ }
+
+ @Override
public void initializeMetadata(boolean newUniverse, int partitionId) throws Exception {
LOGGER.info("Bootstrapping ({}) metadata in partition {}", newUniverse ? "new" : "existing", partitionId);
MetadataNode.INSTANCE.initialize(this, ncExtensionManager.getMetadataIndexesProvider(),
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/DataverseUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/DataverseUtil.java
index 3b7cb00..d132b6b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/DataverseUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/DataverseUtil.java
@@ -31,24 +31,21 @@
private DataverseUtil() {
}
- public static JobSpecification dropDataverseJobSpec(Dataverse dataverse, MetadataProvider metadata) {
- JobSpecification jobSpec = RuntimeUtils.createJobSpecification(metadata.getApplicationContext());
- PartitioningProperties partitioningProperties = metadata.splitAndConstraints(dataverse.getDataverseName());
- FileRemoveOperatorDescriptor frod = new FileRemoveOperatorDescriptor(jobSpec,
- partitioningProperties.getSplitsProvider(), false, partitioningProperties.getComputeStorageMap());
- AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, frod,
- partitioningProperties.getConstraints());
- jobSpec.addRoot(frod);
- return jobSpec;
+ public static JobSpecification dropDataverseJobSpec(Dataverse dataverse, MetadataProvider md) {
+ PartitioningProperties pp = md.splitAndConstraints(dataverse.getDatabaseName(), dataverse.getDataverseName());
+ return dropJobSpec(md, pp);
}
- public static JobSpecification dropDatabaseJobSpec(String database, MetadataProvider metadata) {
+ public static JobSpecification dropDatabaseJobSpec(String database, MetadataProvider md) {
+ PartitioningProperties pp = md.splitAndConstraints(database);
+ return dropJobSpec(md, pp);
+ }
+
+ private static JobSpecification dropJobSpec(MetadataProvider metadata, PartitioningProperties pp) {
JobSpecification jobSpec = RuntimeUtils.createJobSpecification(metadata.getApplicationContext());
- PartitioningProperties partitioningProperties = metadata.splitAndConstraints(database);
- FileRemoveOperatorDescriptor frod = new FileRemoveOperatorDescriptor(jobSpec,
- partitioningProperties.getSplitsProvider(), false, partitioningProperties.getComputeStorageMap());
- AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, frod,
- partitioningProperties.getConstraints());
+ FileRemoveOperatorDescriptor frod =
+ new FileRemoveOperatorDescriptor(jobSpec, pp.getSplitsProvider(), false, pp.getComputeStorageMap());
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, frod, pp.getConstraints());
jobSpec.addRoot(frod);
return jobSpec;
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
index 63958ce..f929677 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
@@ -37,6 +37,7 @@
import org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.metadata.MetadataUtil;
+import org.apache.asterix.common.metadata.NamespacePathResolver;
import org.apache.asterix.common.transactions.ITransactionManager;
import org.apache.asterix.common.transactions.TxnId;
import org.apache.asterix.dataflow.data.nontagged.MissingWriterFactory;
@@ -673,8 +674,10 @@
List<String> nodes = Collections.singletonList(ExecutionTestUtil.integrationUtil.ncs[0].getId());
CcApplicationContext appCtx =
(CcApplicationContext) ExecutionTestUtil.integrationUtil.cc.getApplicationContext();
+ String dvPath = new NamespacePathResolver(false).resolve(primaryIndexInfo.dataset.getDatabaseName(),
+ primaryIndexInfo.dataset.getDataverseName());
FileSplit[] splits = SplitsAndConstraintsUtil.getIndexSplits(appCtx.getClusterStateManager(),
- primaryIndexInfo.dataset, secondaryIndex.getIndexName(), nodes);
+ primaryIndexInfo.dataset, secondaryIndex.getIndexName(), nodes, dvPath);
fileSplitProvider = new ConstantFileSplitProvider(splits);
secondaryIndexTypeTraits = createSecondaryIndexTypeTraits(primaryIndexInfo.recordType,
primaryIndexInfo.metaType, primaryIndexInfo.primaryKeyTypes, secondaryIndexDetails
@@ -760,8 +763,10 @@
List<String> nodes = Collections.singletonList(ExecutionTestUtil.integrationUtil.ncs[0].getId());
CcApplicationContext appCtx =
(CcApplicationContext) ExecutionTestUtil.integrationUtil.cc.getApplicationContext();
+ String dvPath =
+ new NamespacePathResolver(false).resolve(dataset.getDatabaseName(), dataset.getDataverseName());
FileSplit[] splits = SplitsAndConstraintsUtil.getIndexSplits(appCtx.getClusterStateManager(), dataset,
- index.getIndexName(), nodes);
+ index.getIndexName(), nodes, dvPath);
fileSplitProvider = new ConstantFileSplitProvider(splits);
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
index bbdbe82..ea34f48 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
@@ -32,6 +32,7 @@
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.MetadataConstants;
+import org.apache.asterix.common.metadata.NamespacePathResolver;
import org.apache.asterix.common.utils.Servlets;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
@@ -251,8 +252,10 @@
(ICcApplicationContext) integrationUtil.getClusterControllerService().getApplicationContext();
final MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
try {
+ String dvPath =
+ new NamespacePathResolver(false).resolve(dataset.getDatabaseName(), dataset.getDataverseName());
return SplitsAndConstraintsUtil.getIndexSplits(dataset, dataset.getDatasetName(), mdTxnCtx,
- ccAppCtx.getClusterStateManager());
+ ccAppCtx.getClusterStateManager(), dvPath);
} finally {
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
index d5c8465..bdbdfab 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
@@ -32,6 +32,7 @@
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.MetadataConstants;
import org.apache.asterix.common.metadata.MetadataUtil;
+import org.apache.asterix.common.metadata.NamespacePathResolver;
import org.apache.asterix.file.StorageComponentProvider;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
@@ -158,8 +159,10 @@
final Dataset dataset = MetadataManager.INSTANCE.getDataset(mdTxn, MetadataConstants.DEFAULT_DATABASE,
defaultDv, datasetName);
MetadataManager.INSTANCE.commitTransaction(mdTxn);
+ String dvPath =
+ new NamespacePathResolver(false).resolve(dataset.getDatabaseName(), dataset.getDataverseName());
FileSplit[] splits = SplitsAndConstraintsUtil.getIndexSplits(appCtx.getClusterStateManager(), dataset,
- indexName, Arrays.asList("asterix_nc1"));
+ indexName, Arrays.asList("asterix_nc1"), dvPath);
final ConstantFileSplitProvider constantFileSplitProvider =
new ConstantFileSplitProvider(Arrays.copyOfRange(splits, 0, 1));
IndexDataflowHelperFactory helperFactory =
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
index 812e2e5..eabebf7 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
@@ -102,4 +102,8 @@
* @return the cloud properties
*/
CloudProperties getCloudProperties();
+
+ INamespaceResolver getNamespaceResolver();
+
+ INamespacePathResolver getNamespacePathResolver();
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespacePathResolver.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespacePathResolver.java
new file mode 100644
index 0000000..6f33d6f
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespacePathResolver.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.api;
+
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
+
+public interface INamespacePathResolver {
+
+ String resolve(Namespace namespace);
+
+ String resolve(String databaseName, DataverseName dataverseName);
+
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespaceResolver.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespaceResolver.java
new file mode 100644
index 0000000..f281280
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespaceResolver.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.api;
+
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
+
+public interface INamespaceResolver {
+
+ Namespace resolve(DataverseName dataverseName);
+
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/Namespace.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/Namespace.java
new file mode 100644
index 0000000..26b549a
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/Namespace.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.common.metadata;
+
+import java.util.Objects;
+
+public final class Namespace {
+
+ private final String databaseName;
+ private final DataverseName dataverseName;
+
+ public Namespace(String databaseName, DataverseName dataverseName) {
+ this.databaseName = Objects.requireNonNull(databaseName);
+ this.dataverseName = Objects.requireNonNull(dataverseName);
+ }
+
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ public DataverseName getDataverseName() {
+ return dataverseName;
+ }
+
+ @Override
+ public String toString() {
+ return databaseName + "." + dataverseName;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(databaseName, dataverseName);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof Namespace)) {
+ return false;
+ }
+ Namespace that = (Namespace) obj;
+ return Objects.equals(databaseName, that.databaseName) && Objects.equals(dataverseName, that.dataverseName);
+ }
+
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespacePathResolver.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespacePathResolver.java
new file mode 100644
index 0000000..26d1ce0
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespacePathResolver.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.common.metadata;
+
+import org.apache.asterix.common.api.INamespacePathResolver;
+import org.apache.asterix.common.utils.StoragePathUtil;
+
+public class NamespacePathResolver implements INamespacePathResolver {
+
+ private final boolean usingDatabase;
+
+ public NamespacePathResolver(boolean usingDatabase) {
+ this.usingDatabase = false;
+ }
+
+ @Override
+ public String resolve(Namespace namespace) {
+ return StoragePathUtil.prepareDataverseName(namespace.getDataverseName());
+ }
+
+ @Override
+ public String resolve(String databaseName, DataverseName dataverseName) {
+ return StoragePathUtil.prepareDataverseName(dataverseName);
+ }
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespaceResolver.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespaceResolver.java
new file mode 100644
index 0000000..14bba84
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespaceResolver.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.common.metadata;
+
+import org.apache.asterix.common.api.INamespaceResolver;
+
+public class NamespaceResolver implements INamespaceResolver {
+
+ private final boolean usingDatabase;
+
+ public NamespaceResolver(boolean usingDatabase) {
+ this.usingDatabase = false;
+ }
+
+ @Override
+ public Namespace resolve(DataverseName dataverseName) {
+ if (dataverseName == null) {
+ return null;
+ }
+ return new Namespace(MetadataUtil.databaseFor(dataverseName), dataverseName);
+ }
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
index 418e171..9862be7 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
@@ -24,6 +24,7 @@
import java.io.File;
import java.nio.file.Paths;
import java.util.Iterator;
+import java.util.List;
import org.apache.asterix.common.cluster.ClusterPartition;
import org.apache.asterix.common.metadata.DataverseName;
@@ -46,7 +47,7 @@
private static final Logger LOGGER = LogManager.getLogger();
public static final char DATAVERSE_CONTINUATION_MARKER = '^';
- private static String PARTITION_PATH = STORAGE_ROOT_DIR_NAME + File.separator + PARTITION_DIR_PREFIX;
+ private static final String PARTITION_PATH = STORAGE_ROOT_DIR_NAME + File.separator + PARTITION_DIR_PREFIX;
private StoragePathUtil() {
}
@@ -74,21 +75,25 @@
return new DefaultIoDeviceFileSplit(nodeId, relativePath);
}
- public static String prepareStoragePartitionPath(int partitonId) {
- return Paths.get(StorageConstants.STORAGE_ROOT_DIR_NAME, PARTITION_DIR_PREFIX + partitonId).toString();
+ public static String prepareStoragePartitionPath(int partitionId) {
+ return Paths.get(StorageConstants.STORAGE_ROOT_DIR_NAME, PARTITION_DIR_PREFIX + partitionId).toString();
}
public static String prepareIngestionLogPath() {
return Paths.get(StorageConstants.INGESTION_LOGS_DIR_NAME).toString();
}
- public static String prepareDataverseIndexName(DataverseName dataverseName, String datasetName, String idxName,
- long rebalanceCount) {
- return prepareDataverseComponentName(dataverseName, prepareFullIndexName(datasetName, idxName, rebalanceCount));
+ public static String prepareNamespaceIndexName(String datasetName, String idxName, long rebalanceCount,
+ String namespacePath) {
+ return prepareNamespaceComponentName(namespacePath, prepareFullIndexName(datasetName, idxName, rebalanceCount));
}
public static String prepareDataverseName(DataverseName dataverseName) {
- Iterator<String> dvParts = dataverseName.getParts().iterator();
+ List<String> parts = dataverseName.getParts();
+ if (parts.size() < 2) {
+ return parts.get(0);
+ }
+ Iterator<String> dvParts = parts.iterator();
StringBuilder builder = new StringBuilder();
builder.append(dvParts.next());
while (dvParts.hasNext()) {
@@ -97,8 +102,8 @@
return builder.toString();
}
- public static String prepareDataverseComponentName(DataverseName dataverseName, String component) {
- return prepareDataverseName(dataverseName) + File.separatorChar + component;
+ public static String prepareNamespaceComponentName(String namespacePath, String component) {
+ return namespacePath + File.separatorChar + component;
}
private static String prepareFullIndexName(String datasetName, String idxName, long rebalanceCount) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
index 7398d02..759d86e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
@@ -27,6 +27,8 @@
import org.apache.asterix.common.external.IDataSourceAdapter;
import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
import org.apache.asterix.common.library.ILibraryManager;
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.MetadataUtil;
import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IDataParserFactory;
import org.apache.asterix.external.api.IExternalDataSourceFactory;
@@ -156,9 +158,12 @@
this.isFeed = ExternalDataUtils.isFeed(configuration);
this.logIngestionEvents = ExternalDataUtils.isLogIngestionEvents(configuration);
if (logIngestionEvents) {
+ DataverseName dataverseName = ExternalDataUtils.getDatasetDataverse(configuration);
+ String databaseName = MetadataUtil.databaseFor(dataverseName);
+ String namespacePath = appCtx.getNamespacePathResolver().resolve(databaseName, dataverseName);
//TODO(partitioning) make this code reuse DataPartitioningProvider
- feedLogFileSplits = FeedUtils.splitsForAdapter(ExternalDataUtils.getDatasetDataverse(configuration),
- ExternalDataUtils.getFeedName(configuration), dataSourceFactory.getPartitionConstraint());
+ feedLogFileSplits = FeedUtils.splitsForAdapter(namespacePath, ExternalDataUtils.getFeedName(configuration),
+ dataSourceFactory.getPartitionConstraint());
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
index 050d9f0..59b1a02 100755
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
@@ -55,6 +55,8 @@
import javax.net.ssl.SSLContext;
+import org.apache.asterix.common.api.INamespacePathResolver;
+import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
@@ -136,17 +138,20 @@
private final FileReference trashDir;
private final FileReference distDir;
private final Path trashDirPath;
+ //TODO(DB): change for database
private final Map<Pair<DataverseName, String>, ILibrary> libraries = new HashMap<>();
private IPCSystem pythonIPC;
private final ExternalFunctionResultRouter router;
private final IIOManager ioManager;
- private boolean sslEnabled;
+ private final INamespacePathResolver namespacePathResolver;
+ private final boolean sslEnabled;
private Function<ILibraryManager, CloseableHttpClient> uploadClientSupp;
public ExternalLibraryManager(NodeControllerService ncs, IPersistedResourceRegistry reg, FileReference appDir,
IIOManager ioManager) {
this.ncs = ncs;
this.reg = reg;
+ namespacePathResolver = ((INcApplicationContext) ncs.getApplicationContext()).getNamespacePathResolver();
baseDir = appDir.getChild(LIBRARY_MANAGER_BASE_DIR_NAME);
storageDir = baseDir.getChild(STORAGE_DIR_NAME);
storageDirPath = storageDir.getFile().toPath();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
index 5baefcb..bf924fc 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
@@ -32,7 +32,6 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
-import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.runtime.utils.RuntimeUtils;
import org.apache.commons.lang3.StringUtils;
@@ -80,8 +79,8 @@
private FeedUtils() {
}
- private static FileSplit splitsForAdapter(DataverseName dataverseName, String feedName, String nodeName) {
- String relPathFile = StoragePathUtil.prepareDataverseComponentName(dataverseName, feedName);
+ private static FileSplit splitsForAdapter(String namespacePath, String feedName, String nodeName) {
+ String relPathFile = StoragePathUtil.prepareNamespaceComponentName(namespacePath, feedName);
String storagePartitionPath = StoragePathUtil.prepareIngestionLogPath();
// Note: feed adapter instances in a single node share the feed logger
// format: 'ingestion logs dir name'/dataverse_part1[^dataverse_part2[...]]/feed/node
@@ -89,7 +88,7 @@
return StoragePathUtil.getDefaultIoDeviceFileSpiltForNode(nodeName, f.getPath());
}
- public static FileSplit[] splitsForAdapter(DataverseName dataverseName, String feedName,
+ public static FileSplit[] splitsForAdapter(String namespacePath, String feedName,
AlgebricksPartitionConstraint partitionConstraints) throws AsterixException {
if (partitionConstraints.getPartitionConstraintType() == PartitionConstraintType.COUNT) {
throw new AsterixException("Can't create file splits for adapter with count partitioning constraints");
@@ -97,7 +96,7 @@
String[] locations = ((AlgebricksAbsolutePartitionConstraint) partitionConstraints).getLocations();
List<FileSplit> splits = new ArrayList<>();
for (String nd : locations) {
- splits.add(splitsForAdapter(dataverseName, feedName, nd));
+ splits.add(splitsForAdapter(namespacePath, feedName, nd));
}
return splits.toArray(new FileSplit[] {});
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataIndex.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataIndex.java
index 21329ac..082eb2f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataIndex.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataIndex.java
@@ -22,6 +22,7 @@
import java.io.Serializable;
import java.util.List;
+import org.apache.asterix.common.api.INamespacePathResolver;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.transactions.DatasetId;
import org.apache.asterix.om.types.ARecordType;
@@ -67,7 +68,7 @@
public int[] getFieldPermutation();
- public String getFileNameRelativePath();
+ public String getFileNameRelativePath(INamespacePathResolver namespacePathResolver);
public ARecordType getPayloadRecordType();
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index 4bab4c9..b154287 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -26,6 +26,7 @@
import java.util.List;
import org.apache.asterix.common.api.ILSMComponentIdGeneratorFactory;
+import org.apache.asterix.common.api.INamespacePathResolver;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.MetadataProperties;
@@ -392,9 +393,12 @@
public static void enlistMetadataDataset(INCServiceContext ncServiceCtx, IMetadataIndex index,
MetadataIndexesProvider mdIndexesProvider) throws HyracksDataException {
final int datasetId = index.getDatasetId().getId();
+ INamespacePathResolver namespacePathResolver =
+ ((INcApplicationContext) ncServiceCtx.getApplicationContext()).getNamespacePathResolver();
String metadataPartitionPath =
StoragePathUtil.prepareStoragePartitionPath(MetadataNode.INSTANCE.getMetadataStoragePartition());
- String resourceName = metadataPartitionPath + File.separator + index.getFileNameRelativePath();
+ String resourceName =
+ metadataPartitionPath + File.separator + index.getFileNameRelativePath(namespacePathResolver);
FileReference file = ioManager.resolve(resourceName);
index.setFile(file);
ITypeTraits[] typeTraits = index.getTypeTraits();
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndex.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndex.java
index d259643..c06b878 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndex.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndex.java
@@ -23,6 +23,7 @@
import java.util.Arrays;
import java.util.List;
+import org.apache.asterix.common.api.INamespacePathResolver;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.MetadataConstants;
import org.apache.asterix.common.metadata.MetadataIndexImmutableProperties;
@@ -237,10 +238,10 @@
}
@Override
- public String getFileNameRelativePath() {
+ public String getFileNameRelativePath(INamespacePathResolver namespacePathResolver) {
// The rebalance count for metadata dataset is always 0.
- return StoragePathUtil.prepareDataverseIndexName(getDataverseName(), getIndexedDatasetName(), getIndexName(),
- 0);
+ String namespacePath = namespacePathResolver.resolve(getDatabaseName(), getDataverseName());
+ return StoragePathUtil.prepareNamespaceIndexName(getIndexedDatasetName(), getIndexName(), 0, namespacePath);
}
@Override
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 0ecef7d..d1b07c5 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -32,6 +32,7 @@
import java.util.Optional;
import java.util.stream.Collectors;
+import org.apache.asterix.common.api.INamespaceResolver;
import org.apache.asterix.common.cluster.PartitioningProperties;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
@@ -49,6 +50,7 @@
import org.apache.asterix.common.metadata.LockList;
import org.apache.asterix.common.metadata.MetadataConstants;
import org.apache.asterix.common.metadata.MetadataUtil;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.common.storage.ICompressionManager;
import org.apache.asterix.common.transactions.ITxnIdFactory;
import org.apache.asterix.common.transactions.TxnId;
@@ -189,6 +191,7 @@
private boolean blockingOperatorDisabled = false;
private final DataPartitioningProvider dataPartitioningProvider;
+ private final INamespaceResolver namespaceResolver;
private IDataFormat dataFormat = FormatUtils.getDefaultFormat();
public static MetadataProvider create(ICcApplicationContext appCtx, Dataverse defaultDataverse) {
@@ -202,6 +205,7 @@
protected MetadataProvider(ICcApplicationContext appCtx) {
this.appCtx = appCtx;
this.storageComponentProvider = appCtx.getStorageComponentProvider();
+ namespaceResolver = appCtx.getNamespaceResolver();
storageProperties = appCtx.getStorageProperties();
functionManager = ((IFunctionExtensionManager) appCtx.getExtensionManager()).getFunctionManager();
dataPartitioningProvider = (DataPartitioningProvider) appCtx.getDataPartitioningProvider();
@@ -959,8 +963,8 @@
return dataPartitioningProvider.getPartitioningProperties(databaseName);
}
- public PartitioningProperties splitAndConstraints(DataverseName dataverseName) {
- return dataPartitioningProvider.getPartitioningProperties(dataverseName);
+ public PartitioningProperties splitAndConstraints(String databaseName, DataverseName dataverseName) {
+ return dataPartitioningProvider.getPartitioningProperties(databaseName, dataverseName);
}
public FileSplit[] splitsForIndex(MetadataTransactionContext mdTxnCtx, Dataset dataset, String indexName)
@@ -1823,6 +1827,10 @@
return storageComponentProvider;
}
+ public Namespace resolve(DataverseName dataverseName) {
+ return namespaceResolver.resolve(dataverseName);
+ }
+
public PartitioningProperties getPartitioningProperties(Index idx) throws AlgebricksException {
Dataset ds = findDataset(idx.getDatabaseName(), idx.getDataverseName(), idx.getDatasetName());
return getPartitioningProperties(ds, idx.getIndexName());
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DataPartitioningProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DataPartitioningProvider.java
index d99aa13..53c5d44 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DataPartitioningProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DataPartitioningProvider.java
@@ -22,6 +22,8 @@
import java.util.Set;
import java.util.TreeSet;
+import org.apache.asterix.common.api.INamespacePathResolver;
+import org.apache.asterix.common.api.INamespaceResolver;
import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.common.cluster.PartitioningProperties;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
@@ -47,11 +49,15 @@
public abstract class DataPartitioningProvider implements IDataPartitioningProvider {
protected final ICcApplicationContext appCtx;
+ protected final INamespacePathResolver namespacePathResolver;
+ protected final INamespaceResolver namespaceResolver;
protected final ClusterStateManager clusterStateManager;
protected final int storagePartitionsCounts;
DataPartitioningProvider(ICcApplicationContext appCtx) {
this.appCtx = appCtx;
+ this.namespacePathResolver = appCtx.getNamespacePathResolver();
+ this.namespaceResolver = appCtx.getNamespaceResolver();
this.clusterStateManager = (ClusterStateManager) appCtx.getClusterStateManager();
this.storagePartitionsCounts = clusterStateManager.getStoragePartitionsCount();
}
@@ -70,7 +76,7 @@
public abstract PartitioningProperties getPartitioningProperties(String databaseName);
- public abstract PartitioningProperties getPartitioningProperties(DataverseName dataverseName);
+ public abstract PartitioningProperties getPartitioningProperties(String databaseName, DataverseName dataverseName);
public abstract PartitioningProperties getPartitioningProperties(MetadataTransactionContext mdTxnCtx, Dataset ds,
String indexName) throws AlgebricksException;
@@ -81,8 +87,8 @@
Set<String> nodes = new TreeSet<>(Arrays.asList(allCluster.getLocations()));
AlgebricksAbsolutePartitionConstraint locations =
new AlgebricksAbsolutePartitionConstraint(nodes.toArray(new String[0]));
- FileSplit[] feedLogFileSplits =
- FeedUtils.splitsForAdapter(feed.getDataverseName(), feed.getFeedName(), locations);
+ String namespacePath = namespacePathResolver.resolve(feed.getDatabaseName(), feed.getDataverseName());
+ FileSplit[] feedLogFileSplits = FeedUtils.splitsForAdapter(namespacePath, feed.getFeedName(), locations);
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> spC =
StoragePathUtil.splitProviderAndPartitionConstraints(feedLogFileSplits);
int[][] partitionsMap = getOneToOnePartitionsMap(getLocationsCount(spC.second));
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DynamicDataPartitioningProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DynamicDataPartitioningProvider.java
index 803a53b..57392db 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DynamicDataPartitioningProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DynamicDataPartitioningProvider.java
@@ -45,9 +45,10 @@
}
@Override
- public PartitioningProperties getPartitioningProperties(DataverseName dataverseName) {
+ public PartitioningProperties getPartitioningProperties(String databaseName, DataverseName dataverseName) {
+ String namespacePath = namespacePathResolver.resolve(databaseName, dataverseName);
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraints = SplitsAndConstraintsUtil
- .getDataverseSplitProviderAndConstraints(appCtx.getClusterStateManager(), dataverseName);
+ .getDataverseSplitProviderAndConstraints(appCtx.getClusterStateManager(), namespacePath);
int[][] partitionsMap = getOneToOnePartitionsMap(getLocationsCount(splitsAndConstraints.second));
return PartitioningProperties.of(splitsAndConstraints.first, splitsAndConstraints.second, partitionsMap);
}
@@ -55,8 +56,9 @@
@Override
public PartitioningProperties getPartitioningProperties(MetadataTransactionContext mdTxnCtx, Dataset ds,
String indexName) throws AlgebricksException {
- FileSplit[] splits =
- SplitsAndConstraintsUtil.getIndexSplits(ds, indexName, mdTxnCtx, appCtx.getClusterStateManager());
+ String namespacePath = namespacePathResolver.resolve(ds.getDatabaseName(), ds.getDataverseName());
+ FileSplit[] splits = SplitsAndConstraintsUtil.getIndexSplits(ds, indexName, mdTxnCtx,
+ appCtx.getClusterStateManager(), namespacePath);
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraints =
StoragePathUtil.splitProviderAndPartitionConstraints(splits);
int[][] partitionsMap = getOneToOnePartitionsMap(getLocationsCount(splitsAndConstraints.second));
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
index 2abb4f6..da7cd0d 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
@@ -27,7 +27,6 @@
import org.apache.asterix.common.cluster.ClusterPartition;
import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.common.exceptions.MetadataException;
-import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.MetadataConstants;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.metadata.MetadataManager;
@@ -45,47 +44,42 @@
private SplitsAndConstraintsUtil() {
}
- private static FileSplit[] getDatabaseSplits(IClusterStateManager clusterStateManager, String databaseName) {
- List<FileSplit> splits = new ArrayList<>();
- // get all partitions
- for (ClusterPartition clusterPartition : clusterStateManager.getClusterPartitons()) {
- File f = new File(StoragePathUtil.prepareStoragePartitionPath(clusterPartition.getPartitionId()),
- databaseName);
- splits.add(StoragePathUtil.getFileSplitForClusterPartition(clusterPartition, f.getPath()));
- }
- return splits.toArray(new FileSplit[] {});
+ private static FileSplit[] getDatabaseSplits(IClusterStateManager clusterStateManager, String databasePath) {
+ return getSplits(clusterStateManager, databasePath);
}
- private static FileSplit[] getDataverseSplits(IClusterStateManager clusterStateManager,
- DataverseName dataverseName) {
+ private static FileSplit[] getDataverseSplits(IClusterStateManager clusterStateManager, String dataversePath) {
+ return getSplits(clusterStateManager, dataversePath);
+ }
+
+ private static FileSplit[] getSplits(IClusterStateManager clusterStateManager, String path) {
List<FileSplit> splits = new ArrayList<>();
// get all partitions
for (ClusterPartition clusterPartition : clusterStateManager.getClusterPartitons()) {
- File f = new File(StoragePathUtil.prepareStoragePartitionPath(clusterPartition.getPartitionId()),
- StoragePathUtil.prepareDataverseName(dataverseName));
+ File f = new File(StoragePathUtil.prepareStoragePartitionPath(clusterPartition.getPartitionId()), path);
splits.add(StoragePathUtil.getFileSplitForClusterPartition(clusterPartition, f.getPath()));
}
return splits.toArray(new FileSplit[] {});
}
public static FileSplit[] getIndexSplits(Dataset dataset, String indexName, MetadataTransactionContext mdTxnCtx,
- IClusterStateManager csm) throws AlgebricksException {
+ IClusterStateManager csm, String namespacePath) throws AlgebricksException {
try {
NodeGroup nodeGroup = MetadataManager.INSTANCE.getNodegroup(mdTxnCtx, dataset.getNodeGroupName());
if (nodeGroup == null) {
throw new AlgebricksException("Couldn't find node group " + dataset.getNodeGroupName());
}
List<String> nodeList = nodeGroup.getNodeNames();
- return getIndexSplits(csm, dataset, indexName, nodeList);
+ return getIndexSplits(csm, dataset, indexName, nodeList, namespacePath);
} catch (MetadataException me) {
throw new AlgebricksException(me);
}
}
public static FileSplit[] getIndexSplits(IClusterStateManager clusterStateManager, Dataset dataset,
- String indexName, List<String> nodes) {
- final String relPath = StoragePathUtil.prepareDataverseIndexName(dataset.getDataverseName(),
- dataset.getDatasetName(), indexName, dataset.getRebalanceCount());
+ String indexName, List<String> nodes, String namespacePath) {
+ final String relPath = StoragePathUtil.prepareNamespaceIndexName(dataset.getDatasetName(), indexName,
+ dataset.getRebalanceCount(), namespacePath);
final List<ClusterPartition> datasetPartitions = getDatasetPartitions(clusterStateManager, dataset, nodes);
final List<FileSplit> splits = new ArrayList<>();
for (ClusterPartition partition : datasetPartitions) {
@@ -102,8 +96,8 @@
}
public static Pair<IFileSplitProvider, AlgebricksPartitionConstraint> getDataverseSplitProviderAndConstraints(
- IClusterStateManager clusterStateManager, DataverseName dataverseName) {
- FileSplit[] splits = getDataverseSplits(clusterStateManager, dataverseName);
+ IClusterStateManager clusterStateManager, String dataversePath) {
+ FileSplit[] splits = getDataverseSplits(clusterStateManager, dataversePath);
return StoragePathUtil.splitProviderAndPartitionConstraints(splits);
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java
index f8f967a..7206a50 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java
@@ -58,8 +58,8 @@
}
@Override
- public PartitioningProperties getPartitioningProperties(DataverseName dataverseName) {
- SplitComputeLocations dataverseSplits = getDataverseSplits(dataverseName);
+ public PartitioningProperties getPartitioningProperties(String databaseName, DataverseName dataverseName) {
+ SplitComputeLocations dataverseSplits = getDataverseSplits(databaseName, dataverseName);
StorageComputePartitionsMap partitionMap = clusterStateManager.getStorageComputeMap();
int[][] partitionsMap = partitionMap.getComputeToStorageMap(false);
return PartitioningProperties.of(dataverseSplits.getSplitsProvider(), dataverseSplits.getConstraints(),
@@ -77,8 +77,8 @@
partitionsMap);
}
- private SplitComputeLocations getDataverseSplits(DataverseName dataverseName) {
- return getSplits(StoragePathUtil.prepareDataverseName(dataverseName));
+ private SplitComputeLocations getDataverseSplits(String databaseName, DataverseName dataverseName) {
+ return getSplits(namespacePathResolver.resolve(databaseName, dataverseName));
}
private SplitComputeLocations getSplits(String subPath) {
@@ -105,13 +105,14 @@
List<FileSplit> splits = new ArrayList<>();
List<String> locations = new ArrayList<>();
Set<Integer> uniqueLocations = new HashSet<>();
+ String namespacePath = namespacePathResolver.resolve(dataset.getDatabaseName(), dataset.getDataverseName());
StorageComputePartitionsMap partitionMap = clusterStateManager.getStorageComputeMap();
final int datasetPartitions = getNumberOfPartitions(dataset);
boolean metadataDataset = MetadataIndexImmutableProperties.isMetadataDataset(dataset.getDatasetId());
for (int i = 0; i < datasetPartitions; i++) {
int storagePartition = metadataDataset ? StorageConstants.METADATA_PARTITION : i;
- final String relPath = StoragePathUtil.prepareDataverseIndexName(dataset.getDataverseName(),
- dataset.getDatasetName(), indexName, dataset.getRebalanceCount());
+ final String relPath = StoragePathUtil.prepareNamespaceIndexName(dataset.getDatasetName(), indexName,
+ dataset.getRebalanceCount(), namespacePath);
File f = new File(StoragePathUtil.prepareStoragePartitionPath(storagePartition), relPath);
ComputePartition computePartition = partitionMap.getComputePartition(storagePartition);
splits.add(new MappedFileSplit(computePartition.getNodeId(), f.getPath(), 0));