[NO ISSUE][*DB] Minor cleanup / refactoring
Change-Id: I4b314e50314c845dd502c3747d469d866671e4c7
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18233
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Tested-by: Michael Blow <mblow@apache.org>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java
index f2c282b..397836b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java
@@ -68,15 +68,16 @@
/**
* Initialize {@link org.apache.asterix.app.cc.CCExtensionManager} from configuration
*
- * @param list
- * a list of extensions
+ * @param list a list of extensions
* @param namespaceResolver
+ * @param ccServiceCtx
* @throws InstantiationException
* @throws IllegalAccessException
* @throws ClassNotFoundException
* @throws HyracksDataException
*/
- public CCExtensionManager(List<AsterixExtension> list, INamespaceResolver namespaceResolver)
+ public CCExtensionManager(List<AsterixExtension> list, INamespaceResolver namespaceResolver,
+ ICCServiceContext ccServiceCtx)
throws InstantiationException, IllegalAccessException, ClassNotFoundException, HyracksDataException {
Pair<ExtensionId, ILangCompilationProvider> sqlppcp = null;
Pair<ExtensionId, IFunctionManager> fm = null;
@@ -87,7 +88,7 @@
Set<ExtensionId> extensionIds = new HashSet<>();
for (AsterixExtension extensionConf : list) {
IExtension extension = (IExtension) Class.forName(extensionConf.getClassName()).newInstance();
- extension.configure(extensionConf.getArgs());
+ extension.configure(extensionConf.getArgs(), ccServiceCtx);
if (!extensionIds.add(extension.getId())) {
throw new RuntimeDataException(ErrorCode.EXTENSION_ID_CONFLICT, extension.getId());
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCExtensionManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCExtensionManager.java
index 339b8d7..37c7bad 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCExtensionManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCExtensionManager.java
@@ -53,18 +53,15 @@
/**
* Initialize {@code CCExtensionManager} from configuration
*
- * @param list
- * list of user configured extensions
- * @throws InstantiationException
- * if an extension couldn't be created
- * @throws IllegalAccessException
- * if user doesn't have enough acess priveleges
- * @throws ClassNotFoundException
- * if a class was not found
- * @throws HyracksDataException
- * if two extensions conlict with each other
+ * @param list list of user configured extensions
+ * @param ncServiceCtx
+ * @throws InstantiationException if an extension couldn't be created
+ * @throws IllegalAccessException if user doesn't have enough acess priveleges
+ * @throws ClassNotFoundException if a class was not found
+ * @throws HyracksDataException if two extensions conlict with each other
*/
- public NCExtensionManager(List<AsterixExtension> list, boolean usingDatabase, INamespaceResolver namespaceResolver)
+ public NCExtensionManager(List<AsterixExtension> list, boolean usingDatabase, INamespaceResolver namespaceResolver,
+ INCServiceContext ncServiceCtx)
throws InstantiationException, IllegalAccessException, ClassNotFoundException, HyracksDataException {
Pair<ExtensionId, ILangCompilationProvider> sqlppcp = null;
IMetadataExtension tupleTranslatorProviderExtension = null;
@@ -73,7 +70,7 @@
if (list != null) {
for (AsterixExtension extensionConf : list) {
IExtension extension = (IExtension) Class.forName(extensionConf.getClassName()).newInstance();
- extension.configure(extensionConf.getArgs());
+ extension.configure(extensionConf.getArgs(), ncServiceCtx);
switch (extension.getExtensionKind()) {
case LANG:
ILangExtension le = (ILangExtension) extension;
@@ -85,7 +82,7 @@
mdExtensions.add(mde);
//TODO(DB) clean up
tupleTranslatorProviderExtension = ExtensionUtil.extendTupleTranslatorProvider(
- tupleTranslatorProviderExtension, mde, mdIndexesProvider);
+ tupleTranslatorProviderExtension, mde, mdIndexesProvider, ncServiceCtx);
break;
default:
break;
@@ -99,8 +96,8 @@
this.tupleTranslatorProvider = new MetadataTupleTranslatorProvider(metadataIndexesProvider);
} else {
this.metadataIndexesProvider = tupleTranslatorProviderExtension.getMetadataIndexesProvider(usingDatabase);
- this.tupleTranslatorProvider =
- tupleTranslatorProviderExtension.getMetadataTupleTranslatorProvider(metadataIndexesProvider);
+ this.tupleTranslatorProvider = tupleTranslatorProviderExtension
+ .getMetadataTupleTranslatorProvider(metadataIndexesProvider, ncServiceCtx);
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index e72ab54..100e524 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -830,7 +830,7 @@
lockUtil.createDatasetBegin(lockManager, metadataProvider.getLocks(), databaseName, dataverseName, datasetName,
itemTypeDatabase, itemTypeDataverseName, itemTypeName, itemTypeAnonymous, metaItemTypeDatabase,
metaItemTypeDataverseName, metaItemTypeName, metaItemTypeAnonymous, nodegroupName, compactionPolicy,
- defaultCompactionPolicy, dd.getDatasetType(), dd.getDatasetDetailsDecl());
+ defaultCompactionPolicy, dd.getDatasetType(), dd.getDatasetDetailsDecl(), metadataProvider);
try {
doCreateDatasetStatement(metadataProvider, dd, stmtActiveNamespace, datasetName, itemTypeNamespace,
itemTypeExpr, itemTypeName, metaItemTypeExpr, metaItemTypeNamespace, metaItemTypeName, hcc,
@@ -998,7 +998,7 @@
ExternalDataUtils.validate(properties);
ExternalDataUtils.validateType(properties, (ARecordType) itemType);
validateExternalDatasetProperties(externalDetails, properties, dd.getSourceLocation(), mdTxnCtx,
- appCtx);
+ appCtx, metadataProvider);
datasetDetails = new ExternalDatasetDetails(externalDetails.getAdapter(), properties, new Date(),
TransactionState.COMMIT);
break;
@@ -2763,7 +2763,7 @@
}
lockUtil.createDatasetBegin(lockManager, metadataProvider.getLocks(), databaseName, dataverseName, viewName,
itemTypeDatabaseName, viewItemTypeDataverseName, viewItemTypeName, viewItemTypeAnonymous, null, null,
- null, false, null, null, true, DatasetType.VIEW, null);
+ null, false, null, null, true, DatasetType.VIEW, null, metadataProvider);
try {
doCreateView(metadataProvider, cvs, databaseName, dataverseName, viewName, itemTypeDatabaseName,
viewItemTypeDataverseName, viewItemTypeName, stmtRewriter, requestParameters);
@@ -3982,7 +3982,7 @@
ExternalDataUtils.normalize(properties);
ExternalDataUtils.validate(properties);
validateExternalDatasetProperties(externalDetails, properties, copyStmt.getSourceLocation(), mdTxnCtx,
- appCtx);
+ appCtx, metadataProvider);
CompiledCopyFromFileStatement cls = new CompiledCopyFromFileStatement(databaseName, dataverseName,
copyStmt.getDatasetName(), itemType, externalDetails.getAdapter(), properties);
cls.setSourceLocation(stmt.getSourceLocation());
@@ -5664,7 +5664,8 @@
protected void validateExternalDatasetProperties(ExternalDetailsDecl externalDetails,
Map<String, String> properties, SourceLocation srcLoc, MetadataTransactionContext mdTxnCtx,
- IApplicationContext appCtx) throws AlgebricksException, HyracksDataException {
+ IApplicationContext appCtx, MetadataProvider metadataProvider)
+ throws AlgebricksException, HyracksDataException {
// Validate adapter specific properties
String adapter = externalDetails.getAdapter();
Map<String, String> details = new HashMap<>(properties);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
index 7d3400b..64c16e9 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
@@ -176,7 +176,7 @@
boolean useDatabaseResolution = cloudDeployment && isDbResolutionEnabled;
INamespaceResolver namespaceResolver = new NamespaceResolver(useDatabaseResolution);
INamespacePathResolver namespacePathResolver = new NamespacePathResolver(useDatabaseResolution);
- ccExtensionManager = new CCExtensionManager(new ArrayList<>(getExtensions()), namespaceResolver);
+ ccExtensionManager = new CCExtensionManager(new ArrayList<>(getExtensions()), namespaceResolver, ccServiceCtx);
IGlobalRecoveryManager globalRecoveryManager = createGlobalRecoveryManager();
final CCConfig ccConfig = controllerService.getCCConfig();
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index 4acc039..fc22f45 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -166,8 +166,8 @@
boolean useDatabaseResolution = cloudDeployment && isDbResolutionEnabled;
NamespaceResolver namespaceResolver = new NamespaceResolver(useDatabaseResolution);
NamespacePathResolver namespacePathResolver = new NamespacePathResolver(useDatabaseResolution);
- ncExtensionManager =
- new NCExtensionManager(new ArrayList<>(getExtensions()), cloudDeployment, namespaceResolver);
+ ncExtensionManager = new NCExtensionManager(new ArrayList<>(getExtensions()), cloudDeployment,
+ namespaceResolver, ncServiceCtx);
runtimeContext = createNCApplicationContext(ncServiceCtx, ncExtensionManager, getPropertiesFactory(),
namespaceResolver, namespacePathResolver);
MetadataProperties metadataProperties = runtimeContext.getMetadataProperties();
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java
index 4374f01..406a148 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java
@@ -31,6 +31,7 @@
import org.apache.asterix.om.functions.IFunctionManager;
import org.apache.asterix.translator.IStatementExecutorFactory;
import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.api.application.INCServiceContext;
/**
* Provide util methods dealing with extensions
@@ -108,22 +109,21 @@
/**
* Validates no extension conflict and extends tuple translator provider
*
- * @param metadataExtension
- * place holder for tuple translator provider extension
- * @param mde
- * user defined metadata extension
+ * @param metadataExtension place holder for tuple translator provider extension
+ * @param mde user defined metadata extension
* @param metadataIndexesProvider
+ * @param ncServiceCtx
* @return the metadata extension if the extension defines a metadata tuple translator, null otherwise
- * @throws RuntimeDataException
- * if an extension conflict was detected
+ * @throws RuntimeDataException if an extension conflict was detected
*/
public static IMetadataExtension extendTupleTranslatorProvider(IMetadataExtension metadataExtension,
- IMetadataExtension mde, MetadataIndexesProvider metadataIndexesProvider) throws RuntimeDataException {
+ IMetadataExtension mde, MetadataIndexesProvider metadataIndexesProvider, INCServiceContext ncServiceCtx)
+ throws RuntimeDataException {
if (metadataExtension != null) {
throw new RuntimeDataException(ErrorCode.EXTENSION_COMPONENT_CONFLICT, metadataExtension.getId(),
mde.getId(), IMetadataExtension.class.getSimpleName());
}
- return mde.getMetadataTupleTranslatorProvider(metadataIndexesProvider) == null ? null : mde;
+ return mde.getMetadataTupleTranslatorProvider(metadataIndexesProvider, ncServiceCtx) == null ? null : mde;
}
/**
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IExtension.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IExtension.java
index 7e9879a..8f342a4 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IExtension.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IExtension.java
@@ -21,6 +21,7 @@
import java.util.List;
import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.api.application.IServiceContext;
/**
* an interface for an extension that provides a mechanism to override system behaviour
@@ -57,8 +58,9 @@
* This method is called on system boot
*
* @param args
+ * @param serviceCtx
*/
- void configure(List<Pair<String, String>> args);
+ void configure(List<Pair<String, String>> args, IServiceContext serviceCtx);
/**
* @return The extension point implemented by this extension
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/IMetadataLockUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/IMetadataLockUtil.java
index 795e40f..4cf938e 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/IMetadataLockUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/IMetadataLockUtil.java
@@ -22,6 +22,7 @@
import org.apache.asterix.common.api.IMetadataLockManager;
import org.apache.asterix.common.config.DatasetConfig;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
import com.google.common.collect.ImmutableList;
@@ -50,8 +51,8 @@
DataverseName itemTypeDataverseName, String itemTypeName, boolean itemTypeAnonymous,
String metaItemTypeDatabase, DataverseName metaItemTypeDataverseName, String metaItemTypeName,
boolean metaItemTypeAnonymous, String nodeGroupName, String compactionPolicyName,
- boolean isDefaultCompactionPolicy, DatasetConfig.DatasetType datasetType, Object datasetDetails)
- throws AlgebricksException;
+ boolean isDefaultCompactionPolicy, DatasetConfig.DatasetType datasetType, Object datasetDetails,
+ IMetadataProvider metadataProvider) throws AlgebricksException;
void dropDatasetBegin(IMetadataLockManager lockManager, LockList locks, String database,
DataverseName dataverseName, String datasetName) throws AlgebricksException;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index d56924f..fcdddd8 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -39,7 +39,7 @@
// used to specify the stream factory for an adapter that has a stream data source
public static final String KEY_STREAM = "stream";
//TODO(DB): check adapter configuration
- public static final String KEY_DATABASE_DATAVERSE = "dataset-database";
+ public static final String KEY_DATASET_DATABASE = "dataset-database";
// used to specify the dataverse of the adapter
public static final String KEY_DATASET_DATAVERSE = "dataset-dataverse";
// used to specify the socket addresses when reading data from sockets
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index b02122d..3298759 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -212,7 +212,7 @@
}
public static String getDatasetDatabase(Map<String, String> configuration) throws AsterixException {
- return configuration.get(ExternalDataConstants.KEY_DATABASE_DATAVERSE);
+ return configuration.get(ExternalDataConstants.KEY_DATASET_DATABASE);
}
public static DataverseName getDatasetDataverse(Map<String, String> configuration) throws AsterixException {
@@ -353,8 +353,8 @@
if (!configuration.containsKey(ExternalDataConstants.KEY_IS_FEED)) {
configuration.put(ExternalDataConstants.KEY_IS_FEED, ExternalDataConstants.TRUE);
}
- configuration.computeIfAbsent(ExternalDataConstants.KEY_LOG_INGESTION_EVENTS, k -> ExternalDataConstants.TRUE);
- configuration.put(ExternalDataConstants.KEY_DATABASE_DATAVERSE, databaseName);
+ configuration.putIfAbsent(ExternalDataConstants.KEY_LOG_INGESTION_EVENTS, ExternalDataConstants.TRUE);
+ configuration.put(ExternalDataConstants.KEY_DATASET_DATABASE, databaseName);
configuration.put(ExternalDataConstants.KEY_DATASET_DATAVERSE, dataverseName.getCanonicalForm());
configuration.put(ExternalDataConstants.KEY_FEED_NAME, feedName);
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataExtension.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataExtension.java
index d9037cc..3f64ff7 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataExtension.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataExtension.java
@@ -46,7 +46,8 @@
* @return The tuple translator provider that must be used by the {@code IMetadataNode } to read and write core
* {@code IMetadataEntity} objects
*/
- MetadataTupleTranslatorProvider getMetadataTupleTranslatorProvider(MetadataIndexesProvider metadataIndexesProvider);
+ MetadataTupleTranslatorProvider getMetadataTupleTranslatorProvider(MetadataIndexesProvider metadataIndexesProvider,
+ INCServiceContext ncServiceCtx);
MetadataIndexesProvider getMetadataIndexesProvider(boolean usingDatabase);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index e0f44ea..0a73ec9 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -975,7 +975,7 @@
Map<String, String> configuration, ARecordType itemType, IWarningCollector warningCollector,
IExternalFilterEvaluatorFactory filterEvaluatorFactory) throws AlgebricksException {
try {
- configuration.put(ExternalDataConstants.KEY_DATABASE_DATAVERSE, dataset.getDatabaseName());
+ configuration.put(ExternalDataConstants.KEY_DATASET_DATABASE, dataset.getDatabaseName());
configuration.put(ExternalDataConstants.KEY_DATASET_DATAVERSE,
dataset.getDataverseName().getCanonicalForm());
return AdapterFactoryProvider.getAdapterFactory(getApplicationContext().getServiceContext(), adapterName,
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java
index 6f44f4c..0b649c4 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java
@@ -29,6 +29,7 @@
import org.apache.asterix.common.metadata.IMetadataLockUtil;
import org.apache.asterix.common.metadata.LockList;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
@@ -68,8 +69,8 @@
DataverseName itemTypeDataverseName, String itemTypeName, boolean itemTypeAnonymous,
String metaItemTypeDatabase, DataverseName metaItemTypeDataverseName, String metaItemTypeName,
boolean metaItemTypeAnonymous, String nodeGroupName, String compactionPolicyName,
- boolean isDefaultCompactionPolicy, DatasetConfig.DatasetType datasetType, Object datasetDetails)
- throws AlgebricksException {
+ boolean isDefaultCompactionPolicy, DatasetConfig.DatasetType datasetType, Object datasetDetails,
+ IMetadataProvider metadataProvider) throws AlgebricksException {
createDatasetBeginPre(lockMgr, locks, database, dataverseName, itemTypeDatabase, itemTypeDataverseName,
itemTypeName, itemTypeAnonymous, metaItemTypeDatabase, metaItemTypeDataverseName, metaItemTypeName,
metaItemTypeAnonymous, nodeGroupName, compactionPolicyName, isDefaultCompactionPolicy);
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index 25a67ff..b9f277a 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -539,7 +539,7 @@
if (file.exists()) {
delete(fileRef);
} else {
- FileUtils.createParentDirectories(file);
+ file.getParentFile().mkdirs();
}
FileUtil.writeAndForce(file.toPath(), bytes);
} catch (IOException e) {