[ASTERIXDB-3370][COMP] Collect accessed entities during compilation
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Collect datasets/views/functions/link for copy-to statement
- Add visitor to collect accessed datasets/views/functions/copyto
Change-Id: Icb2529cd876bbbf40b92635aca0aa9693e9cd154
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18212
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 6424280..3e674fd 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -448,16 +448,16 @@
handleLibraryDropStatement(metadataProvider, stmt, hcc, requestParameters);
break;
case CREATE_SYNONYM:
- handleCreateSynonymStatement(metadataProvider, stmt);
+ handleCreateSynonymStatement(metadataProvider, stmt, requestParameters);
break;
case SYNONYM_DROP:
- handleDropSynonymStatement(metadataProvider, stmt);
+ handleDropSynonymStatement(metadataProvider, stmt, requestParameters);
break;
case CREATE_VIEW:
handleCreateViewStatement(metadataProvider, stmt, stmtRewriter, requestParameters);
break;
case VIEW_DROP:
- handleViewDropStatement(metadataProvider, stmt);
+ handleViewDropStatement(metadataProvider, stmt, requestParameters);
break;
case LOAD:
if (stats.getProfileType() == Stats.ProfileType.FULL) {
@@ -2963,7 +2963,8 @@
}
}
- public void handleViewDropStatement(MetadataProvider metadataProvider, Statement stmt) throws Exception {
+ public void handleViewDropStatement(MetadataProvider metadataProvider, Statement stmt,
+ IRequestParameters requestParameters) throws Exception {
ViewDropStatement stmtDrop = (ViewDropStatement) stmt;
SourceLocation sourceLoc = stmtDrop.getSourceLocation();
String viewName = stmtDrop.getViewName().getValue();
@@ -2976,14 +2977,14 @@
}
lockUtil.dropDatasetBegin(lockManager, metadataProvider.getLocks(), databaseName, dataverseName, viewName);
try {
- doDropView(metadataProvider, stmtDrop, databaseName, dataverseName, viewName);
+ doDropView(metadataProvider, stmtDrop, databaseName, dataverseName, viewName, requestParameters);
} finally {
metadataProvider.getLocks().unlock();
}
}
protected boolean doDropView(MetadataProvider metadataProvider, ViewDropStatement stmtViewDrop, String databaseName,
- DataverseName dataverseName, String viewName) throws Exception {
+ DataverseName dataverseName, String viewName, IRequestParameters requestParameters) throws Exception {
SourceLocation sourceLoc = stmtViewDrop.getSourceLocation();
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
metadataProvider.setMetadataTxnContext(mdTxnCtx);
@@ -3787,7 +3788,8 @@
}
}
- protected void handleCreateSynonymStatement(MetadataProvider metadataProvider, Statement stmt) throws Exception {
+ protected void handleCreateSynonymStatement(MetadataProvider metadataProvider, Statement stmt,
+ IRequestParameters requestParameters) throws Exception {
CreateSynonymStatement css = (CreateSynonymStatement) stmt;
metadataProvider.validateDatabaseObjectName(css.getNamespace(), css.getSynonymName(), css.getSourceLocation());
Namespace stmtActiveNamespace = getActiveNamespace(css.getNamespace());
@@ -3843,7 +3845,8 @@
}
}
- protected void handleDropSynonymStatement(MetadataProvider metadataProvider, Statement stmt) throws Exception {
+ protected void handleDropSynonymStatement(MetadataProvider metadataProvider, Statement stmt,
+ IRequestParameters requestParameters) throws Exception {
SynonymDropStatement stmtSynDrop = (SynonymDropStatement) stmt;
String synonymName = stmtSynDrop.getSynonymName();
metadataProvider.validateDatabaseObjectName(stmtSynDrop.getNamespace(), synonymName,
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
index cbbb9de..183b5e0 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
@@ -73,6 +73,7 @@
import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppGroupingSetsVisitor;
import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppInlineUdfsVisitor;
import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppListInputFunctionRewriteVisitor;
+import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppLoadAccessedDataset;
import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppRightJoinRewriteVisitor;
import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppSpecialFunctionNameRewriteVisitor;
import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppWindowAggregationSugarVisitor;
@@ -200,6 +201,9 @@
// Rewrites RIGHT OUTER JOINs into LEFT OUTER JOINs if possible
rewriteRightJoins();
+ // Load all the accessed datasets
+ loadAccessedDatasets();
+
// Inlines functions and views
loadAndInlineUdfsAndViews();
@@ -336,6 +340,11 @@
rewriteTopExpr(visitor, null);
}
+ protected void loadAccessedDatasets() throws CompilationException {
+ SqlppLoadAccessedDataset visitor = new SqlppLoadAccessedDataset(context);
+ rewriteTopExpr(visitor, null);
+ }
+
protected void loadAndInlineUdfsAndViews() throws CompilationException {
Pair<Map<FunctionSignature, FunctionDecl>, Map<DatasetFullyQualifiedName, ViewDecl>> udfAndViewDecls =
loadUdfsAndViews(topStatement);
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java
new file mode 100644
index 0000000..355b484
--- /dev/null
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.sqlpp.rewrites.visitor;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.expression.CallExpr;
+import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
+import org.apache.asterix.lang.common.statement.ExternalDetailsDecl;
+import org.apache.asterix.lang.common.util.ExpressionUtils;
+import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppSimpleExpressionVisitor;
+import org.apache.asterix.metadata.entities.EntityDetails;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+
+public class SqlppLoadAccessedDataset extends AbstractSqlppSimpleExpressionVisitor {
+
+ protected final LangRewritingContext context;
+
+ public SqlppLoadAccessedDataset(LangRewritingContext context) {
+ this.context = context;
+ }
+
+ @Override
+ public Expression visit(CopyToStatement stmtCopy, ILangExpression arg) throws CompilationException {
+ ExternalDetailsDecl externalDetailsDecl = stmtCopy.getExternalDetailsDecl();
+ Map<String, String> properties = externalDetailsDecl.getProperties();
+ String databaseName = properties.getOrDefault("database", "Default");
+ String dataverseNameString = properties.getOrDefault("dataverse", "Default");
+ String linkName = properties.getOrDefault("name", "");
+ DataverseName dataverseName;
+ try {
+ dataverseName = DataverseName.createFromCanonicalForm(dataverseNameString);
+ } catch (AsterixException e) {
+ throw new IllegalStateException(e);
+ }
+
+ context.getMetadataProvider().addAccessedEntity(
+ new EntityDetails(databaseName, dataverseName, linkName, EntityDetails.EntityType.LINK));
+ return super.visit(stmtCopy, arg);
+ }
+
+ @Override
+ public Expression visit(CallExpr expression, ILangExpression arg) {
+ if (BuiltinFunctions.DATASET.equals(expression.getFunctionSignature().createFunctionIdentifier())) {
+ List<Expression> exprs = expression.getExprList();
+ String databaseName, dataverseNameArg, datasetName;
+ EntityDetails.EntityType entityType;
+ entityType = exprs.size() > 3 && Boolean.TRUE.equals(ExpressionUtils.getBooleanLiteral(exprs.get(3)))
+ ? EntityDetails.EntityType.VIEW : EntityDetails.EntityType.DATASET;
+
+ databaseName = ExpressionUtils.getStringLiteral(exprs.get(0));
+ dataverseNameArg = ExpressionUtils.getStringLiteral(exprs.get(1));
+ DataverseName dataverseName;
+ try {
+ dataverseName = DataverseName.createFromCanonicalForm(dataverseNameArg);
+ } catch (AsterixException e) {
+ throw new IllegalStateException(e);
+ }
+ datasetName = ExpressionUtils.getStringLiteral(exprs.get(2));
+
+ context.getMetadataProvider()
+ .addAccessedEntity(new EntityDetails(databaseName, dataverseName, datasetName, entityType));
+ } else {
+ FunctionSignature signature = expression.getFunctionSignature();
+ String functionName = signature.getName() + "(" + signature.getArity() + ")";
+ context.getMetadataProvider().addAccessedEntity(new EntityDetails(signature.getDatabaseName(),
+ signature.getDataverseName(), functionName, EntityDetails.EntityType.FUNCTION));
+ }
+ return expression;
+ }
+}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index e861a3f..6ec29da 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -25,10 +25,13 @@
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.stream.Collectors;
import org.apache.asterix.common.api.INamespaceResolver;
@@ -76,6 +79,7 @@
import org.apache.asterix.metadata.entities.DatasourceAdapter;
import org.apache.asterix.metadata.entities.Datatype;
import org.apache.asterix.metadata.entities.Dataverse;
+import org.apache.asterix.metadata.entities.EntityDetails;
import org.apache.asterix.metadata.entities.Feed;
import org.apache.asterix.metadata.entities.FeedConnection;
import org.apache.asterix.metadata.entities.FeedPolicyEntity;
@@ -200,6 +204,8 @@
private final INamespaceResolver namespaceResolver;
private IDataFormat dataFormat = FormatUtils.getDefaultFormat();
+ private final Set<EntityDetails> getAccessedEntities;
+
public static MetadataProvider createWithDefaultNamespace(ICcApplicationContext appCtx) {
java.util.function.Function<ICcApplicationContext, IMetadataProvider<?, ?>> factory =
((ICCExtensionManager) appCtx.getExtensionManager()).getMetadataProviderFactory();
@@ -225,6 +231,7 @@
dataPartitioningProvider = (DataPartitioningProvider) appCtx.getDataPartitioningProvider();
locks = new LockList();
config = new HashMap<>();
+ getAccessedEntities = new HashSet<>();
setDefaultNamespace(MetadataConstants.DEFAULT_NAMESPACE);
}
@@ -1945,6 +1952,14 @@
}
}
+ public void addAccessedEntity(EntityDetails entityDetails) {
+ getAccessedEntities.add(entityDetails);
+ }
+
+ public Set<EntityDetails> getGetAccessedEntities() {
+ return Collections.unmodifiableSet(getAccessedEntities);
+ }
+
private void validateDatabaseObjectNameImpl(String name, SourceLocation sourceLoc) throws AlgebricksException {
if (name == null || name.isEmpty()) {
throw new AsterixException(ErrorCode.INVALID_DATABASE_OBJECT_NAME, sourceLoc, "");
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java
new file mode 100644
index 0000000..c030176
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.entities;
+
+import org.apache.asterix.common.metadata.DataverseName;
+
+public class EntityDetails {
+
+ public enum EntityType {
+ DATASET,
+ VIEW,
+ FUNCTION,
+ LINK
+ }
+
+ private final String databaseName;
+ private final DataverseName dataverseName;
+ private final String datasetName;
+ private final EntityType entityType;
+
+ public EntityDetails(String databaseName, DataverseName dataverseName, String datasetName, EntityType entityType) {
+ this.databaseName = databaseName;
+ this.dataverseName = dataverseName;
+ this.datasetName = datasetName;
+ this.entityType = entityType;
+ }
+
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ public DataverseName getDataverseName() {
+ return dataverseName;
+ }
+
+ public String getDatasetName() {
+ return datasetName;
+ }
+
+ public EntityType getEntityType() {
+ return entityType;
+ }
+}