[ASTERIXDB-3370][COMP] Collect and authorise dependency for views and functions
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Add ensureAuthorised in IReceptionist method to authorise the dependency
- Fix naming for accessed entities
- Add dependencies to accessed entity field
Change-Id: Ic3b1898307f7099dba3ea135db6c2e794de27bde
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18226
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/Receptionist.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/Receptionist.java
index 84bee6a..ed68d8b 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/Receptionist.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/Receptionist.java
@@ -28,6 +28,7 @@
import org.apache.asterix.common.api.ISchedulableClientRequest;
import org.apache.asterix.common.api.RequestReference;
import org.apache.http.HttpHeaders;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.util.NetworkUtil;
@@ -54,4 +55,10 @@
public void ensureSchedulable(ISchedulableClientRequest schedulableRequest) throws HyracksDataException {
// currently we don't have any restrictions
}
+
+ @Override
+ public void ensureAuthorized(ICommonRequestParameters requestParameters, IMetadataProvider metadataProvider)
+ throws HyracksDataException {
+
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 3e674fd..e72ab54 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -2935,7 +2935,8 @@
Collections.emptyList());
List<List<DependencyFullyQualifiedName>> dependencies =
- ViewUtil.getViewDependencies(viewDecl, foreignKeys, queryRewriter);
+ ViewUtil.getViewDependencies(metadataProvider, viewDecl, foreignKeys, queryRewriter);
+ appCtx.getReceptionist().ensureAuthorized(requestParameters, metadataProvider);
ViewDetails viewDetails = new ViewDetails(cvs.getViewBody(), dependencies, cvs.getDefaultNull(),
primaryKeyFields, foreignKeys, datetimeFormat, dateFormat, timeFormat);
@@ -3238,7 +3239,8 @@
Collections.emptyList());
List<List<DependencyFullyQualifiedName>> dependencies =
- FunctionUtil.getFunctionDependencies(fd, queryRewriter);
+ FunctionUtil.getFunctionDependencies(metadataProvider, fd, queryRewriter);
+ appCtx.getReceptionist().ensureAuthorized(requestParameters, metadataProvider);
newInlineTypes = Collections.emptyMap();
function = new Function(functionSignature, paramNames, null, null, cfs.getFunctionBody(),
@@ -4221,6 +4223,7 @@
clfrqs.setSourceLocation(stmt.getSourceLocation());
JobSpecification jobSpec =
rewriteCompileQuery(hcc, metadataProvider, clfrqs.getQuery(), clfrqs, stmtParams, null);
+ appCtx.getReceptionist().ensureAuthorized(requestParameters, metadataProvider);
afterCompile();
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IReceptionist.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IReceptionist.java
index 95ed22e..d4ba5ca 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IReceptionist.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IReceptionist.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.common.api;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.http.api.IServletRequest;
@@ -47,4 +48,14 @@
* @throws HyracksDataException
*/
void ensureSchedulable(ISchedulableClientRequest schedulableRequest) throws HyracksDataException;
+
+ /**
+ * Ensures a client's request is authorized
+ *
+ * @param requestParameters
+ * @param metadataProvider
+ * @throws HyracksDataException
+ */
+ void ensureAuthorized(ICommonRequestParameters requestParameters, IMetadataProvider metadataProvider)
+ throws HyracksDataException;
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java
index 3e91e22..c9f8640 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java
@@ -51,6 +51,8 @@
import org.apache.asterix.lang.common.statement.ViewDecl;
import org.apache.asterix.lang.common.struct.UnaryExprType;
import org.apache.asterix.lang.common.visitor.GatherFunctionCallsVisitor;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.EntityDetails;
import org.apache.asterix.object.base.AdmArrayNode;
import org.apache.asterix.object.base.AdmBigIntNode;
import org.apache.asterix.object.base.AdmBooleanNode;
@@ -241,8 +243,8 @@
}
}
- public static void collectDependencies(Expression expression, IQueryRewriter rewriter,
- List<DependencyFullyQualifiedName> outDatasetDependencies,
+ public static void collectDependencies(MetadataProvider metadataProvider, Expression expression,
+ IQueryRewriter rewriter, List<DependencyFullyQualifiedName> outDatasetDependencies,
List<DependencyFullyQualifiedName> outSynonymDependencies,
List<DependencyFullyQualifiedName> outFunctionDependencies) throws CompilationException {
// Duplicate elimination
@@ -260,6 +262,13 @@
if (FunctionUtil.isBuiltinDatasetFunction(signature)) {
Triple<DatasetFullyQualifiedName, Boolean, DatasetFullyQualifiedName> dsArgs =
FunctionUtil.parseDatasetFunctionArguments(functionCall);
+ DatasetFullyQualifiedName datasetFullyQualifiedName = dsArgs.first;
+ EntityDetails.EntityType entityType =
+ dsArgs.second ? EntityDetails.EntityType.VIEW : EntityDetails.EntityType.DATASET;
+ metadataProvider
+ .addAccessedEntity(new EntityDetails(datasetFullyQualifiedName.getDatabaseName(),
+ datasetFullyQualifiedName.getDataverseName(),
+ datasetFullyQualifiedName.getDatasetName(), entityType));
DatasetFullyQualifiedName synonymReference = dsArgs.third;
if (synonymReference != null) {
// resolved via synonym -> store synonym name as a dependency
@@ -280,6 +289,9 @@
}
} else {
if (seenFunctions.add(signature)) {
+ String functionName = signature.getName() + "(" + signature.getArity() + ")";
+ metadataProvider.addAccessedEntity(new EntityDetails(signature.getDatabaseName(),
+ signature.getDataverseName(), functionName, EntityDetails.EntityType.FUNCTION));
outFunctionDependencies.add(new DependencyFullyQualifiedName(signature.getDatabaseName(),
signature.getDataverseName(), signature.getName(),
Integer.toString(signature.getArity())));
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
index ad70fca..ffd2089 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
@@ -228,8 +228,8 @@
};
}
- public static List<List<DependencyFullyQualifiedName>> getFunctionDependencies(FunctionDecl fd,
- IQueryRewriter rewriter) throws CompilationException {
+ public static List<List<DependencyFullyQualifiedName>> getFunctionDependencies(MetadataProvider metadataProvider,
+ FunctionDecl fd, IQueryRewriter rewriter) throws CompilationException {
Expression normBody = fd.getNormalizedFuncBody();
if (normBody == null) {
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, fd.getSourceLocation(),
@@ -240,8 +240,8 @@
List<DependencyFullyQualifiedName> datasetDependencies = new ArrayList<>();
List<DependencyFullyQualifiedName> synonymDependencies = new ArrayList<>();
List<DependencyFullyQualifiedName> functionDependencies = new ArrayList<>();
- ExpressionUtils.collectDependencies(normBody, rewriter, datasetDependencies, synonymDependencies,
- functionDependencies);
+ ExpressionUtils.collectDependencies(metadataProvider, normBody, rewriter, datasetDependencies,
+ synonymDependencies, functionDependencies);
List<DependencyFullyQualifiedName> typeDependencies = Collections.emptyList();
return Function.createDependencies(datasetDependencies, functionDependencies, typeDependencies,
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ViewUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ViewUtil.java
index 23a2d1a..238af43 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ViewUtil.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ViewUtil.java
@@ -43,6 +43,7 @@
import org.apache.asterix.lang.common.statement.ViewDecl;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.ViewDetails;
import org.apache.asterix.metadata.utils.TypeUtil;
import org.apache.asterix.om.functions.BuiltinFunctions;
@@ -78,8 +79,9 @@
}
}
- public static List<List<DependencyFullyQualifiedName>> getViewDependencies(ViewDecl viewDecl,
- List<ViewDetails.ForeignKey> foreignKeys, IQueryRewriter rewriter) throws CompilationException {
+ public static List<List<DependencyFullyQualifiedName>> getViewDependencies(MetadataProvider metadataProvider,
+ ViewDecl viewDecl, List<ViewDetails.ForeignKey> foreignKeys, IQueryRewriter rewriter)
+ throws CompilationException {
Expression normBody = viewDecl.getNormalizedViewBody();
if (normBody == null) {
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, viewDecl.getSourceLocation(),
@@ -90,8 +92,8 @@
List<DependencyFullyQualifiedName> datasetDependencies = new ArrayList<>();
List<DependencyFullyQualifiedName> synonymDependencies = new ArrayList<>();
List<DependencyFullyQualifiedName> functionDependencies = new ArrayList<>();
- ExpressionUtils.collectDependencies(normBody, rewriter, datasetDependencies, synonymDependencies,
- functionDependencies);
+ ExpressionUtils.collectDependencies(metadataProvider, normBody, rewriter, datasetDependencies,
+ synonymDependencies, functionDependencies);
if (foreignKeys != null) {
DatasetFullyQualifiedName viewName = viewDecl.getViewName();
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java
index 355b484..8b44df3 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java
@@ -24,13 +24,14 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.common.metadata.DatasetFullyQualifiedName;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.ILangExpression;
import org.apache.asterix.lang.common.expression.CallExpr;
import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
-import org.apache.asterix.lang.common.statement.CopyToStatement;
-import org.apache.asterix.lang.common.statement.ExternalDetailsDecl;
+import org.apache.asterix.lang.common.statement.FunctionDecl;
+import org.apache.asterix.lang.common.statement.ViewDecl;
import org.apache.asterix.lang.common.util.ExpressionUtils;
import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppSimpleExpressionVisitor;
import org.apache.asterix.metadata.entities.EntityDetails;
@@ -45,33 +46,15 @@
}
@Override
- public Expression visit(CopyToStatement stmtCopy, ILangExpression arg) throws CompilationException {
- ExternalDetailsDecl externalDetailsDecl = stmtCopy.getExternalDetailsDecl();
- Map<String, String> properties = externalDetailsDecl.getProperties();
- String databaseName = properties.getOrDefault("database", "Default");
- String dataverseNameString = properties.getOrDefault("dataverse", "Default");
- String linkName = properties.getOrDefault("name", "");
- DataverseName dataverseName;
- try {
- dataverseName = DataverseName.createFromCanonicalForm(dataverseNameString);
- } catch (AsterixException e) {
- throw new IllegalStateException(e);
- }
-
- context.getMetadataProvider().addAccessedEntity(
- new EntityDetails(databaseName, dataverseName, linkName, EntityDetails.EntityType.LINK));
- return super.visit(stmtCopy, arg);
+ public Expression visit(CallExpr expression, ILangExpression arg) throws CompilationException {
+ addAccessedEntities(expression);
+ return super.visit(expression, arg);
}
- @Override
- public Expression visit(CallExpr expression, ILangExpression arg) {
+ private void addAccessedEntities(CallExpr expression) {
if (BuiltinFunctions.DATASET.equals(expression.getFunctionSignature().createFunctionIdentifier())) {
List<Expression> exprs = expression.getExprList();
String databaseName, dataverseNameArg, datasetName;
- EntityDetails.EntityType entityType;
- entityType = exprs.size() > 3 && Boolean.TRUE.equals(ExpressionUtils.getBooleanLiteral(exprs.get(3)))
- ? EntityDetails.EntityType.VIEW : EntityDetails.EntityType.DATASET;
-
databaseName = ExpressionUtils.getStringLiteral(exprs.get(0));
dataverseNameArg = ExpressionUtils.getStringLiteral(exprs.get(1));
DataverseName dataverseName;
@@ -82,14 +65,28 @@
}
datasetName = ExpressionUtils.getStringLiteral(exprs.get(2));
+ EntityDetails.EntityType entityType = EntityDetails.EntityType.DATASET;
+ if (exprs.size() > 3 && Boolean.TRUE.equals(ExpressionUtils.getBooleanLiteral(exprs.get(3)))) {
+ DatasetFullyQualifiedName viewDatasetName =
+ new DatasetFullyQualifiedName(databaseName, dataverseName, datasetName);
+ Map<DatasetFullyQualifiedName, ViewDecl> declaredViews = context.getDeclaredViews();
+ if (declaredViews.containsKey(viewDatasetName)) {
+ return;
+ }
+ entityType = EntityDetails.EntityType.VIEW;
+ }
+
context.getMetadataProvider()
.addAccessedEntity(new EntityDetails(databaseName, dataverseName, datasetName, entityType));
} else {
FunctionSignature signature = expression.getFunctionSignature();
+ Map<FunctionSignature, FunctionDecl> declaredFunctions = context.getDeclaredFunctions();
+ if (declaredFunctions.containsKey(signature)) {
+ return;
+ }
String functionName = signature.getName() + "(" + signature.getArity() + ")";
context.getMetadataProvider().addAccessedEntity(new EntityDetails(signature.getDatabaseName(),
signature.getDataverseName(), functionName, EntityDetails.EntityType.FUNCTION));
}
- return expression;
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 6ec29da..e0f44ea 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -204,7 +204,7 @@
private final INamespaceResolver namespaceResolver;
private IDataFormat dataFormat = FormatUtils.getDefaultFormat();
- private final Set<EntityDetails> getAccessedEntities;
+ private final Set<EntityDetails> accessedEntities;
public static MetadataProvider createWithDefaultNamespace(ICcApplicationContext appCtx) {
java.util.function.Function<ICcApplicationContext, IMetadataProvider<?, ?>> factory =
@@ -231,7 +231,7 @@
dataPartitioningProvider = (DataPartitioningProvider) appCtx.getDataPartitioningProvider();
locks = new LockList();
config = new HashMap<>();
- getAccessedEntities = new HashSet<>();
+ accessedEntities = new HashSet<>();
setDefaultNamespace(MetadataConstants.DEFAULT_NAMESPACE);
}
@@ -1953,11 +1953,11 @@
}
public void addAccessedEntity(EntityDetails entityDetails) {
- getAccessedEntities.add(entityDetails);
+ accessedEntities.add(entityDetails);
}
- public Set<EntityDetails> getGetAccessedEntities() {
- return Collections.unmodifiableSet(getAccessedEntities);
+ public Set<EntityDetails> getAccessedEntities() {
+ return Collections.unmodifiableSet(accessedEntities);
}
private void validateDatabaseObjectNameImpl(String name, SourceLocation sourceLoc) throws AlgebricksException {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java
index c030176..a341a39 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java
@@ -25,19 +25,18 @@
public enum EntityType {
DATASET,
VIEW,
- FUNCTION,
- LINK
+ FUNCTION
}
private final String databaseName;
private final DataverseName dataverseName;
- private final String datasetName;
+ private final String entityName;
private final EntityType entityType;
- public EntityDetails(String databaseName, DataverseName dataverseName, String datasetName, EntityType entityType) {
+ public EntityDetails(String databaseName, DataverseName dataverseName, String entityName, EntityType entityType) {
this.databaseName = databaseName;
this.dataverseName = dataverseName;
- this.datasetName = datasetName;
+ this.entityName = entityName;
this.entityType = entityType;
}
@@ -49,8 +48,8 @@
return dataverseName;
}
- public String getDatasetName() {
- return datasetName;
+ public String getEntityName() {
+ return entityName;
}
public EntityType getEntityType() {