[NO ISSUE] Add "compile-only" request parameter
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Add "compile-only" request parameter to QueryServiceServlet
- Minor compiler enhancements as follows:
- Add ICompiledDmlStatement.getCategory()
- Add Statement.Category.toString()
- ConstantFoldingRule should check that input type is
a record when processing 'field-access-by-name'
Change-Id: I123c039232bf453c5cb41bf67b4a1ff14c231bf8
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13404
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dmitry.lychagin@couchbase.com>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index 0c2eeba..f8e84ad 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@ -233,12 +233,15 @@
try {
if (expr.getFunctionIdentifier().equals(BuiltinFunctions.FIELD_ACCESS_BY_NAME)) {
- ARecordType rt = (ARecordType) _emptyTypeEnv.getType(expr.getArguments().get(0).getValue());
- String str = ConstantExpressionUtil.getStringConstant(expr.getArguments().get(1).getValue());
- int k = rt.getFieldIndex(str);
- if (k >= 0) {
- // wait for the ByNameToByIndex rule to apply
- return new Pair<>(changed, expr);
+ IAType argType = (IAType) _emptyTypeEnv.getType(expr.getArguments().get(0).getValue());
+ if (argType.getTypeTag() == ATypeTag.OBJECT) {
+ ARecordType rt = (ARecordType) argType;
+ String str = ConstantExpressionUtil.getStringConstant(expr.getArguments().get(1).getValue());
+ int k = rt.getFieldIndex(str);
+ if (k >= 0) {
+ // wait for the ByNameToByIndex rule to apply
+ return new Pair<>(changed, expr);
+ }
}
}
IAObject c = FUNC_ID_TO_CONSTANT.get(expr.getFunctionIdentifier());
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
index e85fec8..088676c 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
@@ -193,6 +193,8 @@
DataverseName getDataverseName();
String getDatasetName();
+
+ byte getCategory();
}
public static class CompiledCreateIndexStatement extends AbstractCompiledStatement
@@ -227,6 +229,11 @@
public Statement.Kind getKind() {
return Statement.Kind.CREATE_INDEX;
}
+
+ @Override
+ public byte getCategory() {
+ return Statement.Category.DDL;
+ }
}
public static class CompiledLoadFromFileStatement extends AbstractCompiledStatement
@@ -272,6 +279,11 @@
public Statement.Kind getKind() {
return Statement.Kind.LOAD;
}
+
+ @Override
+ public byte getCategory() {
+ return Statement.Category.UPDATE;
+ }
}
public static class CompiledInsertStatement extends AbstractCompiledStatement implements ICompiledDmlStatement {
@@ -322,6 +334,11 @@
public Statement.Kind getKind() {
return Statement.Kind.INSERT;
}
+
+ @Override
+ public byte getCategory() {
+ return Statement.Category.UPDATE;
+ }
}
public static class CompiledUpsertStatement extends CompiledInsertStatement {
@@ -370,6 +387,11 @@
public Statement.Kind getKind() {
return Statement.Kind.SUBSCRIBE_FEED;
}
+
+ @Override
+ public byte getCategory() {
+ return Statement.Category.UPDATE;
+ }
}
public static class CompiledDeleteStatement extends AbstractCompiledStatement implements ICompiledDmlStatement {
@@ -415,6 +437,10 @@
return Statement.Kind.DELETE;
}
+ @Override
+ public byte getCategory() {
+ return Statement.Category.UPDATE;
+ }
}
public static class CompiledCompactStatement extends AbstractCompiledStatement {
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
index de31277..33646f0 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
@@ -133,7 +133,7 @@
private PlanFormat planFormat;
// Standard execution flags.
- private final boolean executeQuery;
+ private boolean executeQuery;
private final boolean generateJobSpec;
private final boolean optimize;
private long maxWarnings;
@@ -217,6 +217,10 @@
return executeQuery;
}
+ public void setExecuteQuery(boolean executeQuery) {
+ this.executeQuery = executeQuery;
+ }
+
/**
* Retrieve the value of the "optimize" flag.
*/
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
index feb48df..412e5c5 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
@@ -71,6 +71,7 @@
LOGICAL_PLAN("logical-plan"),
OPTIMIZED_LOGICAL_PLAN("optimized-logical-plan"),
PARSE_ONLY("parse-only"),
+ COMPILE_ONLY("compile-only"),
READ_ONLY("readonly"),
JOB("job"),
PROFILE("profile"),
@@ -127,6 +128,7 @@
private boolean pretty = false;
private boolean expressionTree = false;
private boolean parseOnly = false; // don't execute; simply check for syntax correctness and named parameters.
+ private boolean compileOnly = false; // don't execute; compile only.
private boolean readOnly = false; // only allow statements belonging to QUERY category, fail for other categories.
private boolean rewrittenExpressionTree = false;
private boolean logicalPlan = false;
@@ -290,6 +292,14 @@
return parseOnly;
}
+ public void setCompileOnly(boolean compileOnly) {
+ this.compileOnly = compileOnly;
+ }
+
+ public boolean isCompileOnly() {
+ return compileOnly;
+ }
+
public void setReadOnly(boolean readOnly) {
this.readOnly = readOnly;
}
@@ -442,6 +452,7 @@
parseBoolean(req, Parameter.REWRITTEN_EXPRESSION_TREE.str(), valGetter, isRewrittenExpressionTree()));
setLogicalPlan(parseBoolean(req, Parameter.LOGICAL_PLAN.str(), valGetter, isLogicalPlan()));
setParseOnly(parseBoolean(req, Parameter.PARSE_ONLY.str(), valGetter, isParseOnly()));
+ setCompileOnly(parseBoolean(req, Parameter.COMPILE_ONLY.str, valGetter, isCompileOnly()));
setReadOnly(parseBoolean(req, Parameter.READ_ONLY.str(), valGetter, isReadOnly()));
setOptimizedLogicalPlan(
parseBoolean(req, Parameter.OPTIMIZED_LOGICAL_PLAN.str(), valGetter, isOptimizedLogicalPlan()));
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index f66403d..74682cf 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -484,6 +484,7 @@
sessionConfig.setFmt(format);
sessionConfig.setPlanFormat(planFormat);
sessionConfig.setMaxWarnings(param.getMaxWarnings());
+ sessionConfig.setExecuteQuery(!param.isCompileOnly());
sessionConfig.set(SessionConfig.FORMAT_WRAPPER_ARRAY, true);
sessionConfig.set(SessionConfig.OOB_EXPR_TREE, param.isExpressionTree());
sessionConfig.set(SessionConfig.OOB_REWRITTEN_EXPR_TREE, param.isRewrittenExpressionTree());
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 2fa4c7b..0baf2b2 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -108,6 +108,7 @@
import org.apache.asterix.lang.common.expression.IndexedTypeExpression;
import org.apache.asterix.lang.common.expression.TypeExpression;
import org.apache.asterix.lang.common.expression.TypeReferenceExpression;
+import org.apache.asterix.lang.common.expression.VariableExpr;
import org.apache.asterix.lang.common.statement.AdapterDropStatement;
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
@@ -193,6 +194,7 @@
import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.metadata.utils.MetadataUtil;
import org.apache.asterix.metadata.utils.TypeUtil;
+import org.apache.asterix.om.base.ANull;
import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
@@ -443,10 +445,10 @@
metadataProvider.setMaxResultReads(maxResultReads);
}
handleInsertUpsertStatement(metadataProvider, stmt, hcc, resultSet, resultDelivery, outMetadata,
- stats, false, requestParameters, stmtParams, stmtRewriter);
+ stats, requestParameters, stmtParams, stmtRewriter);
break;
case DELETE:
- handleDeleteStatement(metadataProvider, stmt, hcc, false, stmtParams, stmtRewriter);
+ handleDeleteStatement(metadataProvider, stmt, hcc, stmtParams, stmtRewriter);
break;
case CREATE_FEED:
handleCreateFeedStatement(metadataProvider, stmt);
@@ -3418,7 +3420,7 @@
afterCompile();
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
- if (spec != null) {
+ if (spec != null && sessionConfig.isExecuteQuery()) {
runJob(hcc, spec);
}
} catch (Exception e) {
@@ -3433,7 +3435,7 @@
public JobSpecification handleInsertUpsertStatement(MetadataProvider metadataProvider, Statement stmt,
IHyracksClientConnection hcc, IResultSet resultSet, ResultDelivery resultDelivery,
- ResultMetadata outMetadata, Stats stats, boolean compileOnly, IRequestParameters requestParameters,
+ ResultMetadata outMetadata, Stats stats, IRequestParameters requestParameters,
Map<String, IAObject> stmtParams, IStatementRewriter stmtRewriter) throws Exception {
InsertStatement stmtInsertUpsert = (InsertStatement) stmt;
String datasetName = stmtInsertUpsert.getDatasetName();
@@ -3463,7 +3465,7 @@
rewriteCompileInsertUpsert(hcc, metadataProvider, stmtInsertUpsert, stmtParams);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
- return jobSpec;
+ return !sessionConfig.isExecuteQuery() ? null : jobSpec;
} catch (Exception e) {
if (bActiveTxn) {
abort(e, e, mdTxnCtx);
@@ -3471,14 +3473,6 @@
throw e;
}
};
- if (compileOnly) {
- locker.lock();
- try {
- return compiler.compile();
- } finally {
- locker.unlock();
- }
- }
if (stmtInsertUpsert.getReturnExpression() != null) {
deliverResult(hcc, resultSet, compiler, metadataProvider, locker, resultDelivery, outMetadata, stats,
@@ -3499,8 +3493,8 @@
}
public JobSpecification handleDeleteStatement(MetadataProvider metadataProvider, Statement stmt,
- IHyracksClientConnection hcc, boolean compileOnly, Map<String, IAObject> stmtParams,
- IStatementRewriter stmtRewriter) throws Exception {
+ IHyracksClientConnection hcc, Map<String, IAObject> stmtParams, IStatementRewriter stmtRewriter)
+ throws Exception {
DeleteStatement stmtDelete = (DeleteStatement) stmt;
String datasetName = stmtDelete.getDatasetName();
metadataProvider.validateDatabaseObjectName(stmtDelete.getDataverseName(), datasetName,
@@ -3522,7 +3516,7 @@
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
- if (jobSpec != null && !compileOnly) {
+ if (jobSpec != null && sessionConfig.isExecuteQuery()) {
runJob(hcc, jobSpec);
}
return jobSpec;
@@ -3542,7 +3536,7 @@
Map<String, IAObject> stmtParams, IRequestParameters requestParameters)
throws AlgebricksException, ACIDException {
- Map<VarIdentifier, IAObject> externalVars = createExternalVariables(stmtParams);
+ Map<VarIdentifier, IAObject> externalVars = createExternalVariables(query, stmtParams);
// Query Rewriting (happens under the same ongoing metadata transaction)
Pair<IReturningStatement, Integer> rewrittenResult = apiFramework.reWriteQuery(declaredFunctions, null,
@@ -3559,7 +3553,7 @@
throws AlgebricksException, ACIDException {
SourceLocation sourceLoc = insertUpsert.getSourceLocation();
- Map<VarIdentifier, IAObject> externalVars = createExternalVariables(stmtParams);
+ Map<VarIdentifier, IAObject> externalVars = createExternalVariables(insertUpsert, stmtParams);
// Insert/upsert statement rewriting (happens under the same ongoing metadata transaction)
Pair<IReturningStatement, Integer> rewrittenResult = apiFramework.reWriteQuery(declaredFunctions, null,
@@ -4612,16 +4606,31 @@
return i == 0;
}
- private Map<VarIdentifier, IAObject> createExternalVariables(Map<String, IAObject> stmtParams) {
- if (stmtParams == null || stmtParams.isEmpty()) {
- return Collections.emptyMap();
+ private Map<VarIdentifier, IAObject> createExternalVariables(IReturningStatement stmt,
+ Map<String, IAObject> stmtParams) throws CompilationException {
+ if (sessionConfig.isExecuteQuery()) {
+ if (stmtParams == null || stmtParams.isEmpty()) {
+ return Collections.emptyMap();
+ }
+ IQueryRewriter queryRewriter = rewriterFactory.createQueryRewriter();
+ Map<VarIdentifier, IAObject> result = new HashMap<>();
+ for (Map.Entry<String, IAObject> me : stmtParams.entrySet()) {
+ result.put(queryRewriter.toExternalVariableName(me.getKey()), me.getValue());
+ }
+ return result;
+ } else {
+ // compile only. extract statement parameters from the statement body and bind to NULL
+ IQueryRewriter queryRewriter = rewriterFactory.createQueryRewriter();
+ Set<VariableExpr> extVars = queryRewriter.getExternalVariables(stmt.getBody());
+ if (extVars.isEmpty()) {
+ return Collections.emptyMap();
+ }
+ Map<VarIdentifier, IAObject> result = new HashMap<>();
+ for (VariableExpr extVar : extVars) {
+ result.put(extVar.getVar(), ANull.NULL);
+ }
+ return result;
}
- IQueryRewriter queryRewriter = rewriterFactory.createQueryRewriter();
- Map<VarIdentifier, IAObject> m = new HashMap<>();
- for (Map.Entry<String, IAObject> me : stmtParams.entrySet()) {
- m.put(queryRewriter.toExternalVariableName(me.getKey()), me.getValue());
- }
- return m;
}
protected void validateDatasetState(MetadataProvider metadataProvider, Dataset dataset, SourceLocation sourceLoc)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/compileonly/compileonly.1.plans.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/compileonly/compileonly.1.plans.sqlpp
new file mode 100644
index 0000000..6095b26
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/compileonly/compileonly.1.plans.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- param compile-only:string=true
+-- param logical-plan:string=true
+-- param plan-format:string=json
+
+select value v from range(1,2) v where v > ?;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/compileonly/compileonly.1.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/compileonly/compileonly.1.regexjson
new file mode 100644
index 0000000..40a764c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/compileonly/compileonly.1.regexjson
@@ -0,0 +1,9 @@
+{
+ "logicalPlan": {
+ "operator":"distribute-result",
+ "expressions":"R{.*}",
+ "operatorId":"R{.*}",
+ "execution-mode":"R{.*}",
+ "inputs":"R{.*}"
+ }
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 3871f40..d32fafe 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -29,6 +29,11 @@
&TemporalQueries;
<test-group name="api">
<test-case FilePath="api">
+ <compilation-unit name="compileonly">
+ <output-dir compare="Text">compileonly</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
<compilation-unit name="multiple-param-values">
<output-dir compare="Text">multiple-param-values</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java
index 663074e..2594cdd 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java
@@ -46,6 +46,21 @@
private Category() {
}
+
+ public static String toString(byte category) {
+ switch (category) {
+ case QUERY:
+ return "query";
+ case UPDATE:
+ return "update";
+ case DDL:
+ return "ddl";
+ case PROCEDURE:
+ return "procedure";
+ default:
+ throw new IllegalArgumentException(String.valueOf(category));
+ }
+ }
}
enum Kind {
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/temporal/ATimeParserFactory.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/temporal/ATimeParserFactory.java
index 8b332cc..ca3552b 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/temporal/ATimeParserFactory.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/temporal/ATimeParserFactory.java
@@ -153,7 +153,7 @@
}
if (length > offset) {
- timezone = parseTimezonePart(timeString, start + offset);
+ parseTimezonePart(timeString, start + offset); // parsed, then ignored
}
return GregorianCalendarSystem.getInstance().getChronon(hour, min, sec, millis);