[NO ISSUE][API] Add Execution Plans to Query Services
- user model changes: no
- storage format changes: no
- interface changes: yes
- Add execution plans parameters to query service.
- Remove HTML code generation from APIFramework.
Change-Id: I99215243aae2cb96174671d109084a82af877334
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2566
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>
diff --git a/asterixdb/asterix-algebra/pom.xml b/asterixdb/asterix-algebra/pom.xml
index 5a61c96..79dc7bc 100644
--- a/asterixdb/asterix-algebra/pom.xml
+++ b/asterixdb/asterix-algebra/pom.xml
@@ -242,5 +242,9 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlans.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlans.java
new file mode 100644
index 0000000..d77164c
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlans.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.translator;
+
+import java.io.Serializable;
+
+public class ExecutionPlans implements Serializable {
+
+ private String expressionTree;
+ private String rewrittenExpressionTree;
+ private String logicalPlan;
+ private String optimizedLogicalPlan;
+ private String job;
+
+ public String getExpressionTree() {
+ return expressionTree;
+ }
+
+ public void setExpressionTree(String expressionTree) {
+ this.expressionTree = expressionTree;
+ }
+
+ public String getRewrittenExpressionTree() {
+ return rewrittenExpressionTree;
+ }
+
+ public void setRewrittenExpressionTree(String rewrittenExpressionTree) {
+ this.rewrittenExpressionTree = rewrittenExpressionTree;
+ }
+
+ public String getLogicalPlan() {
+ return logicalPlan;
+ }
+
+ public void setLogicalPlan(String logicalPlan) {
+ this.logicalPlan = logicalPlan;
+ }
+
+ public String getOptimizedLogicalPlan() {
+ return optimizedLogicalPlan;
+ }
+
+ public void setOptimizedLogicalPlan(String optimizedLogicalPlan) {
+ this.optimizedLogicalPlan = optimizedLogicalPlan;
+ }
+
+ public String getJob() {
+ return job;
+ }
+
+ public void setJob(String job) {
+ this.job = job;
+ }
+}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansHtmlPrintUtil.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansHtmlPrintUtil.java
new file mode 100644
index 0000000..88e8255
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansHtmlPrintUtil.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.translator;
+
+import java.io.PrintWriter;
+
+public class ExecutionPlansHtmlPrintUtil {
+
+ private static final String LOGICAL_PLAN_LBL = "Logical plan";
+ private static final String EXPRESSION_TREE_LBL = "Expression tree";
+ private static final String REWRITTEN_EXPRESSION_TREE_LBL = "Rewritten expression tree";
+ private static final String OPTIMIZED_LOGICAL_PLAN_LBL = "Optimized logical plan";
+ private static final String JOB_LBL = "Job";
+
+ private ExecutionPlansHtmlPrintUtil() {
+ }
+
+ public static void print(PrintWriter output, ExecutionPlans plans) {
+ printNonNull(output, EXPRESSION_TREE_LBL, plans.getExpressionTree());
+ printNonNull(output, REWRITTEN_EXPRESSION_TREE_LBL, plans.getRewrittenExpressionTree());
+ printNonNull(output, LOGICAL_PLAN_LBL, plans.getLogicalPlan());
+ printNonNull(output, OPTIMIZED_LOGICAL_PLAN_LBL, plans.getOptimizedLogicalPlan());
+ printNonNull(output, JOB_LBL, plans.getJob());
+ }
+
+ private static void printNonNull(PrintWriter output, String lbl, String value) {
+ if (value != null) {
+ printFieldPrefix(output, lbl);
+ output.print(value);
+ printFieldPostfix(output);
+ }
+ }
+
+ private static void printFieldPrefix(PrintWriter output, String lbl) {
+ output.println();
+ output.println("<h4>" + lbl + ":</h4>");
+ switch (lbl) {
+ case LOGICAL_PLAN_LBL:
+ output.println("<pre class=query-plan>");
+ break;
+ case OPTIMIZED_LOGICAL_PLAN_LBL:
+ output.println("<pre class=query-optimized-plan>");
+ break;
+ default:
+ output.println("<pre>");
+ break;
+ }
+ }
+
+ private static void printFieldPostfix(PrintWriter output) {
+ output.println("</pre>");
+ }
+}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansJsonPrintUtil.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansJsonPrintUtil.java
new file mode 100644
index 0000000..5c47ca2
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansJsonPrintUtil.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.translator;
+
+import static org.apache.asterix.translator.SessionConfig.PlanFormat.STRING;
+
+import org.apache.hyracks.util.JSONUtil;
+
+public class ExecutionPlansJsonPrintUtil {
+
+ private static final String LOGICAL_PLAN_LBL = "logicalPlan";
+ private static final String EXPRESSION_TREE_LBL = "expressionTree";
+ private static final String REWRITTEN_EXPRESSION_TREE_LBL = "rewrittenExpressionTree";
+ private static final String OPTIMIZED_LOGICAL_PLAN_LBL = "optimizedLogicalPlan";
+ private static final String JOB_LBL = "job";
+
+ private ExecutionPlansJsonPrintUtil() {
+ }
+
+ public static String asJson(ExecutionPlans plans, SessionConfig.PlanFormat format) {
+ final StringBuilder output = new StringBuilder();
+ appendOutputPrefix(output);
+ // TODO only string is currently supported for expression trees
+ appendNonNull(output, EXPRESSION_TREE_LBL, plans.getExpressionTree(), STRING);
+ appendNonNull(output, REWRITTEN_EXPRESSION_TREE_LBL, plans.getRewrittenExpressionTree(), STRING);
+ appendNonNull(output, LOGICAL_PLAN_LBL, plans.getLogicalPlan(), format);
+ appendNonNull(output, OPTIMIZED_LOGICAL_PLAN_LBL, plans.getOptimizedLogicalPlan(), format);
+ appendNonNull(output, JOB_LBL, plans.getJob(), format);
+ appendOutputPostfix(output);
+ return output.toString();
+ }
+
+ private static void appendNonNull(StringBuilder builder, String lbl, String value,
+ SessionConfig.PlanFormat format) {
+ if (value != null) {
+ printFieldPrefix(builder, lbl);
+ switch (format) {
+ case JSON:
+ builder.append(value);
+ break;
+ case STRING:
+ JSONUtil.quoteAndEscape(builder, value);
+ break;
+ default:
+ throw new IllegalStateException("Unrecognized plan format: " + format);
+ }
+ printFieldPostfix(builder);
+ }
+ }
+
+ private static void appendOutputPrefix(StringBuilder builder) {
+ builder.append("{");
+ }
+
+ private static void printFieldPrefix(StringBuilder builder, String lbl) {
+ builder.append("\"" + lbl + "\": ");
+ }
+
+ private static void printFieldPostfix(StringBuilder builder) {
+ builder.append(",");
+ }
+
+ private static void appendOutputPostfix(StringBuilder builder) {
+ // remove extra comma if needed
+ if (builder.length() > 1) {
+ builder.deleteCharAt(builder.length() - 1);
+ }
+ builder.append("}");
+ }
+}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
index d76c421..0ff877b 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
@@ -142,4 +142,11 @@
*/
String getActiveDataverseName(String dataverse);
+ /**
+ * Gets the execution plans that are generated during query compilation
+ *
+ * @return the executions plans
+ */
+ ExecutionPlans getExecutionPlans();
+
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
index cb6d8e5..89619e5 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
@@ -63,7 +63,7 @@
if (fmtString != null) {
String format = ("JSON".equalsIgnoreCase(fmtString) || "CLEAN_JSON".equalsIgnoreCase(fmtString))
? "JSON" : fmtString;
- return PlanFormat.valueOf(format);
+ return PlanFormat.valueOf(format.toUpperCase());
}
} catch (IllegalArgumentException e) {
logger.log(Level.INFO, fmtString + ": unsupported " + label + ", using " + defaultFmt + "instead", e);
@@ -129,7 +129,7 @@
// Output format.
private final OutputFormat fmt;
- private final PlanFormat lpfmt;
+ private final PlanFormat planFormat;
// Standard execution flags.
private final boolean executeQuery;
@@ -143,8 +143,8 @@
this(fmt, PlanFormat.STRING);
}
- public SessionConfig(OutputFormat fmt, PlanFormat lpfmt) {
- this(fmt, true, true, true, lpfmt);
+ public SessionConfig(OutputFormat fmt, PlanFormat planFormat) {
+ this(fmt, true, true, true, planFormat);
}
/**
@@ -168,13 +168,13 @@
}
public SessionConfig(OutputFormat fmt, boolean optimize, boolean executeQuery, boolean generateJobSpec,
- PlanFormat lpfmt) {
+ PlanFormat planFormat) {
this.fmt = fmt;
this.optimize = optimize;
this.executeQuery = executeQuery;
this.generateJobSpec = generateJobSpec;
this.flags = new HashMap<>();
- this.lpfmt = lpfmt;
+ this.planFormat = planFormat;
}
/**
@@ -187,8 +187,8 @@
/**
* Retrieve the PlanFormat for this execution.
*/
- public PlanFormat getLpfmt() {
- return this.lpfmt;
+ public PlanFormat getPlanFormat() {
+ return this.planFormat;
}
/**
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index ad715a4..537625d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.io.PrintWriter;
+import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -67,6 +68,7 @@
import org.apache.asterix.optimizer.rules.am.AbstractIntroduceAccessMethodRule;
import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
+import org.apache.asterix.translator.ExecutionPlans;
import org.apache.asterix.translator.IStatementExecutor.Stats;
import org.apache.asterix.translator.SessionConfig;
import org.apache.asterix.translator.SessionOutput;
@@ -120,8 +122,7 @@
private static final int MIN_FRAME_LIMIT_FOR_JOIN = 5;
// one for query, two for intermediate results, one for final result, and one for reading an inverted list
private static final int MIN_FRAME_LIMIT_FOR_TEXTSEARCH = 5;
- private static final String LPLAN = "Logical plan";
- private static final String OPLAN = "Optimized logical plan";
+ private static final ObjectWriter OBJECT_WRITER = new ObjectMapper().writerWithDefaultPrettyPrinter();
// A white list of supported configurable parameters.
private static final Set<String> CONFIGURABLE_PARAMETER_NAMES =
@@ -137,12 +138,14 @@
private final IAstPrintVisitorFactory astPrintVisitorFactory;
private final ILangExpressionToPlanTranslatorFactory translatorFactory;
private final IRuleSetFactory ruleSetFactory;
+ private final ExecutionPlans executionPlans;
public APIFramework(ILangCompilationProvider compilationProvider) {
this.rewriterFactory = compilationProvider.getRewriterFactory();
this.astPrintVisitorFactory = compilationProvider.getAstPrintVisitorFactory();
this.translatorFactory = compilationProvider.getExpressionToPlanTranslatorFactory();
this.ruleSetFactory = compilationProvider.getRuleSetFactory();
+ executionPlans = new ExecutionPlans();
}
private static class OptimizationContextFactory implements IOptimizationContextFactory {
@@ -165,27 +168,6 @@
}
}
- private void printPlanPrefix(SessionOutput output, String planName) {
- if (output.config().is(SessionConfig.FORMAT_HTML)) {
- output.out().println("<h4>" + planName + ":</h4>");
- if (LPLAN.equalsIgnoreCase(planName)) {
- output.out().println("<pre class = query-plan>");
- } else if (OPLAN.equalsIgnoreCase(planName)) {
- output.out().println("<pre class = query-optimized-plan>");
- } else {
- output.out().println("<pre>");
- }
- } else {
- output.out().println("----------" + planName + ":");
- }
- }
-
- private void printPlanPostfix(SessionOutput output) {
- if (output.config().is(SessionConfig.FORMAT_HTML)) {
- output.out().println("</pre>");
- }
- }
-
public Pair<IReturningStatement, Integer> reWriteQuery(List<FunctionDecl> declaredFunctions,
MetadataProvider metadataProvider, IReturningStatement q, SessionOutput output, boolean inlineUdfs)
throws CompilationException {
@@ -194,10 +176,7 @@
}
SessionConfig conf = output.config();
if (!conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_EXPR_TREE)) {
- output.out().println();
- printPlanPrefix(output, "Expression tree");
- q.accept(astPrintVisitorFactory.createLangVisitor(output.out()), 0);
- printPlanPostfix(output);
+ generateExpressionTree(q);
}
IQueryRewriter rw = rewriterFactory.createQueryRewriter();
rw.rewrite(declaredFunctions, q, metadataProvider, new LangRewritingContext(q.getVarCounter()), inlineUdfs);
@@ -213,14 +192,9 @@
final boolean isLoad = statement != null && statement.getKind() == Statement.Kind.LOAD;
SessionConfig conf = output.config();
- if (!conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_REWRITTEN_EXPR_TREE)) {
- output.out().println();
-
- printPlanPrefix(output, "Rewritten expression tree");
- if (isQuery) {
- query.accept(astPrintVisitorFactory.createLangVisitor(output.out()), 0);
- }
- printPlanPostfix(output);
+ if (isQuery && !conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS)
+ && conf.is(SessionConfig.OOB_REWRITTEN_EXPR_TREE)) {
+ generateRewrittenExpressionTree(query);
}
final TxnId txnId = metadataProvider.getTxnIdFactory().create();
@@ -230,14 +204,9 @@
ILogicalPlan plan = isLoad ? t.translateLoad(statement) : t.translate(query, outputDatasetName, statement);
- if (!conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_LOGICAL_PLAN)) {
- output.out().println();
-
- printPlanPrefix(output, "Logical plan");
- if (isQuery || isLoad) {
- PlanPrettyPrinter.printPlan(plan, getPrettyPrintVisitor(output.config().getLpfmt(), output.out()), 0);
- }
- printPlanPostfix(output);
+ if ((isQuery || isLoad) && !conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS)
+ && conf.is(SessionConfig.OOB_LOGICAL_PLAN)) {
+ generateLogicalPlan(plan, output.config().getPlanFormat());
}
CompilerProperties compilerProperties = metadataProvider.getApplicationContext().getCompilerProperties();
Map<String, String> querySpecificConfig = validateConfig(metadataProvider.getConfig());
@@ -273,12 +242,9 @@
AlgebricksAppendable buffer = new AlgebricksAppendable(output.out());
PlanPrettyPrinter.printPhysicalOps(plan, buffer, 0);
} else {
- printPlanPrefix(output, "Optimized logical plan");
if (isQuery || isLoad) {
- PlanPrettyPrinter.printPlan(plan,
- getPrettyPrintVisitor(output.config().getLpfmt(), output.out()), 0);
+ generateOptimizedLogicalPlan(plan, output.config().getPlanFormat());
}
- printPlanPostfix(output);
}
}
}
@@ -327,8 +293,9 @@
ResourceUtils.getRequiredCapacity(plan, jobLocations, physOptConf);
spec.setRequiredClusterCapacity(jobRequiredCapacity);
}
-
- printJobSpec(query, spec, conf, output);
+ if (isQuery && conf.is(SessionConfig.OOB_HYRACKS_JOB)) {
+ generateJob(spec);
+ }
return spec;
}
@@ -373,23 +340,6 @@
}
}
- protected void printJobSpec(Query rwQ, JobSpecification spec, SessionConfig conf, SessionOutput output)
- throws AlgebricksException {
- if (conf.is(SessionConfig.OOB_HYRACKS_JOB)) {
- printPlanPrefix(output, "Hyracks job");
- if (rwQ != null) {
- try {
- final ObjectWriter objectWriter = new ObjectMapper().writerWithDefaultPrettyPrinter();
- output.out().println(objectWriter.writeValueAsString(spec.toJSON()));
- } catch (IOException e) {
- throw new AlgebricksException(e);
- }
- output.out().println(spec.getUserConstraints());
- }
- printPlanPostfix(output);
- }
- }
-
private AbstractLogicalOperatorPrettyPrintVisitor getPrettyPrintVisitor(SessionConfig.PlanFormat planFormat,
PrintWriter out) {
return planFormat.equals(SessionConfig.PlanFormat.JSON) ? new LogicalOperatorPrettyPrintVisitorJson(out)
@@ -429,6 +379,10 @@
}
}
+ public ExecutionPlans getExecutionPlans() {
+ return executionPlans;
+ }
+
// Chooses the location constraints, i.e., whether to use storage parallelism or use a user-sepcified number
// of cores.
private static AlgebricksAbsolutePartitionConstraint chooseLocations(IClusterInfoCollector clusterInfoCollector,
@@ -524,6 +478,49 @@
return config;
}
+ private void generateExpressionTree(IReturningStatement statement) throws CompilationException {
+ final StringWriter stringWriter = new StringWriter();
+ try (PrintWriter writer = new PrintWriter(stringWriter)) {
+ statement.accept(astPrintVisitorFactory.createLangVisitor(writer), 0);
+ executionPlans.setExpressionTree(stringWriter.toString());
+ }
+ }
+
+ private void generateRewrittenExpressionTree(IReturningStatement statement) throws CompilationException {
+ final StringWriter stringWriter = new StringWriter();
+ try (PrintWriter writer = new PrintWriter(stringWriter)) {
+ statement.accept(astPrintVisitorFactory.createLangVisitor(writer), 0);
+ executionPlans.setRewrittenExpressionTree(stringWriter.toString());
+ }
+ }
+
+ private void generateLogicalPlan(ILogicalPlan plan, SessionConfig.PlanFormat format) throws AlgebricksException {
+ final StringWriter stringWriter = new StringWriter();
+ try (PrintWriter writer = new PrintWriter(stringWriter)) {
+ PlanPrettyPrinter.printPlan(plan, getPrettyPrintVisitor(format, writer), 0);
+ executionPlans.setLogicalPlan(stringWriter.toString());
+ }
+ }
+
+ private void generateOptimizedLogicalPlan(ILogicalPlan plan, SessionConfig.PlanFormat format)
+ throws AlgebricksException {
+ final StringWriter stringWriter = new StringWriter();
+ try (PrintWriter writer = new PrintWriter(stringWriter)) {
+ PlanPrettyPrinter.printPlan(plan, getPrettyPrintVisitor(format, writer), 0);
+ executionPlans.setOptimizedLogicalPlan(stringWriter.toString());
+ }
+ }
+
+ private void generateJob(JobSpecification spec) {
+ final StringWriter stringWriter = new StringWriter();
+ try (PrintWriter writer = new PrintWriter(stringWriter)) {
+ writer.println(OBJECT_WRITER.writeValueAsString(spec.toJSON()));
+ executionPlans.setJob(stringWriter.toString());
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
public static AlgebricksAbsolutePartitionConstraint getJobLocations(JobSpecification spec,
INodeJobTracker jobTracker, AlgebricksAbsolutePartitionConstraint clusterLocations) {
final Set<String> jobParticipatingNodes = jobTracker.getJobParticipatingNodes(spec);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
index b8c737d..bd096dd 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
@@ -51,7 +51,8 @@
RESULTS("results"),
HANDLE("handle"),
ERRORS("errors"),
- METRICS("metrics");
+ METRICS("metrics"),
+ PLANS("plans");
private final String str;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
index a420efc..1713ca5 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
@@ -132,6 +132,7 @@
} else {
sessionOutput.out().append(responseMsg.getResult());
}
+ printExecutionPlans(sessionOutput, responseMsg.getExecutionPlans());
}
private void cancelQuery(INCMessageBroker messageBroker, String nodeId, String clientContextID, Exception exception,
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index 56359e3..714bb53 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -45,6 +45,8 @@
import org.apache.asterix.lang.common.base.IParser;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.translator.ExecutionPlans;
+import org.apache.asterix.translator.ExecutionPlansJsonPrintUtil;
import org.apache.asterix.translator.IRequestParameters;
import org.apache.asterix.translator.IStatementExecutor;
import org.apache.asterix.translator.IStatementExecutor.ResultDelivery;
@@ -73,6 +75,7 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
+
import io.netty.handler.codec.http.HttpResponseStatus;
public class QueryServiceServlet extends AbstractQueryApiServlet {
@@ -141,7 +144,12 @@
MODE("mode"),
TIMEOUT("timeout"),
PLAN_FORMAT("plan-format"),
- MAX_RESULT_READS("max-result-reads");
+ MAX_RESULT_READS("max-result-reads"),
+ EXPRESSION_TREE("expression-tree"),
+ REWRITTEN_EXPRESSION_TREE("rewritten-expression-tree"),
+ LOGICAL_PLAN("logical-plan"),
+ OPTIMIZED_LOGICAL_PLAN("optimized-logical-plan"),
+ JOB("job");
private final String str;
@@ -198,6 +206,12 @@
String clientContextID;
String mode;
String maxResultReads;
+ String planFormat;
+ boolean expressionTree;
+ boolean rewrittenExpressionTree;
+ boolean logicalPlan;
+ boolean optimizedLogicalPlan;
+ boolean job;
@Override
public String toString() {
@@ -213,6 +227,12 @@
on.put("format", format);
on.put("timeout", timeout);
on.put("maxResultReads", maxResultReads);
+ on.put("planFormat", planFormat);
+ on.put("expressionTree", expressionTree);
+ on.put("rewrittenExpressionTree", rewrittenExpressionTree);
+ on.put("logicalPlan", logicalPlan);
+ on.put("optimizedLogicalPlan", optimizedLogicalPlan);
+ on.put("job", job);
return om.writer(new MinimalPrettyPrinter()).writeValueAsString(on);
} catch (JsonProcessingException e) { // NOSONAR
LOGGER.debug("unexpected exception marshalling {} instance to json", getClass(), e);
@@ -307,9 +327,15 @@
SessionOutput.ResultAppender appendStatus = ResultUtil.createResultStatusAppender();
SessionConfig.OutputFormat format = getFormat(param.format);
- //TODO:get the parameters from UI.Currently set to clean_json.
- SessionConfig sessionConfig = new SessionConfig(format);
+ final SessionConfig.PlanFormat planFormat =
+ SessionConfig.PlanFormat.get(param.planFormat, param.planFormat, SessionConfig.PlanFormat.JSON, LOGGER);
+ SessionConfig sessionConfig = new SessionConfig(format, planFormat);
sessionConfig.set(SessionConfig.FORMAT_WRAPPER_ARRAY, true);
+ sessionConfig.set(SessionConfig.OOB_EXPR_TREE, param.expressionTree);
+ sessionConfig.set(SessionConfig.OOB_REWRITTEN_EXPR_TREE, param.rewrittenExpressionTree);
+ sessionConfig.set(SessionConfig.OOB_LOGICAL_PLAN, param.logicalPlan);
+ sessionConfig.set(SessionConfig.OOB_OPTIMIZED_LOGICAL_PLAN, param.optimizedLogicalPlan);
+ sessionConfig.set(SessionConfig.OOB_HYRACKS_JOB, param.job);
sessionConfig.set(SessionConfig.FORMAT_INDENT_JSON, param.pretty);
sessionConfig.set(SessionConfig.FORMAT_QUOTE_RECORD,
format != SessionConfig.OutputFormat.CLEAN_JSON && format != SessionConfig.OutputFormat.LOSSLESS_JSON);
@@ -391,6 +417,13 @@
param.clientContextID = getOptText(jsonRequest, Parameter.CLIENT_ID.str());
param.timeout = getOptText(jsonRequest, Parameter.TIMEOUT.str());
param.maxResultReads = getOptText(jsonRequest, Parameter.MAX_RESULT_READS.str());
+ param.planFormat = getOptText(jsonRequest, Parameter.PLAN_FORMAT.str());
+ param.expressionTree = getOptBoolean(jsonRequest, Parameter.EXPRESSION_TREE.str(), false);
+ param.rewrittenExpressionTree =
+ getOptBoolean(jsonRequest, Parameter.REWRITTEN_EXPRESSION_TREE.str(), false);
+ param.logicalPlan = getOptBoolean(jsonRequest, Parameter.LOGICAL_PLAN.str(), false);
+ param.optimizedLogicalPlan = getOptBoolean(jsonRequest, Parameter.OPTIMIZED_LOGICAL_PLAN.str(), false);
+ param.job = getOptBoolean(jsonRequest, Parameter.JOB.str(), false);
} catch (JsonParseException | JsonMappingException e) {
// if the JSON parsing fails, the statement is empty and we get an empty statement error
GlobalConfig.ASTERIX_LOGGER.log(Level.ERROR, e.getMessage(), e);
@@ -406,6 +439,7 @@
param.clientContextID = request.getParameter(Parameter.CLIENT_ID.str());
param.timeout = request.getParameter(Parameter.TIMEOUT.str());
param.maxResultReads = request.getParameter(Parameter.MAX_RESULT_READS.str());
+ param.planFormat = request.getParameter(Parameter.PLAN_FORMAT.str());
}
return param;
}
@@ -533,6 +567,7 @@
getHyracksDataset(), resultProperties, stats, null, param.clientContextID, optionalParameters);
translator.compileAndExecute(getHyracksClientConnection(), queryCtx, requestParameters);
execution.end();
+ printExecutionPlans(sessionOutput, translator.getExecutionPlans());
}
protected void handleExecuteStatementException(Throwable t, RequestExecutionState state, RequestParameters param) {
@@ -566,4 +601,21 @@
state.setStatus(ResultStatus.FATAL, HttpResponseStatus.INTERNAL_SERVER_ERROR);
}
}
+
+ protected void printExecutionPlans(SessionOutput output, ExecutionPlans executionPlans) {
+ final PrintWriter pw = output.out();
+ pw.print("\t\"");
+ pw.print(ResultFields.PLANS.str());
+ pw.print("\":");
+ final SessionConfig.PlanFormat planFormat = output.config().getPlanFormat();
+ switch (planFormat) {
+ case JSON:
+ case STRING:
+ pw.print(ExecutionPlansJsonPrintUtil.asJson(executionPlans, planFormat));
+ break;
+ default:
+ throw new IllegalStateException("Unrecognized plan format: " + planFormat);
+ }
+ pw.print(",\n");
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
index 53d4f3f..d295304 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
@@ -131,6 +131,7 @@
responseMsg.setResult(outWriter.toString());
responseMsg.setMetadata(outMetadata);
responseMsg.setStats(stats);
+ responseMsg.setExecutionPlans(translator.getExecutionPlans());
} catch (AlgebricksException | HyracksException | TokenMgrError
| org.apache.asterix.aqlplus.parser.TokenMgrError pe) {
// we trust that "our" exceptions are serializable and have a comprehensible error message
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java
index 7475be4..94dd541 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java
@@ -23,6 +23,7 @@
import org.apache.asterix.common.messaging.api.INcAddressedMessage;
import org.apache.asterix.common.messaging.api.MessageFuture;
import org.apache.asterix.messaging.NCMessageBroker;
+import org.apache.asterix.translator.ExecutionPlans;
import org.apache.asterix.translator.IStatementExecutor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -39,6 +40,8 @@
private Throwable error;
+ private ExecutionPlans executionPlans;
+
public ExecuteStatementResponseMessage(long requestMessageId) {
this.requestMessageId = requestMessageId;
}
@@ -84,6 +87,14 @@
this.stats = stats;
}
+ public ExecutionPlans getExecutionPlans() {
+ return executionPlans;
+ }
+
+ public void setExecutionPlans(ExecutionPlans executionPlans) {
+ this.executionPlans = executionPlans;
+ }
+
@Override
public String toString() {
return String.format("%s(id=%s): %d characters", getClass().getSimpleName(), requestMessageId,
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 453bcb5..6a89bda 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -156,6 +156,8 @@
import org.apache.asterix.translator.CompiledStatements.CompiledLoadFromFileStatement;
import org.apache.asterix.translator.CompiledStatements.CompiledUpsertStatement;
import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
+import org.apache.asterix.translator.ExecutionPlans;
+import org.apache.asterix.translator.ExecutionPlansHtmlPrintUtil;
import org.apache.asterix.translator.IRequestParameters;
import org.apache.asterix.translator.IStatementExecutor;
import org.apache.asterix.translator.IStatementExecutorContext;
@@ -1773,6 +1775,7 @@
new CompiledLoadFromFileStatement(dataverseName, loadStmt.getDatasetName().getValue(),
loadStmt.getAdapter(), loadStmt.getProperties(), loadStmt.dataIsAlreadySorted());
JobSpecification spec = apiFramework.compileQuery(hcc, metadataProvider, null, 0, null, sessionOutput, cls);
+ afterCompile();
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
if (spec != null) {
@@ -1864,6 +1867,7 @@
stmtDelete.getDatasetName().getValue(), stmtDelete.getCondition(), stmtDelete.getVarCounter(),
stmtDelete.getQuery());
JobSpecification jobSpec = rewriteCompileQuery(hcc, metadataProvider, clfrqs.getQuery(), clfrqs);
+ afterCompile();
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
@@ -2370,6 +2374,7 @@
metadataProvider.setMetadataTxnContext(mdTxnCtx);
try {
final JobSpecification jobSpec = rewriteCompileQuery(hcc, metadataProvider, query, null);
+ afterCompile();
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
return query.isExplain() || !sessionConfig.isExecuteQuery() ? null : jobSpec;
@@ -2778,6 +2783,11 @@
return (dataverse != null) ? dataverse : activeDataverse.getDataverseName();
}
+ @Override
+ public ExecutionPlans getExecutionPlans() {
+ return apiFramework.getExecutionPlans();
+ }
+
public String getActiveDataverse(Identifier dataverse) {
return getActiveDataverseName(dataverse != null ? dataverse.getValue() : null);
}
@@ -2812,4 +2822,10 @@
IStatementRewriter rewriter = rewriterFactory.createStatementRewriter();
rewriter.rewrite(stmt);
}
+
+ protected void afterCompile() {
+ if (sessionOutput.config().is(SessionConfig.FORMAT_HTML)) {
+ ExecutionPlansHtmlPrintUtil.print(sessionOutput.out(), getExecutionPlans());
+ }
+ }
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
index 890667a..d93555d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
@@ -53,7 +53,8 @@
SIGNATURE("signature"),
STATUS("status"),
TYPE("type"),
- ERRORS("errors");
+ ERRORS("errors"),
+ PLANS("plans");
private static final Map<String, ResultField> fields = new HashMap<>();
@@ -162,6 +163,7 @@
case SIGNATURE:
case STATUS:
case TYPE:
+ case PLANS:
resultBuilder.append(OBJECT_MAPPER.writeValueAsString(fieldValue));
break;
default:
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java
index 6085c1c..baa3174 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java
@@ -149,7 +149,7 @@
return quoteAndEscape(new StringBuilder(), str).toString();
}
- private static StringBuilder quoteAndEscape(StringBuilder sb, String str) {
+ public static StringBuilder quoteAndEscape(StringBuilder sb, String str) {
return escape(sb.append('"'), str).append('"');
}