Several CSV, API, HTTP API, and Web interface improvements.
- APIFramework: internal refactoring to consolidate output PrintWriter,
OutputFormat, and all output flags into SessionConfig
- APIFramework: "HTML" is now a flag, rather than an OutputFormat
- HTTP API: Output format can be select via query parameter in
addition to HTTP Accept header
- CSV: default output is now without header, to improve roundtripping
- CSV: header can be requested via Accept header or "header" query
parameter
- Web interface: Added ability to select output format (JSON, CSV or ADM)
Change-Id: I91398bd30dbd6f3b1f69eb51fbf201010d0e5d93
Reviewed-on: http://fulliautomatix.ics.uci.edu:8443/242
Reviewed-by: Chris Hillery <ceej@lambda.nu>
Tested-by: Chris Hillery <ceej@lambda.nu>
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
index 1ee9a3e..84a01da 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
@@ -153,43 +153,26 @@
}
- /**
- * Used to select the output from the various servlets. Note: "HTML" is
- * primarily intended for use by the built-in web interface. It produces
- * ADM output with various HTML wrappers.
- */
- public enum OutputFormat {
- ADM,
- HTML,
- JSON,
- CSV
- }
-
public static Pair<Query, Integer> reWriteQuery(List<FunctionDecl> declaredFunctions,
- AqlMetadataProvider metadataProvider, Query q, SessionConfig pc, PrintWriter out, OutputFormat pdf)
+ AqlMetadataProvider metadataProvider, Query q, SessionConfig conf)
throws AsterixException {
- if (!pc.isPrintPhysicalOpsOnly() && pc.isPrintExprParam()) {
- out.println();
- switch (pdf) {
- case HTML: {
- out.println("<h4>Expression tree:</h4>");
- out.println("<pre>");
- break;
- }
- default: {
- out.println("----------Expression tree:");
- break;
- }
+ if (conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_EXPR_TREE)) {
+ conf.out().println();
+
+ if (conf.is(SessionConfig.FORMAT_HTML)) {
+ conf.out().println("<h4>Expression tree:</h4>");
+ conf.out().println("<pre>");
+ } else {
+ conf.out().println("----------Expression tree:");
}
+
if (q != null) {
- q.accept(new AQLPrintVisitor(out), 0);
+ q.accept(new AQLPrintVisitor(conf.out()), 0);
}
- switch (pdf) {
- case HTML: {
- out.println("</pre>");
- break;
- }
+
+ if (conf.is(SessionConfig.FORMAT_HTML)) {
+ conf.out().println("</pre>");
}
}
AqlRewriter rw = new AqlRewriter(declaredFunctions, q, metadataProvider);
@@ -200,35 +183,26 @@
public static JobSpecification compileQuery(List<FunctionDecl> declaredFunctions,
AqlMetadataProvider queryMetadataProvider, Query rwQ, int varCounter, String outputDatasetName,
- SessionConfig pc, PrintWriter out, OutputFormat pdf, ICompiledDmlStatement statement)
+ SessionConfig conf, ICompiledDmlStatement statement)
throws AsterixException, AlgebricksException, JSONException, RemoteException, ACIDException {
- if (!pc.isPrintPhysicalOpsOnly() && pc.isPrintRewrittenExprParam()) {
- out.println();
+ if (conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_REWRITTEN_EXPR_TREE)) {
+ conf.out().println();
- switch (pdf) {
- case HTML: {
- out.println("<h4>Rewritten expression tree:</h4>");
- out.println("<pre>");
- break;
- }
- default: {
- out.println("----------Rewritten expression:");
- break;
- }
+ if (conf.is(SessionConfig.FORMAT_HTML)) {
+ conf.out().println("<h4>Rewritten expression tree:</h4>");
+ conf.out().println("<pre>");
+ } else {
+ conf.out().println("----------Rewritten expression:");
}
if (rwQ != null) {
- rwQ.accept(new AQLPrintVisitor(out), 0);
+ rwQ.accept(new AQLPrintVisitor(conf.out()), 0);
}
- switch (pdf) {
- case HTML: {
- out.println("</pre>");
- break;
- }
+ if (conf.is(SessionConfig.FORMAT_HTML)) {
+ conf.out().println("</pre>");
}
-
}
edu.uci.ics.asterix.common.transactions.JobId asterixJobId = JobIdFactory.generateJobId();
@@ -246,31 +220,24 @@
boolean isWriteTransaction = queryMetadataProvider.isWriteTransaction();
LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor();
- if (!pc.isPrintPhysicalOpsOnly() && pc.isPrintLogicalPlanParam()) {
+ if (conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_LOGICAL_PLAN)) {
+ conf.out().println();
- switch (pdf) {
- case HTML: {
- out.println("<h4>Logical plan:</h4>");
- out.println("<pre>");
- break;
- }
- default: {
- out.println("----------Logical plan:");
- break;
- }
+ if (conf.is(SessionConfig.FORMAT_HTML)) {
+ conf.out().println("<h4>Logical plan:</h4>");
+ conf.out().println("<pre>");
+ } else {
+ conf.out().println("----------Logical plan:");
}
if (rwQ != null || statement.getKind() == Kind.LOAD) {
StringBuilder buffer = new StringBuilder();
PlanPrettyPrinter.printPlan(plan, buffer, pvisitor, 0);
- out.print(buffer);
+ conf.out().print(buffer);
}
- switch (pdf) {
- case HTML: {
- out.println("</pre>");
- break;
- }
+ if (conf.is(SessionConfig.FORMAT_HTML)) {
+ conf.out().println("</pre>");
}
}
@@ -305,45 +272,39 @@
builder.setNullableTypeComputer(AqlNullableTypeComputer.INSTANCE);
ICompiler compiler = compilerFactory.createCompiler(plan, queryMetadataProvider, t.getVarCounter());
- if (pc.isOptimize()) {
+ if (conf.isOptimize()) {
compiler.optimize();
//plot optimized logical plan
if (plot)
PlanPlotter.printOptimizedLogicalPlan(plan);
- if (pc.isPrintOptimizedLogicalPlanParam()) {
- if (pc.isPrintPhysicalOpsOnly()) {
+ if (conf.is(SessionConfig.OOB_OPTIMIZED_LOGICAL_PLAN)) {
+ if (conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS)) {
// For Optimizer tests.
StringBuilder buffer = new StringBuilder();
PlanPrettyPrinter.printPhysicalOps(plan, buffer, 0);
- out.print(buffer);
+ conf.out().print(buffer);
} else {
- switch (pdf) {
- case HTML: {
- out.println("<h4>Optimized logical plan:</h4>");
- out.println("<pre>");
- break;
- }
- default: {
- out.println("----------Optimized logical plan:");
- break;
- }
+ if (conf.is(SessionConfig.FORMAT_HTML)) {
+ conf.out().println("<h4>Optimized logical plan:</h4>");
+ conf.out().println("<pre>");
+ } else {
+ conf.out().println("----------Optimized logical plan:");
}
+
if (rwQ != null || statement.getKind() == Kind.LOAD) {
StringBuilder buffer = new StringBuilder();
PlanPrettyPrinter.printPlan(plan, buffer, pvisitor, 0);
- out.print(buffer);
+ conf.out().print(buffer);
}
- switch (pdf) {
- case HTML: {
- out.println("</pre>");
- break;
- }
+
+ if (conf.is(SessionConfig.FORMAT_HTML)) {
+ conf.out().println("</pre>");
}
}
}
}
- if (!pc.isGenerateJobSpec()) {
+ if (!conf.isGenerateJobSpec()) {
return null;
}
@@ -359,16 +320,18 @@
builder.setNullWriterFactory(format.getNullWriterFactory());
builder.setPredicateEvaluatorFactoryProvider(format.getPredicateEvaluatorFactoryProvider());
- switch (pdf) {
+ switch (conf.fmt()) {
case JSON:
builder.setPrinterProvider(format.getJSONPrinterFactoryProvider());
break;
case CSV:
builder.setPrinterProvider(format.getCSVPrinterFactoryProvider());
break;
- default:
+ case ADM:
builder.setPrinterProvider(format.getPrinterFactoryProvider());
break;
+ default:
+ throw new RuntimeException("Unexpected OutputFormat!");
}
builder.setSerializerDeserializerProvider(format.getSerdeProvider());
@@ -379,34 +342,28 @@
isWriteTransaction);
JobSpecification spec = compiler.createJob(AsterixAppContextInfo.getInstance(), jobEventListenerFactory);
- if (pc.isPrintJob()) {
- switch (pdf) {
- case HTML: {
- out.println("<h4>Hyracks job:</h4>");
- out.println("<pre>");
- break;
- }
- default: {
- out.println("----------Hyracks job:");
- break;
- }
+ if (conf.is(SessionConfig.OOB_HYRACKS_JOB)) {
+ if (conf.is(SessionConfig.FORMAT_HTML)) {
+ conf.out().println("<h4>Hyracks job:</h4>");
+ conf.out().println("<pre>");
+ } else {
+ conf.out().println("----------Hyracks job:");
}
+
if (rwQ != null) {
- out.println(spec.toJSON().toString(1));
- out.println(spec.getUserConstraints());
+ conf.out().println(spec.toJSON().toString(1));
+ conf.out().println(spec.getUserConstraints());
}
- switch (pdf) {
- case HTML: {
- out.println("</pre>");
- break;
- }
+
+ if (conf.is(SessionConfig.FORMAT_HTML)) {
+ conf.out().println("</pre>");
}
}
return spec;
}
- public static void executeJobArray(IHyracksClientConnection hcc, JobSpecification[] specs, PrintWriter out,
- OutputFormat pdf) throws Exception {
+ public static void executeJobArray(IHyracksClientConnection hcc, JobSpecification[] specs, PrintWriter out)
+ throws Exception {
for (int i = 0; i < specs.length; i++) {
specs[i].setMaxReattempts(0);
JobId jobId = hcc.startJob(specs[i]);
@@ -419,7 +376,7 @@
}
- public static void executeJobArray(IHyracksClientConnection hcc, Job[] jobs, PrintWriter out, OutputFormat pdf)
+ public static void executeJobArray(IHyracksClientConnection hcc, Job[] jobs, PrintWriter out)
throws Exception {
for (int i = 0; i < jobs.length; i++) {
jobs[i].getJobSpec().setMaxReattempts(0);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/SessionConfig.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/SessionConfig.java
index a2f27b4..a82558f 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/SessionConfig.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/SessionConfig.java
@@ -14,80 +14,188 @@
*/
package edu.uci.ics.asterix.api.common;
+import java.io.PrintWriter;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * SessionConfig captures several different parameters for controlling
+ * the execution of an APIFramework call.
+ * <li> It specifies how the execution will proceed (for instance,
+ * whether to optimize, or whether to execute at all).
+ * <li> It allows you specify where the primary execution output will
+ * be sent.
+ * <li> It also allows you to request additional output for optional
+ * out-of-band data about the execution (query plan, etc).
+ * <li> It allows you to specify the output format for the primary
+ * execution output - JSON, CSV, etc.
+ * <li> It allows you to specify output format-specific parameters.
+ */
+
public class SessionConfig {
- private final boolean optimize;
- private final boolean printExprParam;
- private final boolean printRewrittenExprParam;
- private final boolean printLogicalPlanParam;
- private final boolean printOptimizedLogicalPlanParam;
- private final boolean printPhysicalOpsOnly;
- private final boolean executeQuery;
- private final boolean generateJobSpec;
- private final boolean printJob;
+ /**
+ * Used to specify the output format for the primary execution.
+ */
+ public enum OutputFormat {
+ ADM,
+ JSON,
+ CSV
+ };
/**
- * Note: the various "print" options below will cause additional output
- * to be generated when invoking servlet functions. This output is NOT
- * guaranteed to match the OutputFormat (JSON, ADM...) except when the
- * OutputFormat is "HTML". This is primarily for use by the built-in
- * web interface (APIServlet).
- * @param optimize
- * @param printExprParam
- * @param printRewrittenExprParam
- * @param printLogicalPlanParam
- * @param printOptimizedLogicalPlanParam
- * @param printPhysicalOpsOnly
- * @param executeQuery
- * @param generateJobSpec
- * @param printJob
+ * Produce out-of-band output for Hyracks Job.
*/
- public SessionConfig(boolean optimize, boolean printExprParam, boolean printRewrittenExprParam,
- boolean printLogicalPlanParam, boolean printOptimizedLogicalPlanParam, boolean printPhysicalOpsOnly,
- boolean executeQuery, boolean generateJobSpec, boolean printJob) {
+ public static final String OOB_HYRACKS_JOB = "oob-hyracks-job";
+
+ /**
+ * Produce out-of-band output for Expression Tree.
+ */
+ public static final String OOB_EXPR_TREE = "oob-expr-tree";
+
+ /**
+ * Produce out-of-band output for Rewritten Expression Tree.
+ */
+ public static final String OOB_REWRITTEN_EXPR_TREE = "oob-rewritten-expr-tree";
+
+ /**
+ * Produce out-of-band output for Logical Plan.
+ */
+ public static final String OOB_LOGICAL_PLAN = "oob-logical-plan";
+
+ /**
+ * Produce out-of-band output for Optimized Logical Plan.
+ */
+ public static final String OOB_OPTIMIZED_LOGICAL_PLAN = "oob-optimized-logical-plan";
+
+ /**
+ * Format flag: print only physical ops (for optimizer tests).
+ */
+ public static final String FORMAT_ONLY_PHYSICAL_OPS = "format-only-physical-ops";
+
+ /**
+ * Format flag: wrap out-of-band data in HTML.
+ */
+ public static final String FORMAT_HTML = "format-html";
+
+ /**
+ * Format flag: print CSV header line.
+ */
+ public static final String FORMAT_CSV_HEADER = "format-csv-header";
+
+ // Standard execution flags.
+ private final boolean executeQuery;
+ private final boolean generateJobSpec;
+ private final boolean optimize;
+
+ // Output path for primary execution.
+ private final PrintWriter out;
+
+ // Output format.
+ private final OutputFormat fmt;
+
+ // Flags.
+ private final Map<String,Boolean> flags;
+
+ /**
+ * Create a SessionConfig object with all default values:
+ *
+ * - All format flags set to "false".
+ * - All out-of-band outputs set to "null".
+ * - "Optimize" set to "true".
+ * - "Execute Query" set to "true".
+ * - "Generate Job Spec" set to "true".
+ * @param out PrintWriter for execution output.
+ * @param fmt Output format for execution output.
+ */
+ public SessionConfig(PrintWriter out, OutputFormat fmt) {
+ this(out, fmt, true, true, true);
+ }
+
+ /**
+ * Create a SessionConfig object with all optional values set to defaults:
+ *
+ * - All format flags set to "false".
+ * - All out-of-band outputs set to "false".
+ * @param out PrintWriter for execution output.
+ * @param fmt Output format for execution output.
+ * @param optimize Whether to optimize the execution.
+ * @param executeQuery Whether to execute the query or not.
+ * @param generateJobSpec Whether to generate the Hyracks job specification (if
+ * false, job cannot be executed).
+ */
+ public SessionConfig(PrintWriter out, OutputFormat fmt, boolean optimize, boolean executeQuery, boolean generateJobSpec) {
+ this.out = out;
+ this.fmt = fmt;
this.optimize = optimize;
- this.printExprParam = printExprParam;
- this.printRewrittenExprParam = printRewrittenExprParam;
- this.printLogicalPlanParam = printLogicalPlanParam;
- this.printOptimizedLogicalPlanParam = printOptimizedLogicalPlanParam;
- this.printPhysicalOpsOnly = printPhysicalOpsOnly;
this.executeQuery = executeQuery;
this.generateJobSpec = generateJobSpec;
- this.printJob = printJob;
+ this.flags = new HashMap<String,Boolean>();
}
- public boolean isPrintExprParam() {
- return printExprParam;
+ /**
+ * Retrieve the PrintWriter to produce output to.
+ */
+ public PrintWriter out() {
+ return this.out;
}
- public boolean isPrintRewrittenExprParam() {
- return printRewrittenExprParam;
+ /**
+ * Retrieve the OutputFormat for this execution.
+ */
+ public OutputFormat fmt() {
+ return this.fmt;
}
- public boolean isPrintLogicalPlanParam() {
- return printLogicalPlanParam;
- }
-
- public boolean isPrintOptimizedLogicalPlanParam() {
- return printOptimizedLogicalPlanParam;
- }
-
- public boolean isPrintJob() {
- return printJob;
- }
-
- public boolean isPrintPhysicalOpsOnly() {
- return printPhysicalOpsOnly;
- }
-
+ /**
+ * Retrieve the value of the "execute query" flag.
+ */
public boolean isExecuteQuery() {
return executeQuery;
}
+ /**
+ * Retrieve the value of the "optimize" flag.
+ */
public boolean isOptimize() {
return optimize;
}
+ /**
+ * Retrieve the value of the "generate job spec" flag.
+ */
public boolean isGenerateJobSpec() {
return generateJobSpec;
}
-}
\ No newline at end of file
+
+ /**
+ * Specify all out-of-band settings at once. For convenience of older code.
+ */
+ public void setOOBData(boolean expr_tree, boolean rewritten_expr_tree,
+ boolean logical_plan, boolean optimized_logical_plan,
+ boolean hyracks_job) {
+ this.set(OOB_EXPR_TREE, expr_tree);
+ this.set(OOB_REWRITTEN_EXPR_TREE, rewritten_expr_tree);
+ this.set(OOB_LOGICAL_PLAN, logical_plan);
+ this.set(OOB_OPTIMIZED_LOGICAL_PLAN, optimized_logical_plan);
+ this.set(OOB_HYRACKS_JOB, hyracks_job);
+ }
+
+ /**
+ * Specify a flag.
+ * @param flag One of the OOB_ or FORMAT_ constants from this class.
+ * @param value Value for the flag (all flags default to "false").
+ */
+ public void set(String flag, boolean value) {
+ flags.put(flag, Boolean.valueOf(value));
+ }
+
+ /**
+ * Retrieve the setting of a format-specific flag.
+ * @param flag One of the FORMAT_ constants from this class.
+ * @returns true or false (all flags default to "false").
+ */
+ public boolean is(String flag) {
+ Boolean value = flags.get(flag);
+ return value == null ? false : value.booleanValue();
+ }
+}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/APIServlet.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/APIServlet.java
index be88a1e..4984741 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/APIServlet.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/APIServlet.java
@@ -30,8 +30,8 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import edu.uci.ics.asterix.api.common.APIFramework.OutputFormat;
import edu.uci.ics.asterix.api.common.SessionConfig;
+import edu.uci.ics.asterix.api.common.SessionConfig.OutputFormat;
import edu.uci.ics.asterix.aql.base.Statement;
import edu.uci.ics.asterix.aql.parser.AQLParser;
import edu.uci.ics.asterix.aql.parser.ParseException;
@@ -54,12 +54,23 @@
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response) throws IOException {
- OutputFormat format = OutputFormat.HTML;
- if (request.getContentType().equals("application/json")) {
- format = OutputFormat.JSON;
- } else if (request.getContentType().equals("text/plain")) {
+ OutputFormat format;
+ boolean csv_and_header = false;
+ String output = request.getParameter("output-format");
+ if (output.equals("ADM")) {
format = OutputFormat.ADM;
}
+ else if (output.equals("CSV")) {
+ format = OutputFormat.CSV;
+ }
+ else if (output.equals("CSV-Header")) {
+ format = OutputFormat.CSV;
+ csv_and_header = true;
+ }
+ else {
+ // Default output format
+ format = OutputFormat.JSON;
+ }
String query = request.getParameter("query");
String printExprParam = request.getParameter("print-expr-tree");
@@ -87,11 +98,14 @@
}
AQLParser parser = new AQLParser(query);
List<Statement> aqlStatements = parser.parse();
- SessionConfig sessionConfig = new SessionConfig(true, isSet(printExprParam),
- isSet(printRewrittenExprParam), isSet(printLogicalPlanParam),
- isSet(printOptimizedLogicalPlanParam), false, isSet(executeQuery), true, isSet(printJob));
+ SessionConfig sessionConfig = new SessionConfig(out, format, true, isSet(executeQuery), true);
+ sessionConfig.set(SessionConfig.FORMAT_HTML, true);
+ sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, csv_and_header);
+ sessionConfig.setOOBData(isSet(printExprParam), isSet(printRewrittenExprParam),
+ isSet(printLogicalPlanParam), isSet(printOptimizedLogicalPlanParam),
+ isSet(printJob));
MetadataManager.INSTANCE.init();
- AqlTranslator aqlTranslator = new AqlTranslator(aqlStatements, out, sessionConfig, format);
+ AqlTranslator aqlTranslator = new AqlTranslator(aqlStatements, sessionConfig);
double duration = 0;
long startTime = System.currentTimeMillis();
aqlTranslator.compileAndExecute(hcc, hds, AqlTranslator.ResultDelivery.SYNC);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/QueryResultAPIServlet.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/QueryResultAPIServlet.java
index 5ccbfc8..3f104fe 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/QueryResultAPIServlet.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/QueryResultAPIServlet.java
@@ -26,6 +26,8 @@
import org.json.JSONObject;
import edu.uci.ics.asterix.api.common.APIFramework;
+import edu.uci.ics.asterix.api.common.SessionConfig;
+import edu.uci.ics.asterix.api.common.SessionConfig.OutputFormat;
import edu.uci.ics.asterix.result.ResultReader;
import edu.uci.ics.asterix.result.ResultUtils;
import edu.uci.ics.hyracks.api.client.HyracksConnection;
@@ -78,30 +80,14 @@
ResultReader resultReader = new ResultReader(hcc, hds);
resultReader.open(jobId, rsId);
- APIFramework.OutputFormat format;
- // QQQ This code is duplicated from RESTAPIServlet, and is
- // erroneous anyway. The output format is determined by
- // the initial query and cannot be modified here, so we need
- // to find a way to send the same OutputFormat value here as
- // was originally determined there. Need to save this value on
+ // QQQ The output format is determined by the initial
+ // query and cannot be modified here, so calling back to
+ // initResponse() is really an error. We need to find a
+ // way to send the same OutputFormat value here as was
+ // originally determined there. Need to save this value on
// some object that we can obtain here.
- String accept = request.getHeader("Accept");
- if ((accept == null) || (accept.contains("application/x-adm"))) {
- format = APIFramework.OutputFormat.ADM;
- response.setContentType("application/x-adm");
- } else if (accept.contains("text/html")) {
- format = APIFramework.OutputFormat.HTML;
- response.setContentType("text/html");
- } else if (accept.contains("text/csv")) {
- format = APIFramework.OutputFormat.CSV;
- response.setContentType("text/csv; header=present");
- } else {
- // JSON output is the default; most generally useful for a
- // programmatic HTTP API
- format = APIFramework.OutputFormat.JSON;
- response.setContentType("application/json");
- }
- ResultUtils.displayResults(resultReader, out, format);
+ SessionConfig sessionConfig = RESTAPIServlet.initResponse(request, response);
+ ResultUtils.displayResults(resultReader, sessionConfig);
} catch (Exception e) {
out.println(e.getMessage());
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/RESTAPIServlet.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/RESTAPIServlet.java
index e783741..4e8427d 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/RESTAPIServlet.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/RESTAPIServlet.java
@@ -31,8 +31,8 @@
import org.json.JSONObject;
import edu.uci.ics.asterix.api.common.APIFramework;
-import edu.uci.ics.asterix.api.common.APIFramework.OutputFormat;
import edu.uci.ics.asterix.api.common.SessionConfig;
+import edu.uci.ics.asterix.api.common.SessionConfig.OutputFormat;
import edu.uci.ics.asterix.aql.base.Statement;
import edu.uci.ics.asterix.aql.base.Statement.Kind;
import edu.uci.ics.asterix.aql.parser.AQLParser;
@@ -55,6 +55,67 @@
private static final String HYRACKS_DATASET_ATTR = "edu.uci.ics.asterix.HYRACKS_DATASET";
+ /**
+ * Initialize the Content-Type of the response, and construct a
+ * SessionConfig with the appropriate output writer and output-format
+ * based on the Accept: header and other servlet parameters.
+ */
+ static SessionConfig initResponse(HttpServletRequest request, HttpServletResponse response)
+ throws IOException {
+ response.setCharacterEncoding("utf-8");
+
+ // JSON output is the default; most generally useful for a
+ // programmatic HTTP API
+ OutputFormat format = OutputFormat.JSON;
+
+ // First check the "output" servlet parameter.
+ String output = request.getParameter("output");
+ String accept = request.getHeader("Accept");
+ if (output != null) {
+ if (output.equals("CSV")) {
+ format = OutputFormat.CSV;
+ }
+ else if (output.equals("ADM")) {
+ format = OutputFormat.ADM;
+ }
+ }
+ else {
+ // Second check the Accept: HTTP header.
+ if (accept != null) {
+ if (accept.contains("application/x-adm")) {
+ format = OutputFormat.ADM;
+ } else if (accept.contains("text/csv")) {
+ format = OutputFormat.CSV;
+ }
+ }
+ }
+
+ SessionConfig sessionConfig = new SessionConfig(response.getWriter(), format);
+
+ // Now that format is set, output the content-type
+ switch (format) {
+ case ADM:
+ response.setContentType("application/x-adm");
+ break;
+ case JSON:
+ response.setContentType("application/json");
+ break;
+ case CSV: {
+ // Check for header parameter or in Accept:.
+ if ("present".equals(request.getParameter("header")) ||
+ (accept != null && accept.contains("header=present"))) {
+ response.setContentType("text/csv; header=present");
+ sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, true);
+ }
+ else {
+ response.setContentType("text/csv; header=absent");
+ }
+ }
+ };
+
+ return sessionConfig;
+ }
+
@Override
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException,
IOException {
@@ -72,28 +133,7 @@
public void handleRequest(HttpServletRequest request, HttpServletResponse response, String query)
throws IOException {
- response.setCharacterEncoding("utf-8");
-
- PrintWriter out = response.getWriter();
- APIFramework.OutputFormat format;
- // QQQ For now switch solely based on Accept header. Later will add
- // an "output" query parameter.
- String accept = request.getHeader("Accept");
- if ((accept == null) || (accept.contains("application/x-adm"))) {
- format = OutputFormat.ADM;
- response.setContentType("application/x-adm");
- } else if (accept.contains("text/html")) {
- format = OutputFormat.HTML;
- response.setContentType("text/html");
- } else if (accept.contains("text/csv")) {
- format = OutputFormat.CSV;
- response.setContentType("text/csv; header=present");
- } else {
- // JSON output is the default; most generally useful for a
- // programmatic HTTP API
- format = APIFramework.OutputFormat.JSON;
- response.setContentType("application/json");
- }
+ SessionConfig sessionConfig = initResponse(request, response);
AqlTranslator.ResultDelivery resultDelivery = whichResultDelivery(request);
ServletContext context = getServletContext();
@@ -113,21 +153,19 @@
AQLParser parser = new AQLParser(query);
List<Statement> aqlStatements = parser.parse();
if (!containsForbiddenStatements(aqlStatements)) {
- SessionConfig sessionConfig = new SessionConfig(true, false, false, false, false, false, true, true,
- false);
MetadataManager.INSTANCE.init();
- AqlTranslator aqlTranslator = new AqlTranslator(aqlStatements, out, sessionConfig, format);
+ AqlTranslator aqlTranslator = new AqlTranslator(aqlStatements, sessionConfig);
aqlTranslator.compileAndExecute(hcc, hds, resultDelivery);
}
} catch (ParseException | TokenMgrError | edu.uci.ics.asterix.aqlplus.parser.TokenMgrError pe) {
GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, pe.getMessage(), pe);
String errorMessage = ResultUtils.buildParseExceptionMessage(pe, query);
JSONObject errorResp = ResultUtils.getErrorResponse(2, errorMessage, "", "");
- out.write(errorResp.toString());
+ sessionConfig.out().write(errorResp.toString());
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
} catch (Exception e) {
GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, e.getMessage(), e);
- ResultUtils.apiErrorHandler(out, e);
+ ResultUtils.apiErrorHandler(sessionConfig.out(), e);
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
}
}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/java/AsterixJavaClient.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/java/AsterixJavaClient.java
index e0ba829..54804e7 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/java/AsterixJavaClient.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/java/AsterixJavaClient.java
@@ -19,9 +19,9 @@
import java.util.List;
import edu.uci.ics.asterix.api.common.APIFramework;
-import edu.uci.ics.asterix.api.common.APIFramework.OutputFormat;
import edu.uci.ics.asterix.api.common.Job;
import edu.uci.ics.asterix.api.common.SessionConfig;
+import edu.uci.ics.asterix.api.common.SessionConfig.OutputFormat;
import edu.uci.ics.asterix.aql.base.Statement;
import edu.uci.ics.asterix.aql.parser.AQLParser;
import edu.uci.ics.asterix.aql.parser.ParseException;
@@ -76,21 +76,25 @@
}
MetadataManager.INSTANCE.init();
- SessionConfig pc = new SessionConfig(optimize, false, printRewrittenExpressions, printLogicalPlan,
- printOptimizedPlan, printPhysicalOpsOnly, true, generateBinaryRuntime, printJob);
+ SessionConfig conf = new SessionConfig(writer, OutputFormat.ADM, optimize, true, generateBinaryRuntime);
+ conf.setOOBData(false, printRewrittenExpressions, printLogicalPlan,
+ printOptimizedPlan, printJob);
+ if (printPhysicalOpsOnly) {
+ conf.set(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS, true);
+ }
- AqlTranslator aqlTranslator = new AqlTranslator(aqlStatements, writer, pc, OutputFormat.ADM);
+ AqlTranslator aqlTranslator = new AqlTranslator(aqlStatements, conf);
aqlTranslator.compileAndExecute(hcc, null, AqlTranslator.ResultDelivery.SYNC);
writer.flush();
}
public void execute() throws Exception {
if (dmlJobs != null) {
- APIFramework.executeJobArray(hcc, dmlJobs, writer, OutputFormat.ADM);
+ APIFramework.executeJobArray(hcc, dmlJobs, writer);
}
if (queryJobSpec != null) {
- APIFramework.executeJobArray(hcc, new JobSpecification[] { queryJobSpec }, writer, OutputFormat.ADM);
+ APIFramework.executeJobArray(hcc, new JobSpecification[] { queryJobSpec }, writer);
}
}
-}
\ No newline at end of file
+}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
index 52fc25e..0184a65 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
@@ -37,9 +37,9 @@
import org.json.JSONObject;
import edu.uci.ics.asterix.api.common.APIFramework;
-import edu.uci.ics.asterix.api.common.APIFramework.OutputFormat;
import edu.uci.ics.asterix.api.common.Job;
import edu.uci.ics.asterix.api.common.SessionConfig;
+import edu.uci.ics.asterix.api.common.SessionConfig.OutputFormat;
import edu.uci.ics.asterix.aql.base.Statement;
import edu.uci.ics.asterix.aql.expression.CompactStatement;
import edu.uci.ics.asterix.aql.expression.ConnectFeedStatement;
@@ -188,18 +188,14 @@
public static final boolean IS_DEBUG_MODE = false;//true
private final List<Statement> aqlStatements;
- private final PrintWriter out;
private final SessionConfig sessionConfig;
- private final OutputFormat pdf;
private Dataverse activeDefaultDataverse;
private final List<FunctionDecl> declaredFunctions;
- public AqlTranslator(List<Statement> aqlStatements, PrintWriter out, SessionConfig pc, APIFramework.OutputFormat pdf)
+ public AqlTranslator(List<Statement> aqlStatements, SessionConfig conf)
throws MetadataException, AsterixException {
this.aqlStatements = aqlStatements;
- this.out = out;
- this.sessionConfig = pc;
- this.pdf = pdf;
+ this.sessionConfig = conf;
declaredFunctions = getDeclaredFunctions(aqlStatements);
}
@@ -1740,8 +1736,7 @@
CompiledLoadFromFileStatement cls = new CompiledLoadFromFileStatement(dataverseName, loadStmt
.getDatasetName().getValue(), loadStmt.getAdapter(), loadStmt.getProperties(),
loadStmt.dataIsAlreadySorted());
- JobSpecification spec = APIFramework.compileQuery(null, metadataProvider, null, 0, null, sessionConfig,
- out, pdf, cls);
+ JobSpecification spec = APIFramework.compileQuery(null, metadataProvider, null, 0, null, sessionConfig, cls);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
if (spec != null) {
@@ -1837,12 +1832,12 @@
// Query Rewriting (happens under the same ongoing metadata transaction)
Pair<Query, Integer> reWrittenQuery = APIFramework.reWriteQuery(declaredFunctions, metadataProvider, query,
- sessionConfig, out, pdf);
+ sessionConfig);
// Query Compilation (happens under the same ongoing metadata
// transaction)
JobSpecification spec = APIFramework.compileQuery(declaredFunctions, metadataProvider, reWrittenQuery.first,
- reWrittenQuery.second, stmt == null ? null : stmt.getDatasetName(), sessionConfig, out, pdf, stmt);
+ reWrittenQuery.second, stmt == null ? null : stmt.getDatasetName(), sessionConfig, stmt);
return spec;
@@ -2187,8 +2182,8 @@
handle.put(jobId.getId());
handle.put(metadataProvider.getResultSetId().getId());
response.put("handle", handle);
- out.print(response);
- out.flush();
+ sessionConfig.out().print(response);
+ sessionConfig.out().flush();
hcc.waitForCompletion(jobId);
break;
case SYNC:
@@ -2198,10 +2193,11 @@
// In this case (the normal case), we don't use the
// "response" JSONObject - just stream the results
// to the "out" PrintWriter
- if (pdf == OutputFormat.CSV) {
- ResultUtils.displayCSVHeader(metadataProvider.findOutputRecordType(), out);
+ if (sessionConfig.fmt() == OutputFormat.CSV &&
+ sessionConfig.is(SessionConfig.FORMAT_CSV_HEADER)) {
+ ResultUtils.displayCSVHeader(metadataProvider.findOutputRecordType(), sessionConfig);
}
- ResultUtils.displayResults(resultReader, out, pdf);
+ ResultUtils.displayResults(resultReader, sessionConfig);
hcc.waitForCompletion(jobId);
break;
@@ -2211,8 +2207,8 @@
handle.put(metadataProvider.getResultSetId().getId());
response.put("handle", handle);
hcc.waitForCompletion(jobId);
- out.print(response);
- out.flush();
+ sessionConfig.out().print(response);
+ sessionConfig.out().flush();
break;
default:
break;
@@ -2715,7 +2711,7 @@
private JobId runJob(IHyracksClientConnection hcc, JobSpecification spec, boolean waitForCompletion)
throws Exception {
- JobId[] jobIds = executeJobArray(hcc, new Job[] { new Job(spec) }, out, waitForCompletion);
+ JobId[] jobIds = executeJobArray(hcc, new Job[] { new Job(spec) }, sessionConfig.out(), waitForCompletion);
return jobIds[0];
}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/FeedLifecycleListener.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/FeedLifecycleListener.java
index 9c7ff81..4cdc91c 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/FeedLifecycleListener.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/FeedLifecycleListener.java
@@ -1104,7 +1104,7 @@
public void reviveFeed(String dataverse, String feedName, String dataset, String feedPolicy) {
PrintWriter writer = new PrintWriter(System.out, true);
- SessionConfig pc = new SessionConfig(true, false, false, false, false, false, true, true, false);
+ SessionConfig conf = new SessionConfig(writer, SessionConfig.OutputFormat.ADM);
try {
DataverseDecl dataverseDecl = new DataverseDecl(new Identifier(dataverse));
ConnectFeedStatement stmt = new ConnectFeedStatement(new Identifier(dataverse),
@@ -1113,7 +1113,7 @@
List<Statement> statements = new ArrayList<Statement>();
statements.add(dataverseDecl);
statements.add(stmt);
- AqlTranslator translator = new AqlTranslator(statements, writer, pc, APIFramework.OutputFormat.ADM);
+ AqlTranslator translator = new AqlTranslator(statements, conf);
translator.compileAndExecute(AsterixAppContextInfo.getInstance().getHcc(), null,
AqlTranslator.ResultDelivery.SYNC);
if (LOGGER.isLoggable(Level.INFO)) {
@@ -1147,7 +1147,7 @@
private void endFeed(FeedInfo feedInfo) {
MetadataTransactionContext ctx = null;
PrintWriter writer = new PrintWriter(System.out, true);
- SessionConfig pc = new SessionConfig(true, false, false, false, false, false, true, true, false);
+ SessionConfig conf = new SessionConfig(writer, SessionConfig.OutputFormat.ADM);
try {
ctx = MetadataManager.INSTANCE.beginTransaction();
DisconnectFeedStatement stmt = new DisconnectFeedStatement(new Identifier(
@@ -1159,7 +1159,7 @@
new Identifier(feedInfo.feedConnectionId.getDataverse()));
statements.add(dataverseDecl);
statements.add(stmt);
- AqlTranslator translator = new AqlTranslator(statements, writer, pc, APIFramework.OutputFormat.ADM);
+ AqlTranslator translator = new AqlTranslator(statements, conf);
translator.compileAndExecute(AsterixAppContextInfo.getInstance().getHcc(), null,
AqlTranslator.ResultDelivery.SYNC);
if (LOGGER.isLoggable(Level.INFO)) {
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/result/ResultUtils.java b/asterix-app/src/main/java/edu/uci/ics/asterix/result/ResultUtils.java
index f558eb6..326697f 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/result/ResultUtils.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/result/ResultUtils.java
@@ -32,7 +32,8 @@
import org.json.JSONException;
import org.json.JSONObject;
-import edu.uci.ics.asterix.api.common.APIFramework;
+import edu.uci.ics.asterix.api.common.SessionConfig;
+import edu.uci.ics.asterix.api.common.SessionConfig.OutputFormat;
import edu.uci.ics.asterix.api.http.servlet.APIServlet;
import edu.uci.ics.asterix.om.types.ARecordType;
import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -61,22 +62,29 @@
return s;
}
- public static void displayCSVHeader(ARecordType recordType, PrintWriter out) {
+ public static void displayCSVHeader(ARecordType recordType, SessionConfig conf) {
+ // If HTML-ifying, we have to output this here before the header -
+ // pretty ugly
+ if (conf.is(SessionConfig.FORMAT_HTML)) {
+ conf.out().println("<h4>Results:</h4>");
+ conf.out().println("<pre>");
+ }
+
String[] fieldNames = recordType.getFieldNames();
boolean notfirst = false;
for (String name : fieldNames) {
if (notfirst) {
- out.print(',');
+ conf.out().print(',');
}
notfirst = true;
- out.print('"');
- out.print(name.replace("\"", "\"\""));
- out.print('"');
+ conf.out().print('"');
+ conf.out().print(name.replace("\"", "\"\""));
+ conf.out().print('"');
}
- out.print("\r\n");
+ conf.out().print("\r\n");
}
- public static void displayResults(ResultReader resultReader, PrintWriter out, APIFramework.OutputFormat pdf)
+ public static void displayResults(ResultReader resultReader, SessionConfig conf)
throws HyracksDataException {
IFrameTupleAccessor fta = resultReader.getFrameTupleAccessor();
@@ -90,11 +98,15 @@
// Whether this is the first instance being output
boolean notfirst = false;
- switch (pdf) {
- case HTML:
- out.println("<h4>Results:</h4>");
- out.println("<pre>");
- // Fall through
+ // If we're outputting CSV with a header, the HTML header was already
+ // output by displayCSVHeader(), so skip it here
+ if (conf.is(SessionConfig.FORMAT_HTML) &&
+ ! (conf.fmt() == OutputFormat.CSV && conf.is(SessionConfig.FORMAT_CSV_HEADER))) {
+ conf.out().println("<h4>Results:</h4>");
+ conf.out().println("<pre>");
+ }
+
+ switch (conf.fmt()) {
case CSV:
need_commas = false;
break;
@@ -103,7 +115,7 @@
// Conveniently, JSON and ADM have the same syntax for an
// "ordered list", and our representation of the result of a
// statement is an ordered list of instances.
- out.print("[ ");
+ conf.out().print("[ ");
break;
}
@@ -119,19 +131,19 @@
bbis.setByteBuffer(buffer, start);
byte[] recordBytes = new byte[length];
int numread = bbis.read(recordBytes, 0, length);
- if (pdf == APIFramework.OutputFormat.CSV) {
+ if (conf.fmt() == OutputFormat.CSV) {
if ( (numread > 0) && (recordBytes[numread-1] == '\n') ) {
numread--;
}
}
result = new String(recordBytes, 0, numread, UTF_8);
if (need_commas && notfirst) {
- out.print(", ");
+ conf.out().print(", ");
}
notfirst = true;
- out.print(result);
- if (pdf == APIFramework.OutputFormat.CSV) {
- out.print("\r\n");
+ conf.out().print(result);
+ if (conf.fmt() == OutputFormat.CSV) {
+ conf.out().print("\r\n");
}
}
buffer.clear();
@@ -145,21 +157,21 @@
} while (resultReader.read(buffer) > 0);
}
- out.flush();
+ conf.out().flush();
- switch (pdf) {
- case HTML:
- out.println("</pre>");
- break;
+ switch (conf.fmt()) {
case JSON:
case ADM:
- out.println(" ]");
+ conf.out().println(" ]");
break;
case CSV:
// Nothing to do
break;
}
+ if (conf.is(SessionConfig.FORMAT_HTML)) {
+ conf.out().println("</pre>");
+ }
}
public static JSONObject getErrorResponse(int errorCode, String errorMessage, String errorSummary,
diff --git a/asterix-app/src/main/resources/webui/querytemplate.html b/asterix-app/src/main/resources/webui/querytemplate.html
index 0adb518..bcd177c 100644
--- a/asterix-app/src/main/resources/webui/querytemplate.html
+++ b/asterix-app/src/main/resources/webui/querytemplate.html
@@ -186,15 +186,23 @@
<textarea rows="10" id="qry" name="query" class="query" value="%s" placeholder="Type your AQL query ..."></textarea>
</div>
- <div class="btn-group">
- <button id="checkboxes-on" class="btn">
- <i id="opts" class="icon-ok" style="display:none;"></i>Select Options</button>
- <button id="clear-query-button" class="btn">Clear Query</button>
- <!-- <button id="checkboxes-off" class="btn">Clear All Options</button> -->
- <button type="submit" id="run-btn" class="btn btn-custom-darken">Run</button>
- </div>
+ <div class="btn-group">
+ <button id="checkboxes-on" class="btn">
+ <i id="opts" class="icon-ok" style="display:none;"></i>Select Options</button>
+ <button id="clear-query-button" class="btn">Clear Query</button>
+ <!-- <button id="checkboxes-off" class="btn">Clear All Options</button> -->
+ <button type="submit" id="run-btn" class="btn btn-custom-darken">Run</button>
+ </div>
<div>
+ <label class="checkbox optlabel"> Output Format:<br/>
+ <select name="output-format" class="btn">
+ <option selected value="JSON">JSON</option>
+ <option value="ADM">ADM</option>
+ <option value="CSV">CSV (no header)</option>
+ <option value="CSV-Header">CSV (with header)</option>
+ </select>
+ </label>
<label class="checkbox optlabel"><input type="checkbox" name="print-expr-tree" value="true" /> Print parsed expressions</label>
<label class="checkbox optlabel"><input type="checkbox" name="print-rewritten-expr-tree" value="true" /> Print rewritten expressions</label>
<label class="checkbox optlabel"><input type="checkbox" name="print-logical-plan" value="true" /> Print logical plan</label>