[NO ISSUE] Handle Accept-Charset in QueryResultApiServlet
- exercise non-UTF8 Accept-Charset in TestExecutor
- remove double-buffering on http responses
- minor refactoring / cleanup
Change-Id: I8f37eb684bf2457e5ff451bf5c8fbca742d531f2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3191
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Michael Blow <mblow@apache.org>
diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml
index 8ef9c45..281e1d4 100644
--- a/asterixdb/asterix-app/pom.xml
+++ b/asterixdb/asterix-app/pom.xml
@@ -666,7 +666,6 @@
<dependency>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-am-lsm-invertedindex</artifactId>
- <version>${hyracks.version}</version>
</dependency>
</dependencies>
</project>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
index 8fdcc41..7f74c92 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
@@ -18,12 +18,14 @@
*/
package org.apache.asterix.api.http.server;
+import java.io.IOException;
import java.util.concurrent.ConcurrentMap;
import org.apache.asterix.app.result.ResultHandle;
import org.apache.asterix.app.result.ResultReader;
import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.translator.IStatementExecutor.Stats;
+import org.apache.asterix.translator.SessionConfig;
import org.apache.asterix.translator.SessionOutput;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -91,7 +93,7 @@
// way to send the same OutputFormat value here as was
// originally determined there. Need to save this value on
// some object that we can obtain here.
- SessionOutput sessionOutput = RestApiServlet.initResponse(request, response);
+ SessionOutput sessionOutput = initResponse(request, response);
ResultUtil.printResults(appCtx, resultReader, sessionOutput, new Stats(), null);
} catch (HyracksDataException e) {
final int errorCode = e.getErrorCode();
@@ -112,4 +114,82 @@
}
}
+ /**
+ * Initialize the Content-Type of the response, and construct a
+ * SessionConfig with the appropriate output writer and output-format
+ * based on the Accept: header and other servlet parameters.
+ */
+ static SessionOutput initResponse(IServletRequest request, IServletResponse response) throws IOException {
+ // CLEAN_JSON output is the default; most generally useful for a
+ // programmatic HTTP API
+ SessionConfig.OutputFormat format = SessionConfig.OutputFormat.CLEAN_JSON;
+ // First check the "output" servlet parameter.
+ String output = request.getParameter("output");
+ String accept = request.getHeader("Accept", "");
+ if (output != null) {
+ if ("CSV".equals(output)) {
+ format = SessionConfig.OutputFormat.CSV;
+ } else if ("ADM".equals(output)) {
+ format = SessionConfig.OutputFormat.ADM;
+ }
+ } else {
+ // Second check the Accept: HTTP header.
+ if (accept.contains("application/x-adm")) {
+ format = SessionConfig.OutputFormat.ADM;
+ } else if (accept.contains("text/csv")) {
+ format = SessionConfig.OutputFormat.CSV;
+ }
+ }
+ SessionConfig.PlanFormat planFormat = SessionConfig.PlanFormat.get(request.getParameter("plan-format"),
+ "plan format", SessionConfig.PlanFormat.STRING, LOGGER);
+
+ // If it's JSON, check for the "lossless" flag
+
+ if (format == SessionConfig.OutputFormat.CLEAN_JSON
+ && ("true".equals(request.getParameter("lossless")) || accept.contains("lossless=true"))) {
+ format = SessionConfig.OutputFormat.LOSSLESS_JSON;
+ }
+
+ SessionOutput.ResultAppender appendHandle = (app, handle) -> app.append("{ \"").append("handle")
+ .append("\":" + " \"").append(handle).append("\" }");
+ SessionConfig sessionConfig = new SessionConfig(format, planFormat);
+
+ // If it's JSON or ADM, check for the "wrapper-array" flag. Default is
+ // "true" for JSON and "false" for ADM. (Not applicable for CSV.)
+ boolean wrapperArray =
+ format == SessionConfig.OutputFormat.CLEAN_JSON || format == SessionConfig.OutputFormat.LOSSLESS_JSON;
+ String wrapperParam = request.getParameter("wrapper-array");
+ if (wrapperParam != null) {
+ wrapperArray = Boolean.valueOf(wrapperParam);
+ } else if (accept.contains("wrap-array=true")) {
+ wrapperArray = true;
+ } else if (accept.contains("wrap-array=false")) {
+ wrapperArray = false;
+ }
+ sessionConfig.set(SessionConfig.FORMAT_WRAPPER_ARRAY, wrapperArray);
+ // Now that format is set, output the content-type
+ switch (format) {
+ case ADM:
+ HttpUtil.setContentType(response, "application/x-adm", request);
+ break;
+ case CLEAN_JSON:
+ // No need to reflect "clean-ness" in output type; fall through
+ case LOSSLESS_JSON:
+ HttpUtil.setContentType(response, "application/json", request);
+ break;
+ case CSV:
+ // Check for header parameter or in Accept:.
+ if ("present".equals(request.getParameter("header")) || accept.contains("header=present")) {
+ HttpUtil.setContentType(response, "text/csv; header=present", request);
+ sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, true);
+ } else {
+ HttpUtil.setContentType(response, "text/csv; header=absent", request);
+ }
+ break;
+ default:
+ throw new IOException("Unknown format " + format);
+ }
+ return new SessionOutput(sessionConfig, response.writer(), null, null, appendHandle, null);
+ }
+
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index 625834f..79401b8 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -28,6 +28,8 @@
import java.io.PrintWriter;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
@@ -77,7 +79,6 @@
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
-
import io.netty.handler.codec.http.HttpResponseStatus;
public class QueryServiceServlet extends AbstractQueryApiServlet {
@@ -339,16 +340,18 @@
}
private static void printMetrics(PrintWriter pw, long elapsedTime, long executionTime, long resultCount,
- long resultSize, long processedObjects, long errorCount, long warnCount) {
+ long resultSize, long processedObjects, long errorCount, long warnCount, Charset resultCharset) {
boolean hasErrors = errorCount != 0;
boolean hasWarnings = warnCount != 0;
+ boolean useAscii = !StandardCharsets.UTF_8.equals(resultCharset)
+ && !"μ".contentEquals(resultCharset.decode(resultCharset.encode("μ")));
pw.print("\t\"");
pw.print(ResultFields.METRICS.str());
pw.print("\": {\n");
pw.print("\t");
- ResultUtil.printField(pw, Metrics.ELAPSED_TIME.str(), Duration.formatNanos(elapsedTime));
+ ResultUtil.printField(pw, Metrics.ELAPSED_TIME.str(), Duration.formatNanos(elapsedTime, useAscii));
pw.print("\t");
- ResultUtil.printField(pw, Metrics.EXECUTION_TIME.str(), Duration.formatNanos(executionTime));
+ ResultUtil.printField(pw, Metrics.EXECUTION_TIME.str(), Duration.formatNanos(executionTime, useAscii));
pw.print("\t");
ResultUtil.printField(pw, Metrics.RESULT_COUNT.str(), resultCount, true);
pw.print("\t");
@@ -366,16 +369,33 @@
pw.print("\t}\n");
}
+ private String getOptText(JsonNode node, Parameter parameter) {
+ return getOptText(node, parameter.str());
+ }
+
private String getOptText(JsonNode node, String fieldName) {
final JsonNode value = node.get(fieldName);
return value != null ? value.asText() : null;
}
+ private boolean getOptBoolean(JsonNode node, Parameter parameter, boolean defaultValue) {
+ return getOptBoolean(node, parameter.str(), defaultValue);
+ }
+
private boolean getOptBoolean(JsonNode node, String fieldName, boolean defaultValue) {
final JsonNode value = node.get(fieldName);
return value != null ? value.asBoolean() : defaultValue;
}
+ private String getParameter(IServletRequest request, Parameter parameter) {
+ return request.getParameter(parameter.str());
+ }
+
+ private boolean getOptBoolean(IServletRequest request, Parameter parameter, boolean defaultValue) {
+ String value = request.getParameter(parameter.str());
+ return value == null ? defaultValue : Boolean.parseBoolean(value);
+ }
+
@FunctionalInterface
interface CheckedFunction<I, O> {
O apply(I requestParamValue) throws IOException;
@@ -419,46 +439,41 @@
if (HttpUtil.ContentType.APPLICATION_JSON.equals(contentType)) {
try {
JsonNode jsonRequest = OBJECT_MAPPER.readTree(HttpUtil.getRequestBody(request));
- final String statementParam = Parameter.STATEMENT.str();
- if (jsonRequest.has(statementParam)) {
- param.setStatement(jsonRequest.get(statementParam).asText());
- }
- param.setFormat(toLower(getOptText(jsonRequest, Parameter.FORMAT.str())));
- param.setPretty(getOptBoolean(jsonRequest, Parameter.PRETTY.str(), false));
- param.setMode(toLower(getOptText(jsonRequest, Parameter.MODE.str())));
- param.setClientContextID(getOptText(jsonRequest, Parameter.CLIENT_ID.str()));
- param.setTimeout(getOptText(jsonRequest, Parameter.TIMEOUT.str()));
- param.setMaxResultReads(getOptText(jsonRequest, Parameter.MAX_RESULT_READS.str()));
- param.setPlanFormat(getOptText(jsonRequest, Parameter.PLAN_FORMAT.str()));
- param.setExpressionTree(getOptBoolean(jsonRequest, Parameter.EXPRESSION_TREE.str(), false));
+ param.setStatement(getOptText(jsonRequest, Parameter.STATEMENT));
+ param.setFormat(toLower(getOptText(jsonRequest, Parameter.FORMAT)));
+ param.setPretty(getOptBoolean(jsonRequest, Parameter.PRETTY, false));
+ param.setMode(toLower(getOptText(jsonRequest, Parameter.MODE)));
+ param.setClientContextID(getOptText(jsonRequest, Parameter.CLIENT_ID));
+ param.setTimeout(getOptText(jsonRequest, Parameter.TIMEOUT));
+ param.setMaxResultReads(getOptText(jsonRequest, Parameter.MAX_RESULT_READS));
+ param.setPlanFormat(getOptText(jsonRequest, Parameter.PLAN_FORMAT));
+ param.setExpressionTree(getOptBoolean(jsonRequest, Parameter.EXPRESSION_TREE, false));
param.setRewrittenExpressionTree(
- getOptBoolean(jsonRequest, Parameter.REWRITTEN_EXPRESSION_TREE.str(), false));
- param.setLogicalPlan(getOptBoolean(jsonRequest, Parameter.LOGICAL_PLAN.str(), false));
- param.setOptimizedLogicalPlan(
- getOptBoolean(jsonRequest, Parameter.OPTIMIZED_LOGICAL_PLAN.str(), false));
- param.setJob(getOptBoolean(jsonRequest, Parameter.JOB.str(), false));
- param.setSignature(getOptBoolean(jsonRequest, Parameter.SIGNATURE.str(), true));
+ getOptBoolean(jsonRequest, Parameter.REWRITTEN_EXPRESSION_TREE, false));
+ param.setLogicalPlan(getOptBoolean(jsonRequest, Parameter.LOGICAL_PLAN, false));
+ param.setOptimizedLogicalPlan(getOptBoolean(jsonRequest, Parameter.OPTIMIZED_LOGICAL_PLAN, false));
+ param.setJob(getOptBoolean(jsonRequest, Parameter.JOB, false));
+ param.setSignature(getOptBoolean(jsonRequest, Parameter.SIGNATURE, true));
param.setStatementParams(
getOptStatementParameters(jsonRequest, jsonRequest.fieldNames(), JsonNode::get, v -> v));
- param.setMultiStatement(getOptBoolean(jsonRequest, Parameter.MULTI_STATEMENT.str(), true));
+ param.setMultiStatement(getOptBoolean(jsonRequest, Parameter.MULTI_STATEMENT, true));
} catch (JsonParseException | JsonMappingException e) {
// if the JSON parsing fails, the statement is empty and we get an empty statement error
GlobalConfig.ASTERIX_LOGGER.log(Level.ERROR, e.getMessage(), e);
}
} else {
- param.setStatement(request.getParameter(Parameter.STATEMENT.str()));
+ param.setStatement(getParameter(request, Parameter.STATEMENT));
if (param.getStatement() == null) {
param.setStatement(HttpUtil.getRequestBody(request));
}
- param.setFormat(toLower(request.getParameter(Parameter.FORMAT.str())));
- param.setPretty(Boolean.parseBoolean(request.getParameter(Parameter.PRETTY.str())));
- param.setMode(toLower(request.getParameter(Parameter.MODE.str())));
- param.setClientContextID(request.getParameter(Parameter.CLIENT_ID.str()));
- param.setTimeout(request.getParameter(Parameter.TIMEOUT.str()));
- param.setMaxResultReads(request.getParameter(Parameter.MAX_RESULT_READS.str()));
- param.setPlanFormat(request.getParameter(Parameter.PLAN_FORMAT.str()));
- final String multiStatementParam = request.getParameter(Parameter.MULTI_STATEMENT.str());
- param.setMultiStatement(multiStatementParam == null || Boolean.parseBoolean(multiStatementParam));
+ param.setFormat(toLower(getParameter(request, Parameter.FORMAT)));
+ param.setPretty(Boolean.parseBoolean(getParameter(request, Parameter.PRETTY)));
+ param.setMode(toLower(getParameter(request, Parameter.MODE)));
+ param.setClientContextID(getParameter(request, Parameter.CLIENT_ID));
+ param.setTimeout(getParameter(request, Parameter.TIMEOUT));
+ param.setMaxResultReads(getParameter(request, Parameter.MAX_RESULT_READS));
+ param.setPlanFormat(getParameter(request, Parameter.PLAN_FORMAT));
+ param.setMultiStatement(getOptBoolean(request, Parameter.MULTI_STATEMENT, true));
try {
param.setStatementParams(getOptStatementParameters(request, request.getParameterNames().iterator(),
IServletRequest::getParameter, OBJECT_MAPPER::readTree));
@@ -512,7 +527,7 @@
QueryServiceRequestParameters param = getRequestParameters(request);
LOGGER.info("handleRequest: {}", param);
long elapsedStart = System.nanoTime();
- HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
+ Charset resultCharset = HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
final PrintWriter httpWriter = response.writer();
ResultDelivery delivery = parseResultDelivery(param.getMode());
@@ -573,7 +588,7 @@
execution.finish();
}
printMetrics(sessionOutput.out(), System.nanoTime() - elapsedStart, execution.duration(), stats.getCount(),
- stats.getSize(), stats.getProcessedObjects(), errorCount, warnings.size());
+ stats.getSize(), stats.getProcessedObjects(), errorCount, warnings.size(), resultCharset);
sessionOutput.out().print("}\n");
sessionOutput.out().flush();
if (sessionOutput.out().checkError()) {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java
deleted file mode 100644
index 347b2d7..0000000
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.api.http.server;
-
-import static org.apache.asterix.api.http.server.ServletConstants.HYRACKS_CONNECTION_ATTR;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.ConcurrentMap;
-
-import org.apache.asterix.app.translator.QueryTranslator;
-import org.apache.asterix.app.translator.RequestParameters;
-import org.apache.asterix.common.config.GlobalConfig;
-import org.apache.asterix.common.context.IStorageComponentProvider;
-import org.apache.asterix.common.dataflow.ICcApplicationContext;
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.compiler.provider.ILangCompilationProvider;
-import org.apache.asterix.lang.aql.parser.TokenMgrError;
-import org.apache.asterix.lang.common.base.IParser;
-import org.apache.asterix.lang.common.base.IParserFactory;
-import org.apache.asterix.lang.common.base.Statement;
-import org.apache.asterix.metadata.MetadataManager;
-import org.apache.asterix.translator.IRequestParameters;
-import org.apache.asterix.translator.IStatementExecutor;
-import org.apache.asterix.translator.IStatementExecutor.ResultDelivery;
-import org.apache.asterix.translator.IStatementExecutorFactory;
-import org.apache.asterix.translator.ResultProperties;
-import org.apache.asterix.translator.SessionConfig;
-import org.apache.asterix.translator.SessionConfig.OutputFormat;
-import org.apache.asterix.translator.SessionConfig.PlanFormat;
-import org.apache.asterix.translator.SessionOutput;
-import org.apache.hyracks.api.client.IHyracksClientConnection;
-import org.apache.hyracks.api.result.IResultSet;
-import org.apache.hyracks.http.api.IServletRequest;
-import org.apache.hyracks.http.api.IServletResponse;
-import org.apache.hyracks.http.server.AbstractServlet;
-import org.apache.hyracks.http.server.utils.HttpUtil;
-import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-public abstract class RestApiServlet extends AbstractServlet {
- private static final Logger LOGGER = LogManager.getLogger();
- private final ICcApplicationContext appCtx;
- private final ILangCompilationProvider compilationProvider;
- private final IParserFactory parserFactory;
- private final IStatementExecutorFactory statementExecutorFactory;
- private final IStorageComponentProvider componentProvider;
-
- public RestApiServlet(ConcurrentMap<String, Object> ctx, String[] paths, ICcApplicationContext appCtx,
- ILangCompilationProvider compilationProvider, IStatementExecutorFactory statementExecutorFactory,
- IStorageComponentProvider componentProvider) {
- super(ctx, paths);
- this.appCtx = appCtx;
- this.compilationProvider = compilationProvider;
- this.parserFactory = compilationProvider.getParserFactory();
- this.statementExecutorFactory = statementExecutorFactory;
- this.componentProvider = componentProvider;
- }
-
- /**
- * Initialize the Content-Type of the response, and construct a
- * SessionConfig with the appropriate output writer and output-format
- * based on the Accept: header and other servlet parameters.
- */
- static SessionOutput initResponse(IServletRequest request, IServletResponse response) throws IOException {
- HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_PLAIN, request);
- // CLEAN_JSON output is the default; most generally useful for a
- // programmatic HTTP API
- OutputFormat format = OutputFormat.CLEAN_JSON;
- // First check the "output" servlet parameter.
- String output = request.getParameter("output");
- String accept = request.getHeader("Accept", "");
- if (output != null) {
- if ("CSV".equals(output)) {
- format = OutputFormat.CSV;
- } else if ("ADM".equals(output)) {
- format = OutputFormat.ADM;
- }
- } else {
- // Second check the Accept: HTTP header.
- if (accept.contains("application/x-adm")) {
- format = OutputFormat.ADM;
- } else if (accept.contains("text/csv")) {
- format = OutputFormat.CSV;
- }
- }
- PlanFormat planFormat =
- PlanFormat.get(request.getParameter("plan-format"), "plan format", PlanFormat.STRING, LOGGER);
-
- // If it's JSON, check for the "lossless" flag
-
- if (format == OutputFormat.CLEAN_JSON
- && ("true".equals(request.getParameter("lossless")) || accept.contains("lossless=true"))) {
- format = OutputFormat.LOSSLESS_JSON;
- }
-
- SessionOutput.ResultAppender appendHandle = (app, handle) -> app.append("{ \"").append("handle")
- .append("\":" + " \"").append(handle).append("\" }");
- SessionConfig sessionConfig = new SessionConfig(format, planFormat);
-
- // If it's JSON or ADM, check for the "wrapper-array" flag. Default is
- // "true" for JSON and "false" for ADM. (Not applicable for CSV.)
- boolean wrapperArray = format == OutputFormat.CLEAN_JSON || format == OutputFormat.LOSSLESS_JSON;
- String wrapperParam = request.getParameter("wrapper-array");
- if (wrapperParam != null) {
- wrapperArray = Boolean.valueOf(wrapperParam);
- } else if (accept.contains("wrap-array=true")) {
- wrapperArray = true;
- } else if (accept.contains("wrap-array=false")) {
- wrapperArray = false;
- }
- sessionConfig.set(SessionConfig.FORMAT_WRAPPER_ARRAY, wrapperArray);
- // Now that format is set, output the content-type
- switch (format) {
- case ADM:
- HttpUtil.setContentType(response, "application/x-adm");
- break;
- case CLEAN_JSON:
- // No need to reflect "clean-ness" in output type; fall through
- case LOSSLESS_JSON:
- HttpUtil.setContentType(response, "application/json");
- break;
- case CSV:
- // Check for header parameter or in Accept:.
- if ("present".equals(request.getParameter("header")) || accept.contains("header=present")) {
- HttpUtil.setContentType(response, "text/csv; header=present");
- sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, true);
- } else {
- HttpUtil.setContentType(response, "text/csv; header=absent");
- }
- break;
- default:
- throw new IOException("Unknown format " + format);
- }
- return new SessionOutput(sessionConfig, response.writer(), null, null, appendHandle, null);
- }
-
- @Override
- protected void get(IServletRequest request, IServletResponse response) {
- getOrPost(request, response);
- }
-
- @Override
- protected void post(IServletRequest request, IServletResponse response) {
- getOrPost(request, response);
- }
-
- private void getOrPost(IServletRequest request, IServletResponse response) {
- try {
- String query = query(request);
- // enable cross-origin resource sharing
- response.setHeader("Access-Control-Allow-Origin", "*");
- response.setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
- SessionOutput sessionOutput = initResponse(request, response);
- QueryTranslator.ResultDelivery resultDelivery = whichResultDelivery(request);
- doHandle(response, query, sessionOutput, resultDelivery);
- } catch (Exception e) {
- response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
- LOGGER.log(Level.WARN, "Failure handling request", e);
- return;
- }
- }
-
- private void doHandle(IServletResponse response, String query, SessionOutput sessionOutput,
- ResultDelivery resultDelivery) throws JsonProcessingException {
- try {
- response.setStatus(HttpResponseStatus.OK);
- IHyracksClientConnection hcc = (IHyracksClientConnection) ctx.get(HYRACKS_CONNECTION_ATTR);
- IParser parser = parserFactory.createParser(query);
- List<Statement> aqlStatements = parser.parse();
- validate(aqlStatements);
- MetadataManager.INSTANCE.init();
- IStatementExecutor translator = statementExecutorFactory.create(appCtx, aqlStatements, sessionOutput,
- compilationProvider, componentProvider);
- final IResultSet resultSet = ServletUtil.getResultSet(hcc, appCtx, ctx);
- final IRequestParameters requestParameters = new RequestParameters(resultSet,
- new ResultProperties(resultDelivery), new IStatementExecutor.Stats(), null, null, null, null, true);
- translator.compileAndExecute(hcc, null, requestParameters);
- } catch (AsterixException | TokenMgrError | org.apache.asterix.aqlplus.parser.TokenMgrError pe) {
- response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
- GlobalConfig.ASTERIX_LOGGER.log(Level.ERROR, pe.getMessage(), pe);
- String errorMessage = ResultUtil.buildParseExceptionMessage(pe, query);
- ObjectNode errorResp =
- ResultUtil.getErrorResponse(2, errorMessage, "", ResultUtil.extractFullStackTrace(pe));
- sessionOutput.out().write(OBJECT_MAPPER.writeValueAsString(errorResp));
- } catch (Exception e) {
- GlobalConfig.ASTERIX_LOGGER.log(Level.ERROR, e.getMessage(), e);
- response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
- ResultUtil.apiErrorHandler(sessionOutput.out(), e);
- }
- }
-
- //TODO: Both Get and Post of this API must use the same parameter names
- private String query(IServletRequest request) {
- if (request.getHttpRequest().method() == HttpMethod.POST) {
- return HttpUtil.getRequestBody(request);
- } else {
- return getQueryParameter(request);
- }
- }
-
- private void validate(List<Statement> aqlStatements) throws AsterixException {
- for (Statement st : aqlStatements) {
- if ((st.getCategory() & getAllowedCategories()) == 0) {
- throw new AsterixException(String.format(getErrorMessage(), st.getKind()));
- }
- }
- }
-
- protected QueryTranslator.ResultDelivery whichResultDelivery(IServletRequest request) {
- String mode = request.getParameter("mode");
- if (mode != null) {
- if ("asynchronous".equals(mode) || "async".equals(mode)) {
- return QueryTranslator.ResultDelivery.ASYNC;
- } else if ("asynchronous-deferred".equals(mode) || "deferred".equals(mode)) {
- return QueryTranslator.ResultDelivery.DEFERRED;
- }
- }
- return QueryTranslator.ResultDelivery.IMMEDIATE;
- }
-
- protected abstract String getQueryParameter(IServletRequest request);
-
- protected abstract byte getAllowedCategories();
-
- protected abstract String getErrorMessage();
-}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/result/ResultPrinterTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/result/ResultPrinterTest.java
index 7e851bf..3335cbf 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/result/ResultPrinterTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/result/ResultPrinterTest.java
@@ -28,7 +28,6 @@
import org.apache.asterix.api.http.server.ResultUtil;
import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.config.CompilerProperties;
-import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.test.common.ResultExtractor;
import org.apache.asterix.translator.IStatementExecutor;
import org.apache.asterix.translator.SessionConfig;
@@ -75,7 +74,7 @@
boolean exceptionThrown = false;
try {
// ensure result is valid json and error will be returned and not results.
- ResultExtractor.extract(IOUtils.toInputStream(resultStr, StandardCharsets.UTF_8));
+ ResultExtractor.extract(IOUtils.toInputStream(resultStr, StandardCharsets.UTF_8), StandardCharsets.UTF_8);
} catch (Exception e) {
exceptionThrown = true;
Assert.assertTrue(e.getMessage().contains(expectedException.getMessage()));
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
index ab8969e..37f471e 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
@@ -19,6 +19,7 @@
package org.apache.asterix.common;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
import java.rmi.RemoteException;
import java.util.Arrays;
import java.util.LinkedHashSet;
@@ -131,7 +132,7 @@
public static long getDatasetCount(String datasetName) throws Exception {
final String query = "SELECT VALUE COUNT(*) FROM `" + datasetName + "`;";
final InputStream responseStream = TEST_EXECUTOR.executeQueryService(query,
- TEST_EXECUTOR.getEndpoint(Servlets.QUERY_SERVICE), OUTPUT_FORMAT);
+ TEST_EXECUTOR.getEndpoint(Servlets.QUERY_SERVICE), OUTPUT_FORMAT, StandardCharsets.UTF_8);
final ObjectNode response = OBJECT_MAPPER.readValue(responseStream, ObjectNode.class);
final JsonNode result = response.get("results");
// make sure there is a single value in result
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java
index d20d72d..48488a4 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java
@@ -142,6 +142,7 @@
Assert.assertEquals("1.234567ms", Duration.formatNanos(1234567l));
Assert.assertEquals("123.456µs", Duration.formatNanos(123456l));
Assert.assertEquals("12.345µs", Duration.formatNanos(12345l));
+ Assert.assertEquals("12.345us", Duration.formatNanos(12345l, true));
Assert.assertEquals("1.234µs", Duration.formatNanos(1234l));
Assert.assertEquals("123ns", Duration.formatNanos(123l));
Assert.assertEquals("12ns", Duration.formatNanos(12l));
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
index e85fedf..969d23d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
@@ -21,6 +21,7 @@
import java.io.InputStream;
import java.net.URI;
+import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Iterator;
import java.util.List;
@@ -49,7 +50,7 @@
@Override
public InputStream executeQueryService(String str, TestCaseContext.OutputFormat fmt, URI uri,
- List<TestCase.CompilationUnit.Parameter> params, boolean jsonEncoded,
+ List<TestCase.CompilationUnit.Parameter> params, boolean jsonEncoded, Charset responseCharset,
Predicate<Integer> responseCodeValidator, boolean cancellable) throws Exception {
String clientContextId = UUID.randomUUID().toString();
final List<TestCase.CompilationUnit.Parameter> newParams = cancellable
@@ -57,7 +58,7 @@
Callable<InputStream> query = () -> {
try {
return CancellationTestExecutor.super.executeQueryService(str, fmt, uri, newParams, jsonEncoded,
- responseCodeValidator, true);
+ responseCharset, responseCodeValidator, true);
} catch (Exception e) {
e.printStackTrace();
throw e;
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
index 37e1213..de7dac2 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
@@ -83,16 +83,16 @@
private static final Logger LOGGER = LogManager.getLogger();
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- public static InputStream extract(InputStream resultStream) throws Exception {
- return extract(resultStream, EnumSet.of(ResultField.RESULTS));
+ public static InputStream extract(InputStream resultStream, Charset resultCharset) throws Exception {
+ return extract(resultStream, EnumSet.of(ResultField.RESULTS), resultCharset);
}
- public static InputStream extractMetrics(InputStream resultStream) throws Exception {
- return extract(resultStream, EnumSet.of(ResultField.METRICS));
+ public static InputStream extractMetrics(InputStream resultStream, Charset resultCharset) throws Exception {
+ return extract(resultStream, EnumSet.of(ResultField.METRICS), resultCharset);
}
- public static String extractHandle(InputStream resultStream) throws Exception {
- String result = IOUtils.toString(resultStream, StandardCharsets.UTF_8);
+ public static String extractHandle(InputStream resultStream, Charset responseCharset) throws Exception {
+ String result = IOUtils.toString(resultStream, responseCharset);
ObjectNode resultJson = OBJECT_MAPPER.readValue(result, ObjectNode.class);
final JsonNode handle = resultJson.get("handle");
if (handle != null) {
@@ -107,8 +107,9 @@
return null;
}
- private static InputStream extract(InputStream resultStream, EnumSet<ResultField> resultFields) throws Exception {
- final String resultStr = IOUtils.toString(resultStream, Charset.defaultCharset());
+ private static InputStream extract(InputStream resultStream, EnumSet<ResultField> resultFields,
+ Charset resultCharset) throws Exception {
+ final String resultStr = IOUtils.toString(resultStream, resultCharset);
final ObjectNode result = OBJECT_MAPPER.readValue(resultStr, ObjectNode.class);
LOGGER.debug("+++++++\n" + result + "\n+++++++\n");
@@ -171,7 +172,7 @@
throw new IllegalStateException("Unexpected result field: " + fieldKind);
}
}
- return IOUtils.toInputStream(resultBuilder.toString(), StandardCharsets.UTF_8);
+ return IOUtils.toInputStream(resultBuilder, resultCharset);
}
private static void checkForErrors(ObjectNode result) throws Exception {
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index b143ea9..683d5c8 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.test.common;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
@@ -35,12 +37,14 @@
import java.net.Socket;
import java.net.URI;
import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
@@ -56,6 +60,7 @@
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.asterix.api.http.server.QueryServiceServlet;
@@ -139,11 +144,7 @@
public static final Set<String> NON_CANCELLABLE =
Collections.unmodifiableSet(new HashSet<>(Arrays.asList("store", "validate")));
- private final IPollTask plainExecutor = (testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit,
- queryCount, expectedResultFileCtxs, testFile, actualPath) -> {
- executeTestFile(testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit, queryCount,
- expectedResultFileCtxs, testFile, actualPath);
- };
+ private final IPollTask plainExecutor = this::executeTestFile;
public static final String DELIVERY_ASYNC = "async";
public static final String DELIVERY_DEFERRED = "deferred";
@@ -155,6 +156,8 @@
private static Map<String, InetSocketAddress> ncEndPoints;
private static Map<String, InetSocketAddress> replicationAddress;
+ private static final List<Charset> charsetsRemaining = new ArrayList<>();
+
/*
* Instance members
*/
@@ -211,24 +214,23 @@
}
public void runScriptAndCompareWithResult(File scriptFile, File expectedFile, File actualFile,
- ComparisonEnum compare) throws Exception {
+ ComparisonEnum compare, Charset actualEncoding) throws Exception {
LOGGER.info("Expected results file: {} ", expectedFile);
- BufferedReader readerExpected =
- new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile), "UTF-8"));
- BufferedReader readerActual =
- new BufferedReader(new InputStreamReader(new FileInputStream(actualFile), "UTF-8"));
boolean regex = false;
- try {
+ try (BufferedReader readerExpected =
+ new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile), UTF_8));
+ BufferedReader readerActual =
+ new BufferedReader(new InputStreamReader(new FileInputStream(actualFile), actualEncoding))) {
if (ComparisonEnum.BINARY.equals(compare)) {
if (!IOUtils.contentEquals(new FileInputStream(actualFile), new FileInputStream(expectedFile))) {
throw new Exception("Result for " + scriptFile + ": actual file did not match expected result");
}
return;
} else if (actualFile.toString().endsWith(".regex")) {
- runScriptAndCompareWithResultRegex(scriptFile, expectedFile, actualFile);
+ runScriptAndCompareWithResultRegex(scriptFile, readerExpected, readerActual);
return;
} else if (actualFile.toString().endsWith(".regexadm")) {
- runScriptAndCompareWithResultRegexAdm(scriptFile, expectedFile, actualFile);
+ runScriptAndCompareWithResultRegexAdm(scriptFile, readerExpected, readerActual);
return;
}
String lineExpected, lineActual;
@@ -278,11 +280,8 @@
throw createLineChangedException(scriptFile, "<EOF>", lineActual, num);
}
} catch (Exception e) {
- LOGGER.info("Actual results file: {}", actualFile);
+ LOGGER.info("Actual results file: {} encoding: {}", actualFile, actualEncoding);
throw e;
- } finally {
- readerExpected.close();
- readerActual.close();
}
}
@@ -396,54 +395,48 @@
return true;
}
- public void runScriptAndCompareWithResultRegex(File scriptFile, File expectedFile, File actualFile)
- throws Exception {
+ public void runScriptAndCompareWithResultRegex(File scriptFile, BufferedReader readerExpected,
+ BufferedReader readerActual) throws Exception {
String lineExpected, lineActual;
- try (BufferedReader readerExpected =
- new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile), "UTF-8"));
- BufferedReader readerActual =
- new BufferedReader(new InputStreamReader(new FileInputStream(actualFile), "UTF-8"))) {
- StringBuilder actual = new StringBuilder();
- while ((lineActual = readerActual.readLine()) != null) {
- actual.append(lineActual).append('\n');
+ StringBuilder actual = new StringBuilder();
+ while ((lineActual = readerActual.readLine()) != null) {
+ actual.append(lineActual).append('\n');
+ }
+ while ((lineExpected = readerExpected.readLine()) != null) {
+ if ("".equals(lineExpected.trim())) {
+ continue;
}
- while ((lineExpected = readerExpected.readLine()) != null) {
- if ("".equals(lineExpected.trim())) {
- continue;
- }
- Matcher m = REGEX_LINES_PATTERN.matcher(lineExpected);
- if (!m.matches()) {
- throw new IllegalArgumentException(
- "Each line of regex file must conform to: [-]/regex/[flags]: " + expectedFile);
- }
- String negateStr = m.group(1);
- String expression = m.group(2);
- String flagStr = m.group(3);
- boolean negate = "-".equals(negateStr);
- int flags = Pattern.MULTILINE;
- if (flagStr.contains("m")) {
- flags |= Pattern.DOTALL;
- }
- if (flagStr.contains("i")) {
- flags |= Pattern.CASE_INSENSITIVE;
- }
- Pattern linePattern = Pattern.compile(expression, flags);
- boolean match = linePattern.matcher(actual).find();
- if (match && !negate || negate && !match) {
- continue;
- }
- throw new Exception("Result for " + scriptFile + ": expected pattern '" + expression
- + "' not found in result: " + actual);
+ Matcher m = REGEX_LINES_PATTERN.matcher(lineExpected);
+ if (!m.matches()) {
+ throw new IllegalArgumentException("Each line of regex file must conform to: [-]/regex/[flags]");
}
+ String negateStr = m.group(1);
+ String expression = m.group(2);
+ String flagStr = m.group(3);
+ boolean negate = "-".equals(negateStr);
+ int flags = Pattern.MULTILINE;
+ if (flagStr.contains("m")) {
+ flags |= Pattern.DOTALL;
+ }
+ if (flagStr.contains("i")) {
+ flags |= Pattern.CASE_INSENSITIVE;
+ }
+ Pattern linePattern = Pattern.compile(expression, flags);
+ boolean match = linePattern.matcher(actual).find();
+ if (match && !negate || negate && !match) {
+ continue;
+ }
+ throw new Exception("Result for " + scriptFile + ": expected pattern '" + expression
+ + "' not found in result: " + actual);
}
}
- public void runScriptAndCompareWithResultRegexAdm(File scriptFile, File expectedFile, File actualFile)
- throws Exception {
+ public void runScriptAndCompareWithResultRegexAdm(File scriptFile, BufferedReader expectedFile,
+ BufferedReader actualFile) throws Exception {
StringWriter actual = new StringWriter();
StringWriter expected = new StringWriter();
- IOUtils.copy(new FileInputStream(actualFile), actual, StandardCharsets.UTF_8);
- IOUtils.copy(new FileInputStream(expectedFile), expected, StandardCharsets.UTF_8);
+ IOUtils.copy(actualFile, actual);
+ IOUtils.copy(expectedFile, expected);
Pattern pattern = Pattern.compile(expected.toString(), Pattern.DOTALL | Pattern.MULTILINE);
if (!pattern.matcher(actual.toString()).matches()) {
// figure out where the problem first occurs...
@@ -570,21 +563,27 @@
}
public InputStream executeQueryService(String str, URI uri, OutputFormat fmt) throws Exception {
- return executeQueryService(str, fmt, uri, new ArrayList<>(), false);
+ return executeQueryService(str, fmt, uri, new ArrayList<>(), false, UTF_8);
+ }
+
+ public InputStream executeQueryService(String str, URI uri, OutputFormat fmt, Charset resultCharset)
+ throws Exception {
+ return executeQueryService(str, fmt, uri, new ArrayList<>(), false, resultCharset);
}
public InputStream executeQueryService(String str, OutputFormat fmt, URI uri, List<Parameter> params,
- boolean jsonEncoded) throws Exception {
- return executeQueryService(str, fmt, uri, params, jsonEncoded, null, false);
+ boolean jsonEncoded, Charset responseCharset) throws Exception {
+ return executeQueryService(str, fmt, uri, params, jsonEncoded, responseCharset, null, false);
}
public InputStream executeQueryService(String str, OutputFormat fmt, URI uri, List<Parameter> params,
boolean jsonEncoded, Predicate<Integer> responseCodeValidator) throws Exception {
- return executeQueryService(str, fmt, uri, params, jsonEncoded, responseCodeValidator, false);
+ return executeQueryService(str, fmt, uri, params, jsonEncoded, UTF_8, responseCodeValidator, false);
}
public InputStream executeQueryService(String str, OutputFormat fmt, URI uri, List<Parameter> params,
- boolean jsonEncoded, Predicate<Integer> responseCodeValidator, boolean cancellable) throws Exception {
+ boolean jsonEncoded, Charset responseCharset, Predicate<Integer> responseCodeValidator, boolean cancellable)
+ throws Exception {
List<Parameter> newParams = upsertParam(params, "format", ParameterTypeEnum.STRING, fmt.mimeType());
newParams = upsertParam(newParams, QueryServiceServlet.Parameter.PLAN_FORMAT.str(), ParameterTypeEnum.STRING,
DEFAULT_PLAN_FORMAT);
@@ -602,6 +601,10 @@
// Set accepted output response type
method.setHeader("Origin", uri.getScheme() + uri.getAuthority());
method.setHeader("Accept", OutputFormat.CLEAN_JSON.mimeType());
+ method.setHeader("Accept-Charset", responseCharset.name());
+ if (!responseCharset.equals(UTF_8)) {
+ LOGGER.info("using Accept-Charset: {}", responseCharset.name());
+ }
HttpResponse response = executeHttpRequest(method);
if (responseCodeValidator != null) {
checkResponse(response, responseCodeValidator);
@@ -609,6 +612,49 @@
return response.getEntity().getContent();
}
+ private Charset selectCharset(File result) throws IOException {
+ // choose an encoding that works for this input
+ return selectCharset(FileUtils.readFileToString(result, UTF_8));
+ }
+
+ private Charset selectCharset(String payload) {
+ // choose an encoding that works for this input
+ return nextCharset(charset -> canEncodeDecode(charset, payload));
+ }
+
+ public static Charset nextCharset(Predicate<Charset> test) {
+ synchronized (charsetsRemaining) {
+ while (true) {
+ for (Iterator<Charset> iter = charsetsRemaining.iterator(); iter.hasNext();) {
+ Charset next = iter.next();
+ if (test.test(next)) {
+ iter.remove();
+ return next;
+ }
+ }
+ List<Charset> allCharsets = Charset.availableCharsets().values().stream()
+ .filter(c -> canEncodeDecode(c, "\n\t\\[]{}'\"")).collect(Collectors.toList());
+ Collections.shuffle(allCharsets);
+ charsetsRemaining.addAll(allCharsets);
+ }
+ }
+ }
+
+ // duplicated from hyracks-test-support as transitive dependencies on test-jars are not handled correctly
+ private static boolean canEncodeDecode(Charset charset, String input) {
+ try {
+ if (input.equals(new String(input.getBytes(charset), charset))) {
+ // workaround for https://bugs.openjdk.java.net/browse/JDK-6392670 and similar
+ if (input.equals(charset.decode(charset.encode(CharBuffer.wrap(input))).toString())) {
+ return true;
+ }
+ }
+ } catch (Exception e) {
+ LOGGER.debug("cannot encode / decode {} with {} due to exception", input, charset.displayName(), e);
+ }
+ return false;
+ }
+
protected List<Parameter> upsertParam(List<Parameter> params, String name, ParameterTypeEnum type, String value) {
boolean replaced = false;
List<Parameter> result = new ArrayList<>();
@@ -646,7 +692,6 @@
for (Parameter param : params) {
builder.addParameter(param.getName(), param.getValue());
}
- builder.setCharset(StandardCharsets.UTF_8);
return builder.build();
}
@@ -656,8 +701,8 @@
for (Parameter param : params) {
builder.addParameter(param.getName(), param.getValue());
}
- builder.setCharset(StandardCharsets.UTF_8);
- body.ifPresent(s -> builder.setEntity(new StringEntity(s, StandardCharsets.UTF_8)));
+ builder.setCharset(UTF_8);
+ body.ifPresent(s -> builder.setEntity(new StringEntity(s, UTF_8)));
return builder.build();
}
@@ -681,7 +726,7 @@
for (Parameter param : params) {
builder.addParameter(param.getName(), param.getValue());
}
- builder.setCharset(StandardCharsets.UTF_8);
+ builder.setCharset(UTF_8);
return builder.build();
}
@@ -695,9 +740,9 @@
builder.addParameter(stmtParam, statement);
} else {
// this seems pretty bad - we should probably fix the API and not the client
- builder.setEntity(new StringEntity(statement, StandardCharsets.UTF_8));
+ builder.setEntity(new StringEntity(statement, UTF_8));
}
- builder.setCharset(StandardCharsets.UTF_8);
+ builder.setCharset(UTF_8);
return builder.build();
}
@@ -732,7 +777,7 @@
} catch (JsonProcessingException e) {
e.printStackTrace();
}
- builder.setCharset(StandardCharsets.UTF_8);
+ builder.setCharset(UTF_8);
return builder.build();
}
@@ -765,8 +810,7 @@
// and returns the contents as a string
// This string is later passed to REST API for execution.
public String readTestFile(File testFile) throws Exception {
- BufferedReader reader =
- new BufferedReader(new InputStreamReader(new FileInputStream(testFile), StandardCharsets.UTF_8));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(testFile), UTF_8));
String line;
StringBuilder stringBuilder = new StringBuilder();
String ls = System.getProperty("line.separator");
@@ -820,9 +864,9 @@
future.get();
ByteArrayInputStream bisIn = new ByteArrayInputStream(baos.toByteArray());
StringWriter writerIn = new StringWriter();
- IOUtils.copy(bisIn, writerIn, StandardCharsets.UTF_8);
+ IOUtils.copy(bisIn, writerIn, UTF_8);
StringWriter writerErr = new StringWriter();
- IOUtils.copy(p.getErrorStream(), writerErr, StandardCharsets.UTF_8);
+ IOUtils.copy(p.getErrorStream(), writerErr, UTF_8);
StringBuffer stdOut = writerIn.getBuffer();
if (writerErr.getBuffer().length() > 0) {
@@ -918,18 +962,18 @@
expectedResultFileCtxs);
break;
case "txnqbc": // qbc represents query before crash
- resultStream = query(cUnit, testFile.getName(), statement);
+ resultStream = query(cUnit, testFile.getName(), statement, UTF_8);
qbcFile = getTestCaseQueryBeforeCrashFile(actualPath, testCaseCtx, cUnit);
writeOutputToFile(qbcFile, resultStream);
break;
case "txnqar": // qar represents query after recovery
- resultStream = query(cUnit, testFile.getName(), statement);
+ resultStream = query(cUnit, testFile.getName(), statement, UTF_8);
File qarFile = new File(actualPath + File.separator
+ testCaseCtx.getTestCase().getFilePath().replace(File.separator, "_") + "_" + cUnit.getName()
+ "_qar.adm");
writeOutputToFile(qarFile, resultStream);
qbcFile = getTestCaseQueryBeforeCrashFile(actualPath, testCaseCtx, cUnit);
- runScriptAndCompareWithResult(testFile, qbcFile, qarFile, ComparisonEnum.TEXT);
+ runScriptAndCompareWithResult(testFile, qbcFile, qarFile, ComparisonEnum.TEXT, UTF_8);
break;
case "txneu": // eu represents erroneous update
try {
@@ -1172,7 +1216,7 @@
throw new IllegalArgumentException("Unexpected format for method " + reqType + ": " + extension);
}
if (handleVar != null) {
- String handle = ResultExtractor.extractHandle(resultStream);
+ String handle = ResultExtractor.extractHandle(resultStream, UTF_8);
if (handle != null) {
variableCtx.put(handleVar, handle);
} else {
@@ -1181,15 +1225,15 @@
} else {
if (expectedResultFile == null) {
if (testFile.getName().startsWith(DIAGNOSE)) {
- LOGGER.info("Diagnostic output: {}", IOUtils.toString(resultStream, StandardCharsets.UTF_8));
+ LOGGER.info("Diagnostic output: {}", IOUtils.toString(resultStream, UTF_8));
} else {
- LOGGER.info("Unexpected output: {}", IOUtils.toString(resultStream, StandardCharsets.UTF_8));
+ LOGGER.info("Unexpected output: {}", IOUtils.toString(resultStream, UTF_8));
Assert.fail("no result file for " + testFile.toString() + "; queryCount: " + queryCount
+ ", filectxs.size: " + numResultFiles);
}
} else {
writeOutputToFile(actualResultFile, resultStream);
- runScriptAndCompareWithResult(testFile, expectedResultFile, actualResultFile, compare);
+ runScriptAndCompareWithResult(testFile, expectedResultFile, actualResultFile, compare, UTF_8);
}
}
queryCount.increment();
@@ -1207,23 +1251,25 @@
URI uri = testFile.getName().endsWith("aql") ? getEndpoint(Servlets.QUERY_AQL)
: getEndpoint(Servlets.QUERY_SERVICE);
boolean isJsonEncoded = isJsonEncoded(extractHttpRequestType(statement));
+ Charset responseCharset = expectedResultFile == null ? UTF_8 : selectCharset(expectedResultFile);
InputStream resultStream;
if (DELIVERY_IMMEDIATE.equals(delivery)) {
+ resultStream = executeQueryService(statement, fmt, uri, params, isJsonEncoded, responseCharset, null,
+ isCancellable(reqType));
resultStream =
- executeQueryService(statement, fmt, uri, params, isJsonEncoded, null, isCancellable(reqType));
- resultStream = METRICS_QUERY_TYPE.equals(reqType) ? ResultExtractor.extractMetrics(resultStream)
- : ResultExtractor.extract(resultStream);
+ METRICS_QUERY_TYPE.equals(reqType) ? ResultExtractor.extractMetrics(resultStream, responseCharset)
+ : ResultExtractor.extract(resultStream, responseCharset);
} else {
String handleVar = getHandleVariable(statement);
resultStream = executeQueryService(statement, fmt, uri,
- upsertParam(params, "mode", ParameterTypeEnum.STRING, delivery), isJsonEncoded);
- String handle = ResultExtractor.extractHandle(resultStream);
+ upsertParam(params, "mode", ParameterTypeEnum.STRING, delivery), isJsonEncoded, responseCharset);
+ String handle = ResultExtractor.extractHandle(resultStream, responseCharset);
Assert.assertNotNull("no handle for " + reqType + " test " + testFile.toString(), handleVar);
variableCtx.put(handleVar, toQueryServiceHandle(handle));
}
if (actualResultFile == null) {
if (testFile.getName().startsWith(DIAGNOSE)) {
- LOGGER.info("Diagnostic output: {}", IOUtils.toString(resultStream, StandardCharsets.UTF_8));
+ LOGGER.info("Diagnostic output: {}", IOUtils.toString(resultStream, responseCharset));
} else {
Assert.fail("no result file for " + testFile.toString() + "; queryCount: " + queryCount
+ ", filectxs.size: " + numResultFiles);
@@ -1238,7 +1284,7 @@
+ ", filectxs.size: " + numResultFiles);
}
}
- runScriptAndCompareWithResult(testFile, expectedResultFile, actualResultFile, compare);
+ runScriptAndCompareWithResult(testFile, expectedResultFile, actualResultFile, compare, responseCharset);
if (!reqType.equals("validate")) {
queryCount.increment();
}
@@ -1266,8 +1312,7 @@
throw new Exception(
"Failed to delete an existing result file: " + actualResultFile.getAbsolutePath());
}
- writeOutputToFile(actualResultFile,
- new ByteArrayInputStream(poller.poll().getBytes(StandardCharsets.UTF_8)));
+ writeOutputToFile(actualResultFile, new ByteArrayInputStream(poller.poll().getBytes(UTF_8)));
variableCtx.put(key, actualResultFile);
validate(actualPath, testCaseCtx, cUnit, statement, variableCtx, testFile, ctx, queryCount,
expectedResultFileCtxs);
@@ -1399,8 +1444,8 @@
private InputStream executeUpdateOrDdl(String statement, OutputFormat outputFormat, URI serviceUri)
throws Exception {
- InputStream resultStream = executeQueryService(statement, serviceUri, outputFormat);
- return ResultExtractor.extract(resultStream);
+ InputStream resultStream = executeQueryService(statement, serviceUri, outputFormat, UTF_8);
+ return ResultExtractor.extract(resultStream, UTF_8);
}
protected static boolean isExpected(Exception e, CompilationUnit cUnit) {
@@ -1554,7 +1599,7 @@
String endpoint = "/admin/cluster/node/" + nodeId + "/config";
InputStream executeJSONGet = executeJSONGet(fmt, createEndpointURI(endpoint, null));
StringWriter actual = new StringWriter();
- IOUtils.copy(executeJSONGet, actual, StandardCharsets.UTF_8);
+ IOUtils.copy(executeJSONGet, actual, UTF_8);
String config = actual.toString();
int nodePid = new ObjectMapper().readValue(config, ObjectNode.class).get("pid").asInt();
if (nodePid <= 1) {
@@ -1584,7 +1629,7 @@
String endpoint = "/admin/cluster/node/" + nodeId + "/config";
InputStream executeJSONGet = executeJSONGet(fmt, createEndpointURI(endpoint, null));
StringWriter actual = new StringWriter();
- IOUtils.copy(executeJSONGet, actual, StandardCharsets.UTF_8);
+ IOUtils.copy(executeJSONGet, actual, UTF_8);
String config = actual.toString();
ObjectMapper om = new ObjectMapper();
String logDir = om.readTree(config).findPath("txn.log.dir").asText();
@@ -1761,8 +1806,8 @@
ArrayList<String> toBeDropped = new ArrayList<>();
InputStream resultStream = executeQueryService(
"select dv.DataverseName from Metadata.`Dataverse` as dv order by dv.DataverseName;",
- getEndpoint(Servlets.QUERY_SERVICE), OutputFormat.CLEAN_JSON);
- String out = IOUtils.toString(resultStream, StandardCharsets.UTF_8);
+ getEndpoint(Servlets.QUERY_SERVICE), OutputFormat.CLEAN_JSON, UTF_8);
+ String out = IOUtils.toString(resultStream, UTF_8);
ObjectMapper om = new ObjectMapper();
om.setConfig(om.getDeserializationConfig().with(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT));
JsonNode result;
@@ -1794,8 +1839,8 @@
dropStatement.append(dv);
dropStatement.append(";\n");
resultStream = executeQueryService(dropStatement.toString(), getEndpoint(Servlets.QUERY_SERVICE),
- OutputFormat.CLEAN_JSON);
- ResultExtractor.extract(resultStream);
+ OutputFormat.CLEAN_JSON, UTF_8);
+ ResultExtractor.extract(resultStream, UTF_8);
}
}
} catch (Throwable th) {
@@ -1964,11 +2009,12 @@
return !NON_CANCELLABLE.contains(type);
}
- private InputStream query(CompilationUnit cUnit, String testFile, String statement) throws Exception {
+ private InputStream query(CompilationUnit cUnit, String testFile, String statement, Charset responseCharset)
+ throws Exception {
final URI uri = getQueryServiceUri(testFile);
final InputStream inputStream = executeQueryService(statement, OutputFormat.forCompilationUnit(cUnit), uri,
- cUnit.getParameter(), true, null, false);
- return ResultExtractor.extract(inputStream);
+ cUnit.getParameter(), true, responseCharset, null, false);
+ return ResultExtractor.extract(inputStream, responseCharset);
}
private URI getQueryServiceUri(String extension) throws URISyntaxException {
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java
index e4e300a..5dd41c3 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java
@@ -25,6 +25,7 @@
import java.io.File;
import java.io.FileOutputStream;
import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -149,7 +150,8 @@
}
writer.close();
// Compares the actual result and the expected result.
- runScriptAndCompareWithResult(queryFile, expectedFile, actualResultFile, ComparisonEnum.TEXT);
+ runScriptAndCompareWithResult(queryFile, expectedFile, actualResultFile, ComparisonEnum.TEXT,
+ StandardCharsets.UTF_8);
} catch (Exception e) {
GlobalConfig.ASTERIX_LOGGER.warn("Failed while testing file " + queryFile);
throw e;
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/Duration.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/Duration.java
index a8b43b7..905497e 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/Duration.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/Duration.java
@@ -47,13 +47,25 @@
this.nanoDigits = nanoDigits;
}
+ public String asciiSafeUnit() {
+ return this == MICRO ? "us" : unit;
+ }
+
public static String formatNanos(long nanoTime) {
+ return formatNanos(nanoTime, false);
+ }
+
+ public static String formatNanos(long nanoTime, boolean asciiSafe) {
StringBuilder sb = new StringBuilder();
- formatNanos(nanoTime, sb);
+ formatNanos(nanoTime, sb, asciiSafe);
return sb.toString();
}
public static void formatNanos(long nanoTime, StringBuilder out) {
+ formatNanos(nanoTime, out, false);
+ }
+
+ public static void formatNanos(long nanoTime, StringBuilder out, boolean asciiSafe) {
final String strTime = String.valueOf(Math.abs(nanoTime));
final int len = strTime.length();
for (Duration tu : VALUES) {
@@ -67,7 +79,7 @@
if (k > 0) {
out.append('.').append(strTime, n, k + 1);
}
- out.append(tu.unit);
+ out.append(asciiSafe ? tu.asciiSafeUnit() : tu.unit);
break;
}
}
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/TestMethodTracer.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/TestMethodTracer.java
index b762517..7e29ed5 100644
--- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/TestMethodTracer.java
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/TestMethodTracer.java
@@ -56,7 +56,7 @@
@Override
protected void failed(Throwable e, Description description) {
- LOGGER.log(level, "### {} FAILED ({})", description.getMethodName(), e.getClass().getName());
+ LOGGER.log(level, "### {} FAILED", description.getMethodName(), e);
}
@Override
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/server/RSSFeedServlet.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/server/RSSFeedServlet.java
index 639e036..17c98bc 100644
--- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/server/RSSFeedServlet.java
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/server/RSSFeedServlet.java
@@ -67,13 +67,12 @@
String feedType = req.getParameter(FEED_TYPE);
feedType = (feedType != null) ? feedType : defaultFeedType;
feed.setFeedType(feedType);
- HttpUtil.setContentType(res, MIME_TYPE);
+ HttpUtil.setContentType(res, MIME_TYPE, req);
SyndFeedOutput output = new SyndFeedOutput();
output.output(feed, res.writer());
} catch (FeedException | ParseException ex) {
GlobalConfig.ASTERIX_LOGGER.log(Level.WARN, ex.getMessage(), ex);
- String msg = COULD_NOT_GENERATE_FEED_ERROR;
- res.writer().print(msg);
+ res.writer().print(COULD_NOT_GENERATE_FEED_ERROR);
res.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
}
}
diff --git a/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh
index e337cb0..e0cff32 100755
--- a/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh
+++ b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh
@@ -86,7 +86,7 @@
fi
fi
-export JAVA_VERSION=$(java -version 2>&1 | head -1 | awk '{ print $3 }' | tr -d '"')
+export JAVA_VERSION=$($JAVACMD -version 2>&1 | head -1 | awk '{ print $3 }' | tr -d '"')
case $JAVA_VERSION in
1.8*|1.9*|10*|11*)
;;
diff --git a/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh b/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh
index 522fb7c..80647d4 100755
--- a/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh
+++ b/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh
@@ -99,11 +99,14 @@
JAVACMD=`which java`
fi
fi
-"$JAVACMD" -version 2>&1 | grep -q '1\.[89]' || {
- echo "JAVA_HOME must be at version 1.8 or later:"
- "$JAVACMD" -version
+export JAVA_VERSION=$($JAVACMD -version 2>&1 | head -1 | awk '{ print $3 }' | tr -d '"')
+case $JAVA_VERSION in
+ 1.8*|1.9*|10*|11*)
+ ;;
+ *)
+ echo JAVA_HOME must be at version 1.8 or later, but is: $JAVA_VERSION
exit 2
-}
+esac
DIRNAME=$(dirname "$0")
[ $(echo $DIRNAME | wc -l) -ne 1 ] && {
echo "Paths with spaces are not supported"
diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SampleLocalClusterIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SampleLocalClusterIT.java
index 0479bb2..cd72591 100644
--- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SampleLocalClusterIT.java
+++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SampleLocalClusterIT.java
@@ -24,6 +24,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
@@ -121,7 +122,7 @@
public void test1_sanityQuery() throws Exception {
TestExecutor testExecutor = new TestExecutor();
InputStream resultStream = testExecutor.executeQueryService("1+1;",
- testExecutor.getEndpoint(Servlets.QUERY_SERVICE), OutputFormat.ADM);
+ testExecutor.getEndpoint(Servlets.QUERY_SERVICE), OutputFormat.ADM, StandardCharsets.UTF_8);
final ObjectMapper objectMapper = new ObjectMapper();
final ObjectNode response = objectMapper.readValue(resultStream, ObjectNode.class);
final JsonNode result = response.get("results");
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index fbf17c1..0336790 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -1085,6 +1085,12 @@
</dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ <version>${hyracks.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-dataflow-std</artifactId>
<version>${hyracks.version}</version>
</dependency>
diff --git a/hyracks-fullstack/hyracks/hyracks-http/pom.xml b/hyracks-fullstack/hyracks/hyracks-http/pom.xml
index a67fa15..46e2004 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-http/pom.xml
@@ -95,5 +95,11 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-test-support</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedResponse.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedResponse.java
index e72bee9..ac87592 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedResponse.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedResponse.java
@@ -107,7 +107,7 @@
public synchronized PrintWriter writer() {
if (writer == null) {
Charset charset = io.netty.handler.codec.http.HttpUtil.getCharset(response, StandardCharsets.UTF_8);
- writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(outputStream, charset)));
+ writer = new PrintWriter(new OutputStreamWriter(outputStream, charset));
}
return writer;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FullResponse.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FullResponse.java
index 85a0a43..b42db39 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FullResponse.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FullResponse.java
@@ -52,7 +52,7 @@
public FullResponse(ChannelHandlerContext ctx, FullHttpRequest request) {
this.ctx = ctx;
- baos = new ByteArrayOutputStream();
+ baos = new ByteArrayOutputStream(4096);
response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.INTERNAL_SERVER_ERROR);
keepAlive = HttpUtil.isKeepAlive(request);
if (keepAlive) {
@@ -89,7 +89,7 @@
public synchronized PrintWriter writer() {
if (writer == null) {
Charset charset = io.netty.handler.codec.http.HttpUtil.getCharset(response, StandardCharsets.UTF_8);
- writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(baos, charset)));
+ writer = new PrintWriter(new OutputStreamWriter(baos, charset));
}
return writer;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
index b34b9dc..6e4a273 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
@@ -45,7 +45,7 @@
public class HttpUtil {
private static final Logger LOGGER = LogManager.getLogger();
private static final Pattern PARENT_DIR = Pattern.compile("/[^./]+/\\.\\./");
- private static final String DEFAULT_RESPONSE_CHARSET = StandardCharsets.UTF_8.name();
+ private static final Charset DEFAULT_RESPONSE_CHARSET = StandardCharsets.UTF_8;
private HttpUtil() {
}
@@ -100,15 +100,20 @@
return contentType == null ? null : contentType.split(";")[0];
}
- public static String getRequestBody(IServletRequest request) {
- FullHttpRequest httpRequest = request.getHttpRequest();
- Charset charset = io.netty.handler.codec.http.HttpUtil.getCharset(httpRequest, StandardCharsets.UTF_8);
- return httpRequest.content().toString(charset);
+ public static Charset getRequestCharset(HttpRequest request) {
+ return io.netty.handler.codec.http.HttpUtil.getCharset(request, StandardCharsets.UTF_8);
}
- public static void setContentType(IServletResponse response, String type, IServletRequest fromRequest)
+ public static String getRequestBody(IServletRequest request) {
+ FullHttpRequest httpRequest = request.getHttpRequest();
+ return httpRequest.content().toString(getRequestCharset(httpRequest));
+ }
+
+ public static Charset setContentType(IServletResponse response, String type, IServletRequest fromRequest)
throws IOException {
- response.setHeader(HttpHeaderNames.CONTENT_TYPE, type + "; charset=" + getPreferredCharset(fromRequest));
+ Charset preferredCharset = getPreferredCharset(fromRequest);
+ response.setHeader(HttpHeaderNames.CONTENT_TYPE, type + "; charset=" + preferredCharset.name());
+ return preferredCharset;
}
public static void setContentType(IServletResponse response, String type, String charset) throws IOException {
@@ -169,25 +174,24 @@
return clusterURL;
}
- public static String getPreferredCharset(IServletRequest request) {
+ public static Charset getPreferredCharset(IServletRequest request) {
return getPreferredCharset(request, DEFAULT_RESPONSE_CHARSET);
}
- public static String getPreferredCharset(IServletRequest request, String defaultCharset) {
+ public static Charset getPreferredCharset(IServletRequest request, Charset defaultCharset) {
String acceptCharset = request.getHeader(HttpHeaderNames.ACCEPT_CHARSET);
if (acceptCharset == null) {
return defaultCharset;
}
// If no "q" parameter is present, the default weight is 1 [https://tools.ietf.org/html/rfc7231#section-5.3.1]
- Optional<String> preferredCharset = Stream.of(StringUtils.split(acceptCharset, ","))
- .map(WeightedHeaderValue::new).sorted().map(WeightedHeaderValue::getValue)
- .map(a -> "*".equals(a) ? defaultCharset : a).filter(value -> {
+ Optional<Charset> preferredCharset = Stream.of(StringUtils.split(acceptCharset, ","))
+ .map(WeightedHeaderValue::new).sorted().map(WeightedHeaderValue::getValueDefaultStar).filter(value -> {
if (!Charset.isSupported(value)) {
LOGGER.info("disregarding unsupported charset '{}'", value);
return false;
}
return true;
- }).findFirst();
+ }).map(Charset::forName).findFirst();
return preferredCharset.orElse(defaultCharset);
}
@@ -216,6 +220,10 @@
return value;
}
+ public String getValueDefaultStar() {
+ return "*".equals(value) ? DEFAULT_RESPONSE_CHARSET.name() : value;
+ }
+
public double getWeight() {
return weight;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpAcceptCharsetTest.java b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpAcceptCharsetTest.java
index a166e52..260da99 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpAcceptCharsetTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpAcceptCharsetTest.java
@@ -61,7 +61,7 @@
@Test
public void testAmbiguous() {
IServletRequest request = withCharset("utf-8;q=.75,utf-16;q=.75,utf-32;q=.5");
- String preferredCharset = HttpUtil.getPreferredCharset(request);
+ String preferredCharset = HttpUtil.getPreferredCharset(request).name();
Assert.assertTrue("ambiguous by weight (got: " + preferredCharset + ")",
preferredCharset.toLowerCase().matches("utf-(8|16)"));
}
@@ -83,7 +83,7 @@
} else if (charsetObject instanceof String) {
return ((String) charsetObject).toLowerCase();
} else if (charsetObject instanceof IServletRequest) {
- return HttpUtil.getPreferredCharset((IServletRequest) charsetObject).toLowerCase();
+ return HttpUtil.getPreferredCharset((IServletRequest) charsetObject).name().toLowerCase();
}
throw new IllegalArgumentException("unknown type: " + charsetObject.getClass());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerEncodingTest.java b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerEncodingTest.java
index c71c5e4..b522122 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerEncodingTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerEncodingTest.java
@@ -27,7 +27,6 @@
import java.util.Collection;
import java.util.List;
import java.util.Set;
-import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.http.HttpEntity;
@@ -44,8 +43,7 @@
import org.apache.hyracks.http.server.HttpServerConfigBuilder;
import org.apache.hyracks.http.server.WebManager;
import org.apache.hyracks.test.http.servlet.CompliantEchoServlet;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import org.apache.hyracks.test.string.EncodingUtils;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -57,8 +55,6 @@
@RunWith(Parameterized.class)
public class HttpServerEncodingTest {
- private static final Logger LOGGER = LogManager.getLogger();
-
private static final int PORT = 9898;
private static final String HOST = "localhost";
private static final String PROTOCOL = "http";
@@ -72,7 +68,7 @@
List<Object[]> tests = new ArrayList<>();
Stream.of("encoding is hard", "中文字符", "لا يوجد ترجمة لكُ", STRING_NEEDS_2_JAVA_CHARS_1,
STRING_NEEDS_2_JAVA_CHARS_2).forEach(input -> {
- Set<Charset> legalCharsets = getLegalCharsetsFor(input);
+ Set<Charset> legalCharsets = EncodingUtils.getLegalCharsetsFor(input);
legalCharsets.forEach(charsetIn -> legalCharsets.forEach(charsetOut -> tests
.add(new Object[] { input + ":" + charsetIn.displayName() + "->" + charsetOut.displayName(),
input, charsetIn, charsetOut })));
@@ -80,19 +76,6 @@
return tests;
}
- private static Set<Charset> getLegalCharsetsFor(String input) {
- return Charset.availableCharsets().values().stream().filter(Charset::canEncode)
- .filter(test -> canEncodeDecode(input, test)).collect(Collectors.toSet());
- }
-
- private static boolean canEncodeDecode(String input, Charset charset) {
- if (input.equals(new String(input.getBytes(charset), charset))) {
- return true;
- }
- LOGGER.info("cannot encode / decode {} with {}", input, charset.displayName());
- return false;
- }
-
@Parameter(0)
public String testName;
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/string/EncodingUtils.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/string/EncodingUtils.java
new file mode 100644
index 0000000..9ba4817
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/string/EncodingUtils.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.test.string;
+
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class EncodingUtils {
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private EncodingUtils() {
+ }
+
+ // duplicated in [asterix-app]TestExecutor.java as transitive dependencies on test-jars are not handled correctly
+ public static boolean canEncodeDecode(String input, Charset charset, boolean quiet) {
+ try {
+ if (input.equals(new String(input.getBytes(charset), charset))) {
+ if (!input.equals(charset.decode(charset.encode(CharBuffer.wrap(input))).toString())) {
+ // workaround for https://bugs.openjdk.java.net/browse/JDK-6392670 and similar
+ if (!quiet) {
+ LOGGER.info("cannot encode / decode {} with {} using CharBuffer.wrap(<String>)", input,
+ charset.displayName());
+ }
+ } else {
+ return true;
+ }
+ }
+ if (!quiet) {
+ LOGGER.info("cannot encode / decode {} with {}", input, charset.displayName());
+ }
+ } catch (Exception e) {
+ if (!quiet) {
+ LOGGER.info("cannot encode / decode {} with {}, got exception ({})", input, charset.displayName(),
+ String.valueOf(e));
+ }
+ }
+ return false;
+ }
+
+ public static Set<Charset> getLegalCharsetsFor(String input) {
+ return Charset.availableCharsets().values().stream().filter(Charset::canEncode)
+ .filter(test -> canEncodeDecode(input, test, false)).collect(Collectors.toSet());
+ }
+}