Merge "Merge commit 'ac5b498' from 'stabilization-f69489' into 'master'"
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index 0362820..ca43084 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -841,10 +841,8 @@
}
SourceLocation sourceLoc = fcall.getSourceLocation();
- AbstractFunctionCallExpression f;
- if ((f = lookupUserDefinedFunction(signature, args, sourceLoc)) == null) {
- f = lookupBuiltinFunction(signature.getName(), signature.getArity(), args, sourceLoc);
- }
+
+ AbstractFunctionCallExpression f = lookupFunction(signature, args, sourceLoc);
if (f == null) {
throw new CompilationException(ErrorCode.UNKNOWN_FUNCTION, sourceLoc,
@@ -878,6 +876,15 @@
return varRef;
}
+ protected AbstractFunctionCallExpression lookupFunction(FunctionSignature signature,
+ List<Mutable<ILogicalExpression>> args, SourceLocation sourceLoc) throws CompilationException {
+ AbstractFunctionCallExpression f;
+ if ((f = lookupUserDefinedFunction(signature, args, sourceLoc)) == null) {
+ f = lookupBuiltinFunction(signature.getName(), signature.getArity(), args, sourceLoc);
+ }
+ return f;
+ }
+
private AbstractFunctionCallExpression lookupUserDefinedFunction(FunctionSignature signature,
List<Mutable<ILogicalExpression>> args, SourceLocation sourceLoc) throws CompilationException {
try {
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 4e4b760..f5921fe 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -40,17 +40,18 @@
import java.net.URISyntaxException;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Optional;
+import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -148,6 +149,7 @@
public static final int TRUNCATE_THRESHOLD = 16384;
public static final Set<String> NON_CANCELLABLE =
Collections.unmodifiableSet(new HashSet<>(Arrays.asList("store", "validate")));
+ private static final int MAX_NON_UTF_8_STATEMENT_SIZE = 64 * 1024;
private final IPollTask plainExecutor = this::executeTestFile;
@@ -162,7 +164,7 @@
private static Map<String, InetSocketAddress> replicationAddress;
private final List<Charset> allCharsets;
- private final List<Charset> charsetsRemaining = new ArrayList<>();
+ private final Queue<Charset> charsetsRemaining = new ArrayDeque<>();
/*
* Instance members
@@ -638,37 +640,20 @@
return response.getEntity().getContent();
}
- private Charset selectCharset(File result) throws IOException {
- // choose an encoding that works for this input
- return selectCharset(FileUtils.readFileToString(result, UTF_8));
+ public synchronized void setAvailableCharsets(Charset... charsets) {
+ allCharsets.clear();
+ allCharsets.addAll(Arrays.asList(charsets));
+ charsetsRemaining.clear();
}
- private Charset selectCharset(String payload) {
- // choose an encoding that works for this input
- return nextCharset(charset -> canEncodeDecode(charset, payload));
- }
-
- public void setAvailableCharsets(Collection<Charset> charsets) {
- synchronized (allCharsets) {
- allCharsets.clear();
- allCharsets.addAll(charsets);
- charsetsRemaining.clear();
- }
- }
-
- public Charset nextCharset(Predicate<Charset> test) {
- synchronized (allCharsets) {
- while (true) {
- for (Iterator<Charset> iter = charsetsRemaining.iterator(); iter.hasNext();) {
- Charset next = iter.next();
- if (test.test(next)) {
- iter.remove();
- return next;
- }
- }
- Collections.shuffle(allCharsets);
- charsetsRemaining.addAll(allCharsets);
+ private synchronized Charset nextCharset() {
+ while (true) {
+ Charset nextCharset = charsetsRemaining.poll();
+ if (nextCharset != null) {
+ return nextCharset;
}
+ Collections.shuffle(allCharsets);
+ charsetsRemaining.addAll(allCharsets);
}
}
@@ -769,12 +754,12 @@
for (Parameter param : upsertParam(otherParams, stmtParam, ParameterTypeEnum.STRING, statement)) {
builder.addParameter(param.getName(), param.getValue());
}
- builder.addParameter(stmtParam, statement);
+ builder.setCharset(statement.length() > MAX_NON_UTF_8_STATEMENT_SIZE ? UTF_8 : nextCharset());
} else {
// this seems pretty bad - we should probably fix the API and not the client
- builder.setEntity(new StringEntity(statement, UTF_8));
+ builder.setEntity(new StringEntity(statement,
+ statement.length() > MAX_NON_UTF_8_STATEMENT_SIZE ? UTF_8 : nextCharset()));
}
- builder.setCharset(UTF_8);
return builder.build();
}
@@ -805,11 +790,12 @@
}
}
try {
- builder.setEntity(new StringEntity(om.writeValueAsString(content), ContentType.APPLICATION_JSON));
+ builder.setEntity(new StringEntity(om.writeValueAsString(content),
+ ContentType.create(ContentType.APPLICATION_JSON.getMimeType(),
+ statement.length() > MAX_NON_UTF_8_STATEMENT_SIZE ? UTF_8 : nextCharset())));
} catch (JsonProcessingException e) {
e.printStackTrace();
}
- builder.setCharset(UTF_8);
return builder.build();
}
@@ -1284,7 +1270,7 @@
URI uri = testFile.getName().endsWith("aql") ? getEndpoint(Servlets.QUERY_AQL)
: getEndpoint(Servlets.QUERY_SERVICE);
boolean isJsonEncoded = isJsonEncoded(extractHttpRequestType(statement));
- Charset responseCharset = expectedResultFile == null ? UTF_8 : selectCharset(expectedResultFile);
+ Charset responseCharset = expectedResultFile == null ? UTF_8 : nextCharset();
InputStream resultStream;
if (DELIVERY_IMMEDIATE.equals(delivery)) {
resultStream = executeQueryService(statement, fmt, uri, params, isJsonEncoded, responseCharset, null,
diff --git a/hyracks-fullstack/hyracks/hyracks-http/pom.xml b/hyracks-fullstack/hyracks/hyracks-http/pom.xml
index a0e426b..bf96e79 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-http/pom.xml
@@ -54,12 +54,10 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FormUrlEncodedRequest.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FormUrlEncodedRequest.java
index 08271a6..42712fd 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FormUrlEncodedRequest.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FormUrlEncodedRequest.java
@@ -18,86 +18,36 @@
*/
package org.apache.hyracks.http.server;
-import java.io.IOException;
+import java.nio.charset.Charset;
import java.net.InetSocketAddress;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
+import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.server.utils.HttpUtil;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.codec.http.QueryStringDecoder;
-import io.netty.handler.codec.http.multipart.Attribute;
-import io.netty.handler.codec.http.multipart.HttpPostRequestDecoder;
-import io.netty.handler.codec.http.multipart.InterfaceHttpData;
-import io.netty.handler.codec.http.multipart.MixedAttribute;
public class FormUrlEncodedRequest extends BaseRequest implements IServletRequest {
- private final List<String> names;
- private final List<String> values;
-
- public static IServletRequest create(ChannelHandlerContext ctx, FullHttpRequest request) throws IOException {
- List<String> names = new ArrayList<>();
- List<String> values = new ArrayList<>();
- HttpPostRequestDecoder decoder = new HttpPostRequestDecoder(request);
- try {
- List<InterfaceHttpData> bodyHttpDatas = decoder.getBodyHttpDatas();
- for (InterfaceHttpData data : bodyHttpDatas) {
- if (data.getHttpDataType().equals(InterfaceHttpData.HttpDataType.Attribute)) {
- Attribute attr = (MixedAttribute) data;
- names.add(data.getName());
- values.add(attr.getValue());
- }
- }
- } finally {
- decoder.destroy();
- }
+ public static IServletRequest create(ChannelHandlerContext ctx, FullHttpRequest request) {
+ Charset charset = HttpUtil.getRequestCharset(request);
+ Map<String, List<String>> parameters = new LinkedHashMap<>();
+ URLEncodedUtils.parse(request.content().toString(charset), charset).forEach(
+ pair -> parameters.computeIfAbsent(pair.getName(), a -> new ArrayList<>()).add(pair.getValue()));
+ new QueryStringDecoder(request.uri()).parameters()
+ .forEach((name, value) -> parameters.computeIfAbsent(name, a -> new ArrayList<>()).addAll(value));
InetSocketAddress remoteAddress = (InetSocketAddress) ctx.channel().remoteAddress();
- return new FormUrlEncodedRequest(request, remoteAddress, new QueryStringDecoder(request.uri()).parameters(),
- names, values);
+ return new FormUrlEncodedRequest(request, remoteAddress, parameters);
}
- protected FormUrlEncodedRequest(FullHttpRequest request, InetSocketAddress remoteAddress,
- Map<String, List<String>> parameters, List<String> names, List<String> values) {
+ private FormUrlEncodedRequest(FullHttpRequest request, InetSocketAddress remoteAddress,
+ Map<String, List<String>> parameters) {
super(request, remoteAddress, parameters);
- this.names = names;
- this.values = values;
- }
-
- @Override
- public String getParameter(CharSequence name) {
- for (int i = 0; i < names.size(); i++) {
- if (name.equals(names.get(i))) {
- return values.get(i);
- }
- }
- return HttpUtil.getParameter(parameters, name);
- }
-
- @Override
- public Set<String> getParameterNames() {
- HashSet<String> paramNames = new HashSet<>();
- paramNames.addAll(parameters.keySet());
- paramNames.addAll(names);
- return Collections.unmodifiableSet(paramNames);
- }
-
- @Override
- public Map<String, String> getParameters() {
- HashMap<String, String> paramMap = new HashMap<>();
- paramMap.putAll(super.getParameters());
- for (int i = 0; i < names.size(); i++) {
- paramMap.put(names.get(i), values.get(i));
- }
-
- return Collections.unmodifiableMap(paramMap);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
index 78b5096..34ca2c4 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
@@ -77,17 +77,7 @@
public static String getParameter(Map<String, List<String>> parameters, CharSequence name) {
List<String> parameter = parameters.get(String.valueOf(name));
- if (parameter == null) {
- return null;
- } else if (parameter.size() == 1) {
- return parameter.get(0);
- } else {
- StringBuilder aString = new StringBuilder(parameter.get(0));
- for (int i = 1; i < parameter.size(); i++) {
- aString.append(",").append(parameter.get(i));
- }
- return aString.toString();
- }
+ return parameter == null ? null : String.join(",", parameter);
}
public static IServletRequest toServletRequest(ChannelHandlerContext ctx, FullHttpRequest request)