Merge branch 'gerrit/mad-hatter' into 'master'
Change-Id: Ie4f5abbf708915146ca11f2074ba05bfae753626
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
index 60d44a1..5e53d54 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
@@ -86,6 +86,12 @@
private static final Logger LOGGER = LogManager.getLogger();
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ public static ExtractedResult extract(InputStream resultStream, Charset resultCharset, String outputFormat)
+ throws Exception {
+ return extract(resultStream, EnumSet.of(ResultField.RESULTS, ResultField.WARNINGS), resultCharset,
+ outputFormat);
+ }
+
public static ExtractedResult extract(InputStream resultStream, Charset resultCharset) throws Exception {
return extract(resultStream, EnumSet.of(ResultField.RESULTS, ResultField.WARNINGS), resultCharset);
}
@@ -120,6 +126,25 @@
private static ExtractedResult extract(InputStream resultStream, EnumSet<ResultField> resultFields,
Charset resultCharset) throws Exception {
+ return extract(resultStream, resultFields, resultCharset, "jsonl"); //default output format type is jsonl
+ }
+
+ private static ExtractedResult extract(InputStream resultStream, EnumSet<ResultField> resultFields,
+ Charset resultCharset, String fmt) throws Exception {
+
+ if (fmt.equals("json")) {
+ return extract(resultStream, resultFields, resultCharset, "[", ",", "]");
+ }
+
+ if (fmt.equals("jsonl")) {
+ return extract(resultStream, resultFields, resultCharset, "", "", "");
+ }
+
+ throw new AsterixException("Unkown output format for result of test query");
+ }
+
+ private static ExtractedResult extract(InputStream resultStream, EnumSet<ResultField> resultFields,
+ Charset resultCharset, String openMarker, String separator, String closeMarker) throws Exception {
ExtractedResult extractedResult = new ExtractedResult();
final String resultStr = IOUtils.toString(resultStream, resultCharset);
final ObjectNode result = OBJECT_MAPPER.readValue(resultStr, ObjectNode.class);
@@ -158,7 +183,10 @@
} else {
JsonNode[] fields = Iterators.toArray(fieldValue.elements(), JsonNode.class);
if (fields.length > 1) {
+ String sep = openMarker;
for (JsonNode f : fields) {
+ resultBuilder.append(sep);
+ sep = separator;
if (f.isObject()) {
resultBuilder.append(OBJECT_MAPPER.writeValueAsString(f));
@@ -166,6 +194,7 @@
resultBuilder.append(f.asText());
}
}
+ resultBuilder.append(closeMarker);
}
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 789cdbc..8dc04eb 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -64,7 +64,6 @@
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.asterix.api.http.server.QueryServiceServlet;
@@ -139,6 +138,9 @@
Pattern.compile("polltimeoutsecs=(\\d+)(\\D|$)", Pattern.MULTILINE);
private static final Pattern POLL_DELAY_PATTERN = Pattern.compile("polldelaysecs=(\\d+)(\\D|$)", Pattern.MULTILINE);
private static final Pattern HANDLE_VARIABLE_PATTERN = Pattern.compile("handlevariable=(\\w+)");
+ private static final Pattern RESULT_VARIABLE_PATTERN = Pattern.compile("resultvariable=(\\w+)");
+ private static final Pattern OUTPUTFORMAT_VARIABLE_PATTERN = Pattern.compile("outputformat=(\\w+)");
+
private static final Pattern VARIABLE_REF_PATTERN = Pattern.compile("\\$(\\w+)");
private static final Pattern HTTP_PARAM_PATTERN =
Pattern.compile("param (?<name>[\\w-$]+)(?::(?<type>\\w+))?=(?<value>.*)", Pattern.MULTILINE);
@@ -173,7 +175,7 @@
private static List<InetSocketAddress> ncEndPointsList = new ArrayList<>();
private static Map<String, InetSocketAddress> replicationAddress;
- private final List<Charset> allCharsets;
+ private List<Charset> allCharsets;
private final Queue<Charset> charsetsRemaining = new ArrayDeque<>();
/*
@@ -199,10 +201,7 @@
public TestExecutor(List<InetSocketAddress> endpoints) {
this.endpoints = endpoints;
- this.allCharsets = Stream
- .of("UTF-8", "UTF-16", "UTF-16BE", "UTF-16LE", "UTF-32", "UTF-32BE", "UTF-32LE", "x-UTF-32BE-BOM",
- "x-UTF-32LE-BOM", "x-UTF-16LE-BOM")
- .filter(Charset::isSupported).map(Charset::forName).collect(Collectors.toList());
+ this.allCharsets = Collections.singletonList(UTF_8);
}
public void setLibrarian(IExternalUDFLibrarian librarian) {
@@ -240,6 +239,9 @@
ComparisonEnum compare, Charset actualEncoding) throws Exception {
LOGGER.info("Expected results file: {} ", expectedFile);
boolean regex = false;
+ if (expectedFile.getName().endsWith(".ignore")) {
+ return; //skip the comparison
+ }
try (BufferedReader readerExpected =
new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile), UTF_8));
BufferedReader readerActual =
@@ -259,7 +261,6 @@
ObjectMapper OM = new ObjectMapper();
JsonNode expectedJson = OM.readTree(readerExpected);
JsonNode actualJson = OM.readTree(readerActual);
- System.out.println(OM.writeValueAsString(actualJson));
if (expectedJson == null || actualJson == null) {
throw new NullPointerException("Error parsing expected or actual result file for " + scriptFile);
}
@@ -670,9 +671,12 @@
responseCharset, responseCodeValidator, cancellable);
}
- public synchronized void setAvailableCharsets(Charset... charsets) {
- allCharsets.clear();
- allCharsets.addAll(Arrays.asList(charsets));
+ public void setAvailableCharsets(Charset... charsets) {
+ setAvailableCharsets(Arrays.asList(charsets));
+ }
+
+ public synchronized void setAvailableCharsets(List<Charset> charsets) {
+ allCharsets = charsets;
charsetsRemaining.clear();
}
@@ -1320,13 +1324,14 @@
}
boolean isJsonEncoded = isJsonEncoded(extractHttpRequestType(statement));
-
Charset responseCharset = getResponseCharset(expectedResultFile);
InputStream resultStream;
ExtractedResult extractedResult = null;
+ final String variablesReplaced = replaceVarRefRelaxed(statement, variableCtx);
+ String resultVar = getResultVariable(statement); //Is the result of the statement/query to be used in later tests
if (DELIVERY_IMMEDIATE.equals(delivery)) {
- resultStream = executeQueryService(statement, ctx, fmt, uri, params, isJsonEncoded, responseCharset, null,
- isCancellable(reqType));
+ resultStream = executeQueryService(variablesReplaced, ctx, fmt, uri, params, isJsonEncoded, responseCharset,
+ null, isCancellable(reqType));
switch (reqType) {
case METRICS_QUERY_TYPE:
resultStream = ResultExtractor.extractMetrics(resultStream, responseCharset);
@@ -1338,7 +1343,13 @@
resultStream = ResultExtractor.extractPlans(resultStream, responseCharset);
break;
default:
- extractedResult = ResultExtractor.extract(resultStream, responseCharset);
+ String outputFormatVariable = getOutputFormatVariable(statement);
+ if ((outputFormatVariable == null) || (outputFormatVariable.equals("jsonl"))) {
+ extractedResult = ResultExtractor.extract(resultStream, responseCharset);
+ } else {
+ extractedResult = ResultExtractor.extract(resultStream, responseCharset, "json");
+ }
+
resultStream = extractedResult.getResult();
break;
}
@@ -1358,7 +1369,13 @@
+ ", filectxs.size: " + numResultFiles);
}
} else {
- writeOutputToFile(actualResultFile, resultStream);
+ if (resultVar != null) {
+ String result = IOUtils.toString(resultStream, responseCharset);
+ variableCtx.put(resultVar, result);
+ writeOutputToFile(actualResultFile, new ByteArrayInputStream(result.getBytes(responseCharset)));
+ } else {
+ writeOutputToFile(actualResultFile, resultStream);
+ }
if (expectedResultFile == null) {
if (reqType.equals("store")) {
return extractedResult;
@@ -1586,6 +1603,16 @@
return handleVariableMatcher.find() ? handleVariableMatcher.group(1) : null;
}
+ protected static String getResultVariable(String statement) {
+ final Matcher resultVariableMatcher = RESULT_VARIABLE_PATTERN.matcher(statement);
+ return resultVariableMatcher.find() ? resultVariableMatcher.group(1) : null;
+ }
+
+ protected static String getOutputFormatVariable(String statement) {
+ final Matcher outputFormatVariableMatcher = OUTPUTFORMAT_VARIABLE_PATTERN.matcher(statement);
+ return outputFormatVariableMatcher.find() ? outputFormatVariableMatcher.group(1) : null;
+ }
+
protected static String replaceVarRef(String statement, Map<String, Object> variableCtx) {
String tmpStmt = statement;
Matcher variableReferenceMatcher = VARIABLE_REF_PATTERN.matcher(tmpStmt);
@@ -1599,6 +1626,21 @@
return tmpStmt;
}
+ protected static String replaceVarRefRelaxed(String statement, Map<String, Object> variableCtx) {
+ String tmpStmt = statement;
+ Matcher variableReferenceMatcher = VARIABLE_REF_PATTERN.matcher(tmpStmt);
+ while (variableReferenceMatcher.find()) {
+ String var = variableReferenceMatcher.group(1);
+ Object value = variableCtx.get(var);
+ if (value == null) {
+ continue;
+ }
+ tmpStmt = tmpStmt.replace("$" + var, String.valueOf(value));
+ variableReferenceMatcher = VARIABLE_REF_PATTERN.matcher(tmpStmt);
+ }
+ return tmpStmt;
+ }
+
protected static Optional<String> extractMaxResultReads(String statement) {
final Matcher m = MAX_RESULT_READS_PATTERN.matcher(statement);
while (m.find()) {
diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java
index 88ada80..47e6f80 100644
--- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java
+++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java
@@ -15,10 +15,14 @@
package org.apache.asterix.test.server;
import java.io.File;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.asterix.testframework.context.TestCaseContext;
+import org.junit.BeforeClass;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
@@ -38,6 +42,14 @@
return testArgs;
}
+ @BeforeClass
+ public static void setup() {
+ testExecutor.setAvailableCharsets(Stream
+ .of("UTF-8", "UTF-16", "UTF-16BE", "UTF-16LE", "UTF-32", "UTF-32BE", "UTF-32LE", "x-UTF-32BE-BOM",
+ "x-UTF-32LE-BOM", "x-UTF-16LE-BOM")
+ .filter(Charset::isSupported).map(Charset::forName).collect(Collectors.toList()));
+ }
+
protected static Collection<Object[]> buildTestsInXml(String xmlfile) throws Exception {
Collection<Object[]> testArgs = new ArrayList<Object[]>();
TestCaseContext.Builder b = new TestCaseContext.Builder();
diff --git a/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/FullFrameChannelReadInterface.java b/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/FullFrameChannelReadInterface.java
index 9d7f848..0ce5a87 100644
--- a/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/FullFrameChannelReadInterface.java
+++ b/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/FullFrameChannelReadInterface.java
@@ -47,7 +47,7 @@
credits = 0;
emptyBufferAcceptor = buffer -> {
final int delta = buffer.remaining();
- if (delta != frameSize) {
+ if (bufferFactory != null && delta != frameSize) {
LOGGER.warn("partial frame being recycled; expected size {}, actual size {}", frameSize, delta);
}
synchronized (bufferRecycleLock) {