[NO ISSUE][TEST] Fix extraction of multi-document result in test executor
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Fix extraction/writing within test suite for multiple document result as proper JSON document
Change-Id: I879e045e2ae7f64f662993afb46cad7b29c781b3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/3623
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>
Tested-by: Till Westmann <tillw@apache.org>
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
index 60d44a1..5e53d54 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
@@ -86,6 +86,12 @@
private static final Logger LOGGER = LogManager.getLogger();
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ public static ExtractedResult extract(InputStream resultStream, Charset resultCharset, String outputFormat)
+ throws Exception {
+ return extract(resultStream, EnumSet.of(ResultField.RESULTS, ResultField.WARNINGS), resultCharset,
+ outputFormat);
+ }
+
public static ExtractedResult extract(InputStream resultStream, Charset resultCharset) throws Exception {
return extract(resultStream, EnumSet.of(ResultField.RESULTS, ResultField.WARNINGS), resultCharset);
}
@@ -120,6 +126,25 @@
private static ExtractedResult extract(InputStream resultStream, EnumSet<ResultField> resultFields,
Charset resultCharset) throws Exception {
+ return extract(resultStream, resultFields, resultCharset, "jsonl"); //default output format type is jsonl
+ }
+
+ private static ExtractedResult extract(InputStream resultStream, EnumSet<ResultField> resultFields,
+ Charset resultCharset, String fmt) throws Exception {
+
+ if (fmt.equals("json")) {
+ return extract(resultStream, resultFields, resultCharset, "[", ",", "]");
+ }
+
+ if (fmt.equals("jsonl")) {
+ return extract(resultStream, resultFields, resultCharset, "", "", "");
+ }
+
+ throw new AsterixException("Unkown output format for result of test query");
+ }
+
+ private static ExtractedResult extract(InputStream resultStream, EnumSet<ResultField> resultFields,
+ Charset resultCharset, String openMarker, String separator, String closeMarker) throws Exception {
ExtractedResult extractedResult = new ExtractedResult();
final String resultStr = IOUtils.toString(resultStream, resultCharset);
final ObjectNode result = OBJECT_MAPPER.readValue(resultStr, ObjectNode.class);
@@ -158,7 +183,10 @@
} else {
JsonNode[] fields = Iterators.toArray(fieldValue.elements(), JsonNode.class);
if (fields.length > 1) {
+ String sep = openMarker;
for (JsonNode f : fields) {
+ resultBuilder.append(sep);
+ sep = separator;
if (f.isObject()) {
resultBuilder.append(OBJECT_MAPPER.writeValueAsString(f));
@@ -166,6 +194,7 @@
resultBuilder.append(f.asText());
}
}
+ resultBuilder.append(closeMarker);
}
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 15711d0..8dc04eb 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -138,6 +138,9 @@
Pattern.compile("polltimeoutsecs=(\\d+)(\\D|$)", Pattern.MULTILINE);
private static final Pattern POLL_DELAY_PATTERN = Pattern.compile("polldelaysecs=(\\d+)(\\D|$)", Pattern.MULTILINE);
private static final Pattern HANDLE_VARIABLE_PATTERN = Pattern.compile("handlevariable=(\\w+)");
+ private static final Pattern RESULT_VARIABLE_PATTERN = Pattern.compile("resultvariable=(\\w+)");
+ private static final Pattern OUTPUTFORMAT_VARIABLE_PATTERN = Pattern.compile("outputformat=(\\w+)");
+
private static final Pattern VARIABLE_REF_PATTERN = Pattern.compile("\\$(\\w+)");
private static final Pattern HTTP_PARAM_PATTERN =
Pattern.compile("param (?<name>[\\w-$]+)(?::(?<type>\\w+))?=(?<value>.*)", Pattern.MULTILINE);
@@ -236,6 +239,9 @@
ComparisonEnum compare, Charset actualEncoding) throws Exception {
LOGGER.info("Expected results file: {} ", expectedFile);
boolean regex = false;
+ if (expectedFile.getName().endsWith(".ignore")) {
+ return; //skip the comparison
+ }
try (BufferedReader readerExpected =
new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile), UTF_8));
BufferedReader readerActual =
@@ -1318,13 +1324,14 @@
}
boolean isJsonEncoded = isJsonEncoded(extractHttpRequestType(statement));
-
Charset responseCharset = getResponseCharset(expectedResultFile);
InputStream resultStream;
ExtractedResult extractedResult = null;
+ final String variablesReplaced = replaceVarRefRelaxed(statement, variableCtx);
+ String resultVar = getResultVariable(statement); //Is the result of the statement/query to be used in later tests
if (DELIVERY_IMMEDIATE.equals(delivery)) {
- resultStream = executeQueryService(statement, ctx, fmt, uri, params, isJsonEncoded, responseCharset, null,
- isCancellable(reqType));
+ resultStream = executeQueryService(variablesReplaced, ctx, fmt, uri, params, isJsonEncoded, responseCharset,
+ null, isCancellable(reqType));
switch (reqType) {
case METRICS_QUERY_TYPE:
resultStream = ResultExtractor.extractMetrics(resultStream, responseCharset);
@@ -1336,7 +1343,13 @@
resultStream = ResultExtractor.extractPlans(resultStream, responseCharset);
break;
default:
- extractedResult = ResultExtractor.extract(resultStream, responseCharset);
+ String outputFormatVariable = getOutputFormatVariable(statement);
+ if ((outputFormatVariable == null) || (outputFormatVariable.equals("jsonl"))) {
+ extractedResult = ResultExtractor.extract(resultStream, responseCharset);
+ } else {
+ extractedResult = ResultExtractor.extract(resultStream, responseCharset, "json");
+ }
+
resultStream = extractedResult.getResult();
break;
}
@@ -1356,7 +1369,13 @@
+ ", filectxs.size: " + numResultFiles);
}
} else {
- writeOutputToFile(actualResultFile, resultStream);
+ if (resultVar != null) {
+ String result = IOUtils.toString(resultStream, responseCharset);
+ variableCtx.put(resultVar, result);
+ writeOutputToFile(actualResultFile, new ByteArrayInputStream(result.getBytes(responseCharset)));
+ } else {
+ writeOutputToFile(actualResultFile, resultStream);
+ }
if (expectedResultFile == null) {
if (reqType.equals("store")) {
return extractedResult;
@@ -1584,6 +1603,16 @@
return handleVariableMatcher.find() ? handleVariableMatcher.group(1) : null;
}
+ protected static String getResultVariable(String statement) {
+ final Matcher resultVariableMatcher = RESULT_VARIABLE_PATTERN.matcher(statement);
+ return resultVariableMatcher.find() ? resultVariableMatcher.group(1) : null;
+ }
+
+ protected static String getOutputFormatVariable(String statement) {
+ final Matcher outputFormatVariableMatcher = OUTPUTFORMAT_VARIABLE_PATTERN.matcher(statement);
+ return outputFormatVariableMatcher.find() ? outputFormatVariableMatcher.group(1) : null;
+ }
+
protected static String replaceVarRef(String statement, Map<String, Object> variableCtx) {
String tmpStmt = statement;
Matcher variableReferenceMatcher = VARIABLE_REF_PATTERN.matcher(tmpStmt);
@@ -1597,6 +1626,21 @@
return tmpStmt;
}
+ protected static String replaceVarRefRelaxed(String statement, Map<String, Object> variableCtx) {
+ String tmpStmt = statement;
+ Matcher variableReferenceMatcher = VARIABLE_REF_PATTERN.matcher(tmpStmt);
+ while (variableReferenceMatcher.find()) {
+ String var = variableReferenceMatcher.group(1);
+ Object value = variableCtx.get(var);
+ if (value == null) {
+ continue;
+ }
+ tmpStmt = tmpStmt.replace("$" + var, String.valueOf(value));
+ variableReferenceMatcher = VARIABLE_REF_PATTERN.matcher(tmpStmt);
+ }
+ return tmpStmt;
+ }
+
protected static Optional<String> extractMaxResultReads(String statement) {
final Matcher m = MAX_RESULT_READS_PATTERN.matcher(statement);
while (m.find()) {