Tests For HTTP APIs With Variable Results
Change-Id: I42ae3c974aac89ceced73b17ace4cba2daa97dc0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1170
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/APIQueries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/APIQueries.xml
index 753554c..5f60bce 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/APIQueries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/APIQueries.xml
@@ -43,4 +43,19 @@
<output-dir compare="Text">cluster_state_cc_1</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="cluster_state_cc_stats_1">
+ <output-dir compare="Text">cluster_state_cc_stats_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="cluster_state_nc_threaddump_1">
+ <output-dir compare="Text">cluster_state_nc_threaddump_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
+ <compilation-unit name="version_1">
+ <output-dir compare="Text">version_1</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/cluster_state_cc_stats_1/cluster_state_cc_stats_1.1.cstate.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/cluster_state_cc_stats_1/cluster_state_cc_stats_1.1.cstate.aql
new file mode 100644
index 0000000..1c6bab0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/cluster_state_cc_stats_1/cluster_state_cc_stats_1.1.cstate.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : cluster_state_cc_stats_1
+ * Description : cluster cc stas
+ * Expected Result : Positive
+ * Date : 8th September 2016
+ */
+/cc/stats
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/cluster_state_nc_threaddump_1/cluster_state_nc_threaddump_1.1.cstate.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/cluster_state_nc_threaddump_1/cluster_state_nc_threaddump_1.1.cstate.aql
new file mode 100644
index 0000000..b1aa5d3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/cluster_state_nc_threaddump_1/cluster_state_nc_threaddump_1.1.cstate.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : cluster_state_nc_threaddump_1
+ * Description : NC threaddump
+ * Expected Result : Positive
+ * Date : 8th September 2016
+ */
+/node/asterix_nc2/threaddump
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/version_1/version_1.1.version.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/version_1/version_1.1.version.aql
new file mode 100644
index 0000000..6a75528
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/version_1/version_1.1.version.aql
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : version_1
+ * Description : version api
+ * Expected Result : Success
+ * Date : 8th September 2016
+ */
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_cc_stats_1/cluster_state_cc_stats_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_cc_stats_1/cluster_state_cc_stats_1.1.regexadm
new file mode 100644
index 0000000..196a12b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_cc_stats_1/cluster_state_cc_stats_1.1.regexadm
@@ -0,0 +1,27 @@
+\{
+ "date": ".*",
+ "gcs": \[
+ \{
+ "collection-count": .*,
+ "collection-time": .*,
+ "name": ".*"
+ \},
+ \{
+ "collection-count": .*,
+ "collection-time": .*,
+ "name": ".*"
+ \}
+ \],
+ "heap_committed_size": [0-9]*,
+ "heap_init_size": [0-9]*,
+ "heap_max_size": [0-9]*,
+ "heap_used_size": [0-9]*,
+ "nonheap_committed_size": [0-9]*,
+ "nonheap_init_size": [0-9]*,
+ "nonheap_max_size": -?[0-9]*,
+ "nonheap_used_size": [0-9]*,
+ "peak_thread_count": [0-9]*,
+ "started_thread_count": [0-9]*,
+ "system_load_average": [0-9\.]*,
+ "thread_count": [0-9]*
+\}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_nc_threaddump_1/cluster_state_cc_threaddump_1.1.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_nc_threaddump_1/cluster_state_cc_threaddump_1.1.regex
new file mode 100644
index 0000000..1e5c47b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_nc_threaddump_1/cluster_state_cc_threaddump_1.1.regex
@@ -0,0 +1,10 @@
+/"date": ".*"/
+/"threads": \[/
+/"id": [0-9]+/
+/"lock_name": ".*"/
+/"name": ".*"/
+/"stack": \[/
+/"java.lang.Thread.run\(Thread.java:[0-9]+\)"/
+/"state": "TIMED_WAITING"/
+/"state": "RUNNABLE"/
+/"state": "WAITING"/
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/version_1/version_1.1.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/version_1/version_1.1.regex
new file mode 100644
index 0000000..f6dbe93
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/version_1/version_1.1.regex
@@ -0,0 +1 @@
+/"git.build.version"/
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
index 08cad0e..d0594d2 100644
--- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
@@ -39,6 +39,7 @@
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.asterix.common.config.GlobalConfig;
@@ -72,8 +73,9 @@
// see
// https://stackoverflow.com/questions/417142/what-is-the-maximum-length-of-a-url-in-different-browsers/417184
private static final long MAX_URL_LENGTH = 2000l;
- private static final Pattern JAVA_BLOCK_COMMENT_PATTERN = Pattern.compile("/\\*.*\\*/",
- Pattern.MULTILINE | Pattern.DOTALL);
+ private static final Pattern JAVA_BLOCK_COMMENT_PATTERN =
+ Pattern.compile("/\\*.*\\*/", Pattern.MULTILINE | Pattern.DOTALL);
+ private static final Pattern REGEX_LINES_PATTERN = Pattern.compile("^(-)?/(.*)/([im]*)$");
private static Method managixExecuteMethod = null;
private static final HashMap<Integer, ITestServer> runningTestServers = new HashMap<>();
@@ -118,6 +120,14 @@
new InputStreamReader(new FileInputStream(expectedFile), "UTF-8"));
BufferedReader readerActual = new BufferedReader(
new InputStreamReader(new FileInputStream(actualFile), "UTF-8"));
+ boolean regex = false;
+ if (actualFile.toString().endsWith(".regex")) {
+ runScriptAndCompareWithResultRegex(scriptFile, expectedFile, actualFile);
+ return;
+ } else if (actualFile.toString().endsWith(".regexadm")) {
+ regex = true;
+ }
+
String lineExpected, lineActual;
int num = 1;
try {
@@ -139,7 +149,7 @@
throw new Exception("Result for " + scriptFile + " changed at line " + num + ":\n< " + lineExpected
+ "\n> " + lineActual);
}
- if (!equalStrings(lineSplitsExpected[0], lineSplitsActual[0])) {
+ if (!equalStrings(lineSplitsExpected[0], lineSplitsActual[0], regex)) {
throw new Exception("Result for " + scriptFile + " changed at line " + num + ":\n< " + lineExpected
+ "\n> " + lineActual);
}
@@ -157,7 +167,7 @@
// (for metadata tests)
continue;
}
- if (!equalStrings(splitsByCommaExpected[j], splitsByCommaActual[j])) {
+ if (!equalStrings(splitsByCommaExpected[j], splitsByCommaActual[j], regex)) {
throw new Exception("Result for " + scriptFile + " changed at line " + num + ":\n< "
+ lineExpected + "\n> " + lineActual);
}
@@ -180,54 +190,80 @@
}
- private boolean equalStrings(String s1, String s2) {
- String[] rowsOne = s1.split("\n");
- String[] rowsTwo = s2.split("\n");
+ private boolean equalStrings(String expected, String actual, boolean regexMatch) {
+ String[] rowsExpected = expected.split("\n");
+ String[] rowsActual = actual.split("\n");
- for (int i = 0; i < rowsOne.length; i++) {
- String row1 = rowsOne[i];
- String row2 = rowsTwo[i];
+ for (int i = 0; i < rowsExpected.length; i++) {
+ String expectedRow = rowsExpected[i];
+ String actualRow = rowsActual[i];
- if (row1.equals(row2)) {
+ if (regexMatch) {
+ if (actualRow.matches(expectedRow)) {
+ continue;
+ }
+ } else if (actualRow.equals(expectedRow)) {
continue;
}
- String[] fields1 = row1.split(" ");
- String[] fields2 = row2.split(" ");
+ String[] expectedFields = expectedRow.split(" ");
+ String[] actualFields = actualRow.split(" ");
boolean bagEncountered = false;
- Set<String> bagElements1 = new HashSet<String>();
- Set<String> bagElements2 = new HashSet<String>();
+ Set<String> expectedBagElements = new HashSet<>();
+ Set<String> actualBagElements = new HashSet<>();
- for (int j = 0; j < fields1.length; j++) {
- if (j >= fields2.length) {
+ for (int j = 0; j < expectedFields.length; j++) {
+ if (j >= actualFields.length) {
return false;
- } else if (fields1[j].equals(fields2[j])) {
- bagEncountered = fields1[j].equals("{{");
- if (fields1[j].startsWith("}}")) {
- if (!bagElements1.equals(bagElements2)) {
+ } else if (expectedFields[j].equals(actualFields[j])) {
+ bagEncountered = expectedFields[j].equals("{{");
+ if (expectedFields[j].startsWith("}}")) {
+ if (regexMatch) {
+ if (expectedBagElements.size() != actualBagElements.size()) {
+ return false;
+ }
+ int [] expectedHits = new int [expectedBagElements.size()];
+ int [] actualHits = new int [actualBagElements.size()];
+ int k = 0;
+ for (String expectedElement : expectedBagElements) {
+ int l = 0;
+ for (String actualElement : actualBagElements) {
+ if (actualElement.matches(expectedElement)) {
+ expectedHits[k]++;
+ actualHits[l]++;
+ }
+ l++;
+ }
+ k++;
+ }
+ for (int m = 0; m < expectedHits.length; m++) {
+ if (expectedHits[m] == 0 || actualHits[m] == 0) {
+ return false;
+ }
+ }
+ } else if (!expectedBagElements.equals(actualBagElements)) {
return false;
}
bagEncountered = false;
- bagElements1.clear();
- bagElements2.clear();
+ expectedBagElements.clear();
+ actualBagElements.clear();
}
- continue;
- } else if (fields1[j].indexOf('.') < 0) {
+ } else if (expectedFields[j].indexOf('.') < 0) {
if (bagEncountered) {
- bagElements1.add(fields1[j].replaceAll(",$", ""));
- bagElements2.add(fields2[j].replaceAll(",$", ""));
+ expectedBagElements.add(expectedFields[j].replaceAll(",$", ""));
+ actualBagElements.add(actualFields[j].replaceAll(",$", ""));
continue;
}
return false;
} else {
// If the fields are floating-point numbers, test them
// for equality safely
- fields1[j] = fields1[j].split(",")[0];
- fields2[j] = fields2[j].split(",")[0];
+ expectedFields[j] = expectedFields[j].split(",")[0];
+ actualFields[j] = actualFields[j].split(",")[0];
try {
- Double double1 = Double.parseDouble(fields1[j]);
- Double double2 = Double.parseDouble(fields2[j]);
+ Double double1 = Double.parseDouble(expectedFields[j]);
+ Double double2 = Double.parseDouble(actualFields[j]);
float float1 = (float) double1.doubleValue();
float float2 = (float) double2.doubleValue();
@@ -246,6 +282,56 @@
return true;
}
+ public void runScriptAndCompareWithResultRegex(File scriptFile, File expectedFile, File actualFile)
+ throws Exception {
+ System.err.println("Expected results file: " + expectedFile.toString());
+ String lineExpected, lineActual;
+ int num = 1;
+ try (BufferedReader readerExpected = new BufferedReader(
+ new InputStreamReader(new FileInputStream(expectedFile), "UTF-8"));
+ BufferedReader readerActual = new BufferedReader(
+ new InputStreamReader(new FileInputStream(actualFile), "UTF-8")))
+ {
+ StringBuilder actual = new StringBuilder();
+ while ((lineActual = readerActual.readLine()) != null) {
+ actual.append(lineActual).append('\n');
+ }
+ while ((lineExpected = readerExpected.readLine()) != null) {
+ if ("".equals(lineExpected.trim())) {
+ continue;
+ }
+ Matcher m = REGEX_LINES_PATTERN.matcher(lineExpected);
+ if (!m.matches()) {
+ throw new IllegalArgumentException("Each line of regex file must conform to: [-]/regex/[flags]: "
+ + expectedFile);
+ }
+ String negateStr = m.group(1);
+ String expression = m.group(2);
+ String flagStr = m.group(3);
+ boolean negate = "-".equals(negateStr);
+ int flags = Pattern.MULTILINE;
+ if (flagStr.contains("m")) {
+ flags |= Pattern.DOTALL;
+ }
+ if (flagStr.contains("i")) {
+ flags |= Pattern.CASE_INSENSITIVE;
+ }
+ Pattern linePattern = Pattern.compile(expression, flags);
+ boolean match = linePattern.matcher(actual).find();
+ if (match && !negate || negate && !match) {
+ continue;
+ }
+ throw new Exception("Result for " + scriptFile + ": expected pattern '" + expression +
+ "' not found in result.");
+ }
+ } catch (Exception e) {
+ System.err.println("Actual results file: " + actualFile.toString());
+ throw e;
+ }
+
+ }
+
+
// For tests where you simply want the byte-for-byte output.
private static void writeOutputToFile(File actualFile, InputStream resultStream) throws Exception {
try (FileOutputStream out = new FileOutputStream(actualFile)) {
@@ -406,8 +492,7 @@
String theHandle = IOUtils.toString(resultStream, "UTF-8");
// take the handle and parse it so results can be retrieved
- InputStream handleResult = getHandleResult(theHandle, fmt);
- return handleResult;
+ return getHandleResult(theHandle, fmt);
}
private InputStream getHandleResult(String handle, OutputFormat fmt) throws Exception {
@@ -494,8 +579,7 @@
int beginIndex = queryPath.lastIndexOf(targetWord) + targetWordSize;
int endIndex = queryPath.lastIndexOf(File.separator);
String prefix = queryPath.substring(beginIndex, endIndex);
- String scriptPath = scriptBasePath + prefix + File.separator + scriptFileName;
- return scriptPath;
+ return scriptBasePath + prefix + File.separator + scriptFileName;
}
private static String getProcessOutput(Process p) throws Exception {
@@ -699,6 +783,17 @@
actualResultFile);
queryCount.increment();
break;
+ case "version": // version servlet
+ fmt = OutputFormat.forCompilationUnit(cUnit);
+ resultStream = executeClusterStateQuery(fmt, getEndpoint(Servlets.VERSION));
+ expectedResultFile = expectedResultFileCtxs.get(queryCount.intValue()).getFile();
+ actualResultFile = testCaseCtx.getActualResultFile(cUnit, expectedResultFile, new File(actualPath));
+ actualResultFile.getParentFile().mkdirs();
+ writeOutputToFile(actualResultFile, resultStream);
+ runScriptAndCompareWithResult(testFile, new PrintWriter(System.err), expectedResultFile,
+ actualResultFile);
+ queryCount.increment();
+ break;
case "server": // (start <test server name> <port>
// [<arg1>][<arg2>][<arg3>]...|stop (<port>|all))
try {