[NO ISSUE][TEST] Make test framework generic with respect to endpoint

Refactoring of test framework to configure query parameters, charset, and endpoint

Change-Id: Iaeb217a0cc49e839d7a0c6cbae90d7eb964c3322
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3515
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>
Contrib: Till Westmann <tillw@apache.org>
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
index a34a29c..36a06c0 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
@@ -58,8 +58,9 @@
                 ? upsertParam(params, "client_context_id", ParameterTypeEnum.STRING, clientContextId) : params;
         Callable<InputStream> query = () -> {
             try {
-                return CancellationTestExecutor.super.executeQueryService(str, fmt, uri, newParams, jsonEncoded,
-                        responseCharset, responseCodeValidator, true);
+                return CancellationTestExecutor.super.executeQueryService(str, fmt, uri,
+                        constructQueryParameters(str, fmt, newParams), jsonEncoded, responseCharset,
+                        responseCodeValidator, true);
             } catch (Exception e) {
                 e.printStackTrace();
                 throw e;
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index eb71bb6..af82754 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -85,6 +85,7 @@
 import org.apache.asterix.testframework.xml.TestCase.CompilationUnit.Parameter;
 import org.apache.asterix.testframework.xml.TestGroup;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.output.ByteArrayOutputStream;
 import org.apache.commons.lang3.StringUtils;
@@ -612,18 +613,19 @@
 
     public InputStream executeQueryService(String str, OutputFormat fmt, URI uri, List<Parameter> params,
             boolean jsonEncoded, Charset responseCharset) throws Exception {
-        return executeQueryService(str, fmt, uri, params, jsonEncoded, responseCharset, null, false);
+        return executeQueryService(str, fmt, uri, constructQueryParameters(str, fmt, params), jsonEncoded,
+                responseCharset, null, false);
     }
 
     public InputStream executeQueryService(String str, OutputFormat fmt, URI uri, List<Parameter> params,
             boolean jsonEncoded, Predicate<Integer> responseCodeValidator) throws Exception {
-        return executeQueryService(str, fmt, uri, params, jsonEncoded, UTF_8, responseCodeValidator, false);
+        return executeQueryService(str, fmt, uri, constructQueryParameters(str, fmt, params), jsonEncoded, UTF_8,
+                responseCodeValidator, false);
     }
 
-    public InputStream executeQueryService(String str, OutputFormat fmt, URI uri, List<Parameter> params,
-            boolean jsonEncoded, Charset responseCharset, Predicate<Integer> responseCodeValidator, boolean cancellable)
-            throws Exception {
+    public List<Parameter> constructQueryParameters(String str, OutputFormat fmt, List<Parameter> params) {
         List<Parameter> newParams = upsertParam(params, "format", ParameterTypeEnum.STRING, fmt.mimeType());
+
         newParams = upsertParam(newParams, QueryServiceServlet.Parameter.PLAN_FORMAT.str(), ParameterTypeEnum.STRING,
                 DEFAULT_PLAN_FORMAT);
         final Optional<String> maxReadsOptional = extractMaxResultReads(str);
@@ -631,12 +633,20 @@
             newParams = upsertParam(newParams, QueryServiceServlet.Parameter.MAX_RESULT_READS.str(),
                     ParameterTypeEnum.STRING, maxReadsOptional.get());
         }
+        return newParams;
+    }
+
+    public InputStream executeQueryService(String str, OutputFormat fmt, URI uri, List<Parameter> params,
+            boolean jsonEncoded, Charset responseCharset, Predicate<Integer> responseCodeValidator, boolean cancellable)
+            throws Exception {
+
         final List<Parameter> additionalParams = extractParameters(str);
         for (Parameter param : additionalParams) {
-            newParams = upsertParam(newParams, param.getName(), param.getType(), param.getValue());
+            params = upsertParam(params, param.getName(), param.getType(), param.getValue());
         }
-        HttpUriRequest method = jsonEncoded ? constructPostMethodJson(str, uri, "statement", newParams)
-                : constructPostMethodUrl(str, uri, "statement", newParams);
+
+        HttpUriRequest method = jsonEncoded ? constructPostMethodJson(str, uri, "statement", params)
+                : constructPostMethodUrl(str, uri, "statement", params);
         // Set accepted output response type
         method.setHeader("Origin", uri.getScheme() + uri.getAuthority());
         method.setHeader("Accept", OutputFormat.CLEAN_JSON.mimeType());
@@ -644,6 +654,7 @@
         if (!responseCharset.equals(UTF_8)) {
             LOGGER.info("using Accept-Charset: {}", responseCharset.name());
         }
+
         HttpResponse response = executeHttpRequest(method);
         if (responseCodeValidator != null) {
             checkResponse(response, responseCodeValidator);
@@ -651,6 +662,13 @@
         return response.getEntity().getContent();
     }
 
+    public InputStream executeQueryService(String str, TestFileContext ctx, OutputFormat fmt, URI uri,
+            List<Parameter> params, boolean jsonEncoded, Charset responseCharset,
+            Predicate<Integer> responseCodeValidator, boolean cancellable) throws Exception {
+        return executeQueryService(str, fmt, uri, constructQueryParameters(str, fmt, params), jsonEncoded,
+                responseCharset, responseCodeValidator, cancellable);
+    }
+
     public synchronized void setAvailableCharsets(Charset... charsets) {
         allCharsets.clear();
         allCharsets.addAll(Arrays.asList(charsets));
@@ -759,6 +777,10 @@
         return builder.build();
     }
 
+    protected void setCharset(RequestBuilder builder, int stmtLength) {
+        builder.setCharset(stmtLength > MAX_NON_UTF_8_STATEMENT_SIZE ? UTF_8 : nextCharset());
+    }
+
     protected HttpUriRequest constructPostMethodUrl(String statement, URI uri, String stmtParam,
             List<Parameter> otherParams) {
         Objects.requireNonNull(stmtParam, "statement parameter required");
@@ -766,7 +788,7 @@
         for (Parameter param : upsertParam(otherParams, stmtParam, ParameterTypeEnum.STRING, statement)) {
             builder.addParameter(param.getName(), param.getValue());
         }
-        builder.setCharset(statement.length() > MAX_NON_UTF_8_STATEMENT_SIZE ? UTF_8 : nextCharset());
+        setCharset(builder, statement.length());
         return builder.build();
     }
 
@@ -967,8 +989,9 @@
                 File actualResultFile = expectedResultFile == null ? null
                         : testCaseCtx.getActualResultFile(cUnit, expectedResultFile, new File(actualPath));
                 ExtractedResult extractedResult = executeQuery(OutputFormat.forCompilationUnit(cUnit), statement,
-                        variableCtx, ctx.getType(), testFile, expectedResultFile, actualResultFile, queryCount,
+                        variableCtx, ctx, expectedResultFile, actualResultFile, queryCount,
                         expectedResultFileCtxs.size(), cUnit.getParameter(), ComparisonEnum.TEXT);
+
                 if (testCaseCtx.getTestCase().isCheckWarnings()) {
                     boolean expectedSourceLoc = testCaseCtx.isSourceLocationExpected(cUnit);
                     validateWarnings(extractedResult.getWarnings(), cUnit.getExpectedWarn(), actualWarnCount,
@@ -983,8 +1006,8 @@
                 }
                 actualResultFile = new File(actualPath, testCaseCtx.getTestCase().getFilePath() + File.separatorChar
                         + cUnit.getName() + '.' + ctx.getSeqNum() + ".adm");
-                executeQuery(OutputFormat.forCompilationUnit(cUnit), statement, variableCtx, ctx.getType(), testFile,
-                        null, actualResultFile, queryCount, expectedResultFileCtxs.size(), cUnit.getParameter(),
+                executeQuery(OutputFormat.forCompilationUnit(cUnit), statement, variableCtx, ctx, null,
+                        actualResultFile, queryCount, expectedResultFileCtxs.size(), cUnit.getParameter(),
                         ComparisonEnum.TEXT);
                 variableCtx.put(key, actualResultFile);
                 break;
@@ -1225,9 +1248,9 @@
         }
         File actualResultFile = new File(actualPath, testCaseCtx.getTestCase().getFilePath() + File.separatorChar
                 + cUnit.getName() + '.' + ctx.getSeqNum() + ".adm");
-        executeQuery(OutputFormat.forCompilationUnit(cUnit), statement, variableCtx, "validate", testFile,
-                expectedResultFile, actualResultFile, queryCount, expectedResultFileCtxs.size(), cUnit.getParameter(),
-                ComparisonEnum.TEXT);
+        executeQuery(OutputFormat.forCompilationUnit(cUnit), statement, variableCtx,
+                new TestFileContext(testFile, "validate"), expectedResultFile, actualResultFile, queryCount,
+                expectedResultFileCtxs.size(), cUnit.getParameter(), ComparisonEnum.TEXT);
     }
 
     protected void executeHttpRequest(OutputFormat fmt, String statement, Map<String, Object> variableCtx,
@@ -1278,23 +1301,30 @@
         queryCount.increment();
     }
 
+    protected Charset getResponseCharset(File expectedResultFile) {
+        return expectedResultFile == null ? UTF_8 : nextCharset();
+    }
+
     public ExtractedResult executeQuery(OutputFormat fmt, String statement, Map<String, Object> variableCtx,
-            String reqType, File testFile, File expectedResultFile, File actualResultFile, MutableInt queryCount,
-            int numResultFiles, List<Parameter> params, ComparisonEnum compare) throws Exception {
+            TestFileContext ctx, File expectedResultFile, File actualResultFile, MutableInt queryCount,
+            int numResultFiles, List<Parameter> params, ComparisonEnum compare, URI uri) throws Exception {
+
         String delivery = DELIVERY_IMMEDIATE;
+        String reqType = ctx.getType();
+        File testFile = ctx.getFile();
         if (reqType.equalsIgnoreCase("async")) {
             delivery = DELIVERY_ASYNC;
         } else if (reqType.equalsIgnoreCase("deferred")) {
             delivery = DELIVERY_DEFERRED;
         }
-        URI uri = testFile.getName().endsWith("aql") ? getEndpoint(Servlets.QUERY_AQL)
-                : getEndpoint(Servlets.QUERY_SERVICE);
+
         boolean isJsonEncoded = isJsonEncoded(extractHttpRequestType(statement));
-        Charset responseCharset = expectedResultFile == null ? UTF_8 : nextCharset();
+
+        Charset responseCharset = getResponseCharset(expectedResultFile);
         InputStream resultStream;
         ExtractedResult extractedResult = null;
         if (DELIVERY_IMMEDIATE.equals(delivery)) {
-            resultStream = executeQueryService(statement, fmt, uri, params, isJsonEncoded, responseCharset, null,
+            resultStream = executeQueryService(statement, ctx, fmt, uri, params, isJsonEncoded, responseCharset, null,
                     isCancellable(reqType));
             switch (reqType) {
                 case METRICS_QUERY_TYPE:
@@ -1345,6 +1375,16 @@
         return extractedResult;
     }
 
+    public ExtractedResult executeQuery(OutputFormat fmt, String statement, Map<String, Object> variableCtx,
+            TestFileContext ctx, File expectedResultFile, File actualResultFile, MutableInt queryCount,
+            int numResultFiles, List<Parameter> params, ComparisonEnum compare) throws Exception {
+        URI uri = getEndpoint(ctx.getFile().getName().endsWith("aql") ? Servlets.QUERY_AQL : Servlets.QUERY_SERVICE,
+                FilenameUtils.getExtension(ctx.getFile().getName()));
+        return executeQuery(fmt, statement, variableCtx, ctx, expectedResultFile, actualResultFile, queryCount,
+                numResultFiles, params, compare, uri);
+
+    }
+
     private void polldynamic(TestCaseContext testCaseCtx, TestFileContext ctx, Map<String, Object> variableCtx,
             String statement, boolean isDmlRecoveryTest, ProcessBuilder pb, CompilationUnit cUnit,
             MutableInt queryCount, List<TestFileContext> expectedResultFileCtxs, File testFile, String actualPath,
@@ -1870,6 +1910,10 @@
         return createEndpointURI(Servlets.getAbsolutePath(getPath(servlet)), null);
     }
 
+    public URI getEndpoint(String servlet, String extension) throws URISyntaxException {
+        return createEndpointURI(Servlets.getAbsolutePath(getPath(servlet)), null);
+    }
+
     public static String stripJavaComments(String text) {
         return JAVA_BLOCK_COMMENT_PATTERN.matcher(text).replaceAll("");
     }
@@ -2110,7 +2154,7 @@
             throws Exception {
         final URI uri = getQueryServiceUri(testFile);
         final InputStream inputStream = executeQueryService(statement, OutputFormat.forCompilationUnit(cUnit), uri,
-                cUnit.getParameter(), true, responseCharset, null, false);
+                cUnit.getParameter(), true, responseCharset);
         return ResultExtractor.extract(inputStream, responseCharset).getResult();
     }
 
diff --git a/asterixdb/asterix-test-framework/src/main/java/org/apache/asterix/testframework/context/TestFileContext.java b/asterixdb/asterix-test-framework/src/main/java/org/apache/asterix/testframework/context/TestFileContext.java
index 3ae6dcd..46aa9fd 100644
--- a/asterixdb/asterix-test-framework/src/main/java/org/apache/asterix/testframework/context/TestFileContext.java
+++ b/asterixdb/asterix-test-framework/src/main/java/org/apache/asterix/testframework/context/TestFileContext.java
@@ -31,6 +31,11 @@
         this.file = file;
     }
 
+    public TestFileContext(File file, String type) {
+        this.file = file;
+        this.type = type;
+    }
+
     public File getFile() {
         return file;
     }