[NO ISSUE] Misc utility / cleanup
Change-Id: Iae424fb99220630cb3a240cfef4cbba39b9da06b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11803
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/IPollTask.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/IPollTask.java
index a1ed12b..8ba745b 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/IPollTask.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/IPollTask.java
@@ -19,7 +19,6 @@
package org.apache.asterix.test.common;
import java.io.File;
-import java.util.BitSet;
import java.util.List;
import java.util.Map;
@@ -32,8 +31,7 @@
/**
* Execute the poll task
- *
- * @param testCaseCtx
+ * @param testCaseCtx
* @param ctx
* @param variableCtx
* @param statement
@@ -44,11 +42,9 @@
* @param expectedResultFileCtxs
* @param testFile
* @param actualPath
- * @param expectedWarnings
*/
void execute(TestCaseContext testCaseCtx, TestFileContext ctx, Map<String, Object> variableCtx, String statement,
boolean isDmlRecoveryTest, ProcessBuilder pb, CompilationUnit cUnit, MutableInt queryCount,
- List<TestFileContext> expectedResultFileCtxs, File testFile, String actualPath, BitSet expectedWarnings)
- throws Exception;
+ List<TestFileContext> expectedResultFileCtxs, File testFile, String actualPath) throws Exception;
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 40946d3..f1bf63d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -1143,8 +1143,8 @@
public void executeTestFile(TestCaseContext testCaseCtx, TestFileContext ctx, Map<String, Object> variableCtx,
String statement, boolean isDmlRecoveryTest, ProcessBuilder pb, CompilationUnit cUnit,
- MutableInt queryCount, List<TestFileContext> expectedResultFileCtxs, File testFile, String actualPath,
- BitSet expectedWarnings) throws Exception {
+ MutableInt queryCount, List<TestFileContext> expectedResultFileCtxs, File testFile, String actualPath)
+ throws Exception {
InputStream resultStream;
File qbcFile;
boolean failed = false;
@@ -1152,7 +1152,7 @@
switch (ctx.getType()) {
case "ddl":
ExtractedResult ddlExtractedResult = executeSqlppUpdateOrDdl(statement, OutputFormat.CLEAN_JSON, cUnit);
- validateWarning(ddlExtractedResult, testCaseCtx, cUnit, testFile, expectedWarnings);
+ validateWarning(ddlExtractedResult, testCaseCtx, cUnit, testFile);
break;
case "update":
// isDmlRecoveryTest: set IP address
@@ -1166,11 +1166,11 @@
case "pollpost":
poll(testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit, queryCount,
expectedResultFileCtxs, testFile, actualPath, ctx.getType().substring("poll".length()),
- expectedWarnings, plainExecutor);
+ plainExecutor);
break;
case "polldynamic":
polldynamic(testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit, queryCount,
- expectedResultFileCtxs, testFile, actualPath, expectedWarnings);
+ expectedResultFileCtxs, testFile, actualPath);
break;
case "query":
case "async":
@@ -1195,7 +1195,7 @@
variableCtx, ctx, expectedResultFile, actualResultFile, queryCount,
expectedResultFileCtxs.size(), cUnit.getParameter(), ComparisonEnum.TEXT);
- validateWarning(extractedResult, testCaseCtx, cUnit, testFile, expectedWarnings);
+ validateWarning(extractedResult, testCaseCtx, cUnit, testFile);
break;
case "store":
// This is a query that returns the expected output of a subsequent query
@@ -1610,18 +1610,18 @@
private void polldynamic(TestCaseContext testCaseCtx, TestFileContext ctx, Map<String, Object> variableCtx,
String statement, boolean isDmlRecoveryTest, ProcessBuilder pb, CompilationUnit cUnit,
- MutableInt queryCount, List<TestFileContext> expectedResultFileCtxs, File testFile, String actualPath,
- BitSet expectedWarnings) throws Exception {
+ MutableInt queryCount, List<TestFileContext> expectedResultFileCtxs, File testFile, String actualPath)
+ throws Exception {
IExpectedResultPoller poller = getExpectedResultPoller(statement);
final String key = getKey(statement);
poll(testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit, queryCount, expectedResultFileCtxs,
- testFile, actualPath, "validate", expectedWarnings, new IPollTask() {
+ testFile, actualPath, "validate", new IPollTask() {
@Override
public void execute(TestCaseContext testCaseCtx, TestFileContext ctx,
Map<String, Object> variableCtx, String statement, boolean isDmlRecoveryTest,
ProcessBuilder pb, CompilationUnit cUnit, MutableInt queryCount,
- List<TestFileContext> expectedResultFileCtxs, File testFile, String actualPath,
- BitSet expectedWarnings) throws Exception {
+ List<TestFileContext> expectedResultFileCtxs, File testFile, String actualPath)
+ throws Exception {
File actualResultFile = new File(actualPath, testCaseCtx.getTestCase().getFilePath()
+ File.separatorChar + cUnit.getName() + '.' + ctx.getSeqNum() + ".polled.adm");
if (actualResultFile.exists() && !actualResultFile.delete()) {
@@ -1659,7 +1659,7 @@
private void poll(TestCaseContext testCaseCtx, TestFileContext ctx, Map<String, Object> variableCtx,
String statement, boolean isDmlRecoveryTest, ProcessBuilder pb, CompilationUnit cUnit,
MutableInt queryCount, List<TestFileContext> expectedResultFileCtxs, File testFile, String actualPath,
- String newType, BitSet expectedWarnings, IPollTask pollTask) throws Exception {
+ String newType, IPollTask pollTask) throws Exception {
// polltimeoutsecs=nnn, polldelaysecs=nnn
int timeoutSecs = getTimeoutSecs(statement);
int retryDelaySecs = getRetryDelaySecs(statement);
@@ -1682,7 +1682,7 @@
try {
startSemaphore.release();
pollTask.execute(testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit,
- queryCount, expectedResultFileCtxs, testFile, actualPath, expectedWarnings);
+ queryCount, expectedResultFileCtxs, testFile, actualPath);
} finally {
endSemaphore.release();
}
@@ -2092,13 +2092,13 @@
public void executeTest(String actualPath, TestCaseContext testCaseCtx, ProcessBuilder pb,
boolean isDmlRecoveryTest, TestGroup failedGroup, TestGroup passedGroup) throws Exception {
MutableInt queryCount = new MutableInt(0);
- int numOfErrors = 0;
+ testCaseCtx.numOfErrors = 0;
int numOfFiles = 0;
List<CompilationUnit> cUnits = testCaseCtx.getTestCase().getCompilationUnit();
for (CompilationUnit cUnit : cUnits) {
- List<String> expectedErrors = cUnit.getExpectedError();
- BitSet expectedWarnings = new BitSet(cUnit.getExpectedWarn().size());
- expectedWarnings.set(0, cUnit.getExpectedWarn().size());
+ testCaseCtx.expectedErrors = cUnit.getExpectedError();
+ testCaseCtx.expectedWarnings = new BitSet(cUnit.getExpectedWarn().size());
+ testCaseCtx.expectedWarnings.set(0, cUnit.getExpectedWarn().size());
LOGGER.info(
"Starting [TEST]: " + testCaseCtx.getTestCase().getFilePath() + "/" + cUnit.getName() + " ... ");
Map<String, Object> variableCtx = new HashMap<>();
@@ -2120,7 +2120,7 @@
boolean loopCmd = testFile.getName().endsWith(".loop.cmd");
if (!testFile.getName().startsWith(DIAGNOSE)) {
executeTestFile(testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit,
- queryCount, expectedResultFileCtxs, testFile, actualPath, expectedWarnings);
+ queryCount, expectedResultFileCtxs, testFile, actualPath);
}
if (loopCmd) {
// this was a loop file and we have exited the loop; reset the loop iteration
@@ -2138,9 +2138,9 @@
}
loopIteration++;
} catch (Exception e) {
- numOfErrors++;
- boolean unexpected = isUnExpected(e, expectedErrors, numOfErrors, queryCount,
- testCaseCtx.isSourceLocationExpected(cUnit));
+ testCaseCtx.numOfErrors++;
+ boolean unexpected = isUnExpected(e, testCaseCtx.expectedErrors, testCaseCtx.numOfErrors,
+ queryCount, testCaseCtx.isSourceLocationExpected(cUnit));
if (unexpected) {
LOGGER.error("testFile {} raised an unexpected exception", testFile, e);
if (failedGroup != null) {
@@ -2152,12 +2152,12 @@
}
}
if (numOfFiles == testFileCtxs.size()) {
- if (numOfErrors < cUnit.getExpectedError().size()) {
+ if (testCaseCtx.numOfErrors < cUnit.getExpectedError().size()) {
LOGGER.error("Test {} failed to raise (an) expected exception(s)", cUnit.getName());
throw new Exception(
"Test \"" + cUnit.getName() + "\" FAILED; expected exception was not thrown...");
}
- ensureWarnings(expectedWarnings, cUnit);
+ ensureWarnings(testCaseCtx.expectedWarnings, cUnit);
LOGGER.info(
"[TEST]: " + testCaseCtx.getTestCase().getFilePath() + "/" + cUnit.getName() + " PASSED ");
if (passedGroup != null) {
@@ -2416,7 +2416,7 @@
final File file = ctx.getFile();
final String statement = readTestFile(file);
executeTestFile(testCaseCtx, ctx, variableCtx, statement, false, pb, cUnit, new MutableInt(-1),
- Collections.emptyList(), file, null, new BitSet());
+ Collections.emptyList(), file, null);
}
}
} catch (Exception diagnosticFailure) {
@@ -2794,11 +2794,11 @@
}
protected void validateWarning(ExtractedResult result, TestCaseContext testCaseCtx, CompilationUnit cUnit,
- File testFile, BitSet expectedWarnings) throws Exception {
+ File testFile) throws Exception {
if (testCaseCtx.getTestCase().isCheckWarnings()) {
boolean expectedSourceLoc = testCaseCtx.isSourceLocationExpected(cUnit);
- validateWarnings(result.getWarnings(), cUnit.getExpectedWarn(), expectedWarnings, expectedSourceLoc,
- testFile);
+ validateWarnings(result.getWarnings(), cUnit.getExpectedWarn(), testCaseCtx.expectedWarnings,
+ expectedSourceLoc, testFile);
}
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index 6ecdc0e..c3c94f6 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -30,7 +30,6 @@
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Paths;
-import java.util.BitSet;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -250,8 +249,8 @@
public void executeTestFile(TestCaseContext testCaseCtx, TestFileContext ctx, Map<String, Object> variableCtx,
String statement, boolean isDmlRecoveryTest, ProcessBuilder pb, TestCase.CompilationUnit cUnit,
- MutableInt queryCount, List<TestFileContext> expectedResultFileCtxs, File testFile, String actualPath,
- BitSet expectedWarnings) throws Exception {
+ MutableInt queryCount, List<TestFileContext> expectedResultFileCtxs, File testFile, String actualPath)
+ throws Exception {
String[] lines;
switch (ctx.getType()) {
case "container":
@@ -267,7 +266,7 @@
break;
default:
super.executeTestFile(testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit,
- queryCount, expectedResultFileCtxs, testFile, actualPath, expectedWarnings);
+ queryCount, expectedResultFileCtxs, testFile, actualPath);
}
}
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
index 27a46c1..8372d87 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
@@ -31,7 +31,6 @@
import java.nio.file.Paths;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
-import java.util.BitSet;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -510,8 +509,8 @@
public void executeTestFile(TestCaseContext testCaseCtx, TestFileContext ctx, Map<String, Object> variableCtx,
String statement, boolean isDmlRecoveryTest, ProcessBuilder pb, TestCase.CompilationUnit cUnit,
- MutableInt queryCount, List<TestFileContext> expectedResultFileCtxs, File testFile, String actualPath,
- BitSet expectedWarnings) throws Exception {
+ MutableInt queryCount, List<TestFileContext> expectedResultFileCtxs, File testFile, String actualPath)
+ throws Exception {
String[] lines;
switch (ctx.getType()) {
case "container":
@@ -527,7 +526,7 @@
break;
default:
super.executeTestFile(testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit,
- queryCount, expectedResultFileCtxs, testFile, actualPath, expectedWarnings);
+ queryCount, expectedResultFileCtxs, testFile, actualPath);
}
}
}
diff --git a/asterixdb/asterix-test-framework/src/main/java/org/apache/asterix/testframework/context/TestCaseContext.java b/asterixdb/asterix-test-framework/src/main/java/org/apache/asterix/testframework/context/TestCaseContext.java
index 859e0bf..c73434d 100644
--- a/asterixdb/asterix-test-framework/src/main/java/org/apache/asterix/testframework/context/TestCaseContext.java
+++ b/asterixdb/asterix-test-framework/src/main/java/org/apache/asterix/testframework/context/TestCaseContext.java
@@ -21,6 +21,7 @@
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.BitSet;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -37,77 +38,18 @@
public class TestCaseContext {
- /**
- * For specifying the desired output formatting of results.
- */
- public enum OutputFormat {
- NONE("", ""),
- ADM("adm", "application/x-adm"),
- LOSSLESS_JSON("json", "application/json; lossless=true"),
- CLEAN_JSON("json", "application/json"),
- CSV("csv", "text/csv"),
- CSV_HEADER("csv-header", "text/csv; header=present"),
- AST("ast", "application/x-ast"),
- PLAN("plan", "application/x-plan"),
- BINARY("", "application/octet-stream");
-
- private final String extension;
- private final String mimetype;
-
- OutputFormat(String ext, String mime) {
- this.extension = ext;
- this.mimetype = mime;
- }
-
- public String extension() {
- return extension;
- }
-
- public String mimeType() {
- return mimetype;
- }
-
- //
- public static OutputFormat forCompilationUnit(CompilationUnit cUnit) {
- switch (cUnit.getOutputDir().getCompare()) {
- case TEXT:
- return OutputFormat.ADM;
- case LOSSLESS_JSON:
- return OutputFormat.LOSSLESS_JSON;
- case CLEAN_JSON:
- return OutputFormat.CLEAN_JSON;
- case CSV:
- return OutputFormat.CSV;
- case CSV_HEADER:
- return OutputFormat.CSV_HEADER;
- case BINARY:
- return OutputFormat.BINARY;
- case INSPECT:
- case IGNORE:
- return OutputFormat.NONE;
- case AST:
- return OutputFormat.AST;
- case PLAN:
- return OutputFormat.PLAN;
- default:
- assert false : "Unknown ComparisonEnum!";
- return OutputFormat.NONE;
- }
- }
- }
-
public static final String DEFAULT_TESTSUITE_XML_NAME = "testsuite.xml";
public static final String ONLY_TESTSUITE_XML_NAME = "only.xml";
public static final String DEFAULT_REPEATED_TESTSUITE_XML_NAME = "repeatedtestsuite.xml";
private File tsRoot;
-
private TestSuite testSuite;
-
private TestGroup[] testGroups;
-
private TestCase testCase;
private Map<String, Object> kv;
+ public int numOfErrors;
+ public List<String> expectedErrors;
+ public BitSet expectedWarnings;
public TestCaseContext(File tsRoot, TestSuite testSuite, TestGroup[] testGroups, TestCase testCase) {
this.tsRoot = tsRoot;
@@ -234,16 +176,16 @@
}
public static class Builder {
- private final boolean m_doSlow;
- private final Pattern m_re;
+ private final boolean doSlow;
+ private final Pattern re;
public Builder() {
- m_doSlow = System.getProperty("runSlowAQLTests", "false").equals("true");
- String re = System.getProperty("testre");
- if (re == null) {
- m_re = null;
+ doSlow = System.getProperty("runSlowAQLTests", "false").equals("true");
+ String testre = System.getProperty("testre");
+ if (testre == null) {
+ this.re = null;
} else {
- m_re = Pattern.compile(re);
+ this.re = Pattern.compile(testre);
}
}
@@ -280,13 +222,13 @@
private void addContexts(File tsRoot, TestSuite ts, List<TestGroup> tgPath, List<TestCaseContext> tccs) {
TestGroup tg = tgPath.get(tgPath.size() - 1);
for (TestCase tc : tg.getTestCase()) {
- if (m_doSlow || tc.getCategory() != CategoryEnum.SLOW) {
+ if (doSlow || tc.getCategory() != CategoryEnum.SLOW) {
boolean matches = false;
- if (m_re != null) {
+ if (re != null) {
// Check all compilation units for matching
// name. If ANY match, add the test.
for (TestCase.CompilationUnit cu : tc.getCompilationUnit()) {
- if (m_re.matcher(cu.getName()).find()) {
+ if (re.matcher(cu.getName()).find()) {
matches = true;
break;
}
@@ -304,4 +246,63 @@
addContexts(tsRoot, ts, tgPath, tg.getTestGroup(), tccs);
}
}
+
+ /**
+ * For specifying the desired output formatting of results.
+ */
+ public enum OutputFormat {
+ NONE("", ""),
+ ADM("adm", "application/x-adm"),
+ LOSSLESS_JSON("json", "application/json; lossless=true"),
+ CLEAN_JSON("json", "application/json"),
+ CSV("csv", "text/csv"),
+ CSV_HEADER("csv-header", "text/csv; header=present"),
+ AST("ast", "application/x-ast"),
+ PLAN("plan", "application/x-plan"),
+ BINARY("", "application/octet-stream");
+
+ private final String extension;
+ private final String mimetype;
+
+ OutputFormat(String ext, String mime) {
+ this.extension = ext;
+ this.mimetype = mime;
+ }
+
+ public String extension() {
+ return extension;
+ }
+
+ public String mimeType() {
+ return mimetype;
+ }
+
+ //
+ public static OutputFormat forCompilationUnit(CompilationUnit cUnit) {
+ switch (cUnit.getOutputDir().getCompare()) {
+ case TEXT:
+ return OutputFormat.ADM;
+ case LOSSLESS_JSON:
+ return OutputFormat.LOSSLESS_JSON;
+ case CLEAN_JSON:
+ return OutputFormat.CLEAN_JSON;
+ case CSV:
+ return OutputFormat.CSV;
+ case CSV_HEADER:
+ return OutputFormat.CSV_HEADER;
+ case BINARY:
+ return OutputFormat.BINARY;
+ case INSPECT:
+ case IGNORE:
+ return OutputFormat.NONE;
+ case AST:
+ return OutputFormat.AST;
+ case PLAN:
+ return OutputFormat.PLAN;
+ default:
+ assert false : "Unknown ComparisonEnum!";
+ return OutputFormat.NONE;
+ }
+ }
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksThrowingConsumer.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksThrowingConsumer.java
index b677132..765f9d7 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksThrowingConsumer.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksThrowingConsumer.java
@@ -18,9 +18,25 @@
*/
package org.apache.hyracks.api.util;
+import java.util.function.Consumer;
+
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import com.google.common.util.concurrent.UncheckedExecutionException;
+
@FunctionalInterface
public interface HyracksThrowingConsumer<V> {
- void process(V value) throws HyracksDataException;
+ void accept(V value) throws HyracksDataException;
+
+ @SuppressWarnings("Duplicates")
+ static <T> Consumer<T> asUnchecked(HyracksThrowingConsumer<T> consumer) {
+ return input -> {
+ try {
+ consumer.accept(input);
+ } catch (HyracksDataException e) {
+ throw new UncheckedExecutionException(e);
+ }
+ };
+ }
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/Span.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/Span.java
index dc8bc11..1a40360 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/Span.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/Span.java
@@ -18,6 +18,7 @@
*/
package org.apache.hyracks.util;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
@@ -70,6 +71,12 @@
}
@Override
+ public boolean await(CountDownLatch latch) throws InterruptedException {
+ latch.await();
+ return true;
+ }
+
+ @Override
public String toString() {
return "<INFINITE>";
}
@@ -143,6 +150,10 @@
TimeUnit.NANOSECONDS.timedWait(monitor, remaining(TimeUnit.NANOSECONDS));
}
+ public boolean await(CountDownLatch latch) throws InterruptedException {
+ return latch.await(remaining(TimeUnit.NANOSECONDS), TimeUnit.NANOSECONDS);
+ }
+
public void loopUntilExhausted(ThrowingAction action) throws Exception {
loopUntilExhausted(action, 0, TimeUnit.NANOSECONDS);
}