diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
index ac4d1c7..1744836 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
@@ -22,6 +22,7 @@
 import java.io.InputStream;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
+import java.util.Iterator;
 import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.Callable;
@@ -33,7 +34,6 @@
 import org.apache.asterix.test.runtime.SqlppExecutionWithCancellationTest;
 import org.apache.asterix.testframework.context.TestCaseContext;
 import org.apache.asterix.testframework.xml.TestCase;
-import org.apache.asterix.testframework.xml.TestCase.CompilationUnit;
 import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.methods.HttpUriRequest;
@@ -92,18 +92,26 @@
     }
 
     @Override
-    protected boolean isUnExpected(Exception e, CompilationUnit cUnit, int numOfErrors, MutableInt queryCount) {
-        if (super.isUnExpected(e, cUnit, numOfErrors, queryCount)) {
-            String errorMsg = getErrorMessage(e);
-            // Expected, "HYR0025" means a user cancelled the query.)
-            if (errorMsg.startsWith("HYR0025")) {
-                SqlppExecutionWithCancellationTest.numCancelledQueries++;
-                queryCount.increment();
-            } else {
-                return true;
+    protected boolean isUnExpected(Exception e, List<String> expectedErrors, int numOfErrors, MutableInt queryCount) {
+        // Get the expected exception
+        for (Iterator<String> iter = expectedErrors.iterator(); iter.hasNext();) {
+            String expectedError = iter.next();
+            if (e.toString().contains(expectedError)) {
+                System.err.println("...but that was expected.");
+                iter.remove();
+                return false;
             }
         }
-        return false;
+        String errorMsg = getErrorMessage(e);
+        // Expected, "HYR0025" means a user cancelled the query.)
+        if (errorMsg.startsWith("HYR0025")) {
+            SqlppExecutionWithCancellationTest.numCancelledQueries++;
+            queryCount.increment();
+            return false;
+        } else {
+            System.err.println("Expected to find one of the following in error text:\n+++++\n" + expectedErrors + "\n+++++");
+            return true;
+        }
     }
 
     public static String getErrorMessage(Throwable th) {
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 1770f1b..bc246e7 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -1238,6 +1238,7 @@
         int numOfFiles = 0;
         List<CompilationUnit> cUnits = testCaseCtx.getTestCase().getCompilationUnit();
         for (CompilationUnit cUnit : cUnits) {
+            List<String> expectedErrors = cUnit.getExpectedError();
             LOGGER.info(
                     "Starting [TEST]: " + testCaseCtx.getTestCase().getFilePath() + "/" + cUnit.getName() + " ... ");
             Map<String, Object> variableCtx = new HashMap<>();
@@ -1253,8 +1254,7 @@
                 } catch (Exception e) {
                     System.err.println("testFile " + testFile.toString() + " raised an exception: " + e);
                     numOfErrors++;
-                    boolean unExpectedFailure = isUnExpected(e, cUnit, numOfErrors, queryCount);
-                    if (unExpectedFailure) {
+                    if (isUnExpected(e, expectedErrors, numOfErrors, queryCount)) {
                         e.printStackTrace();
                         System.err.println("...Unexpected!");
                         if (failedGroup != null) {
@@ -1263,13 +1263,14 @@
                         throw new Exception("Test \"" + testFile + "\" FAILED!", e);
                     }
                 } finally {
-                    if (numOfFiles == testFileCtxs.size() && numOfErrors < cUnit.getExpectedError().size()) {
-                        System.err.println("...Unexpected!");
-                        Exception e = new Exception(
-                                "Test \"" + cUnit.getName() + "\" FAILED!\nExpected error was not thrown...");
-                        System.err.println(e);
-                        throw e;
-                    } else if (numOfFiles == testFileCtxs.size()) {
+                    if (numOfFiles == testFileCtxs.size()) {
+                        if (numOfErrors < cUnit.getExpectedError().size()) {
+                            System.err.println("...Unexpected!");
+                            Exception e = new Exception(
+                                    "Test \"" + cUnit.getName() + "\" FAILED!\nExpected error was not thrown...");
+                            System.err.println(e);
+                            throw e;
+                        }
                         LOGGER.info("[TEST]: " + testCaseCtx.getTestCase().getFilePath() + "/" + cUnit.getName()
                                 + " PASSED ");
                     }
@@ -1278,13 +1279,13 @@
         }
     }
 
-    protected boolean isUnExpected(Exception e, CompilationUnit cUnit, int numOfErrors, MutableInt queryCount) {
+    protected boolean isUnExpected(Exception e, List<String> expectedErrors, int numOfErrors, MutableInt queryCount) {
         String expectedError = null;
-        if (cUnit.getExpectedError().size() < numOfErrors) {
+        if (expectedErrors.size() < numOfErrors) {
             return true;
         } else {
             // Get the expected exception
-            expectedError = cUnit.getExpectedError().get(numOfErrors - 1);
+            expectedError = expectedErrors.get(numOfErrors - 1);
             if (e.toString().contains(expectedError)) {
                 System.err.println("...but that was expected.");
                 return false;