Merge branch 'peili/crash_recovery_tests' into salsubaiee/master_fix_search_comparators
diff --git a/asterix-app/src/test/java/edu/uci/ics/asterix/test/metadata/MetadataTest.java b/asterix-app/src/test/java/edu/uci/ics/asterix/test/metadata/MetadataTest.java
index 6da18ce..771b87f 100644
--- a/asterix-app/src/test/java/edu/uci/ics/asterix/test/metadata/MetadataTest.java
+++ b/asterix-app/src/test/java/edu/uci/ics/asterix/test/metadata/MetadataTest.java
@@ -105,7 +105,7 @@
 
     @Test
     public void test() throws Exception {
-        TestsUtils.executeTest(PATH_ACTUAL, tcCtx, null);
+        TestsUtils.executeTest(PATH_ACTUAL, tcCtx, null, false);
     }
 
 }
diff --git a/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/ExecutionTest.java b/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/ExecutionTest.java
index c8fa5f2..193cf06 100644
--- a/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/ExecutionTest.java
+++ b/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/ExecutionTest.java
@@ -131,6 +131,6 @@
 
     @Test
     public void test() throws Exception {
-        TestsUtils.executeTest(PATH_ACTUAL, tcCtx, null);
+        TestsUtils.executeTest(PATH_ACTUAL, tcCtx, null, false);
     }
 }
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/load-with-ngram-index/load-with-ngram-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-ngram-index/load-with-ngram-index.1.ddl.aql
new file mode 100644
index 0000000..9dfefe0
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-ngram-index/load-with-ngram-index.1.ddl.aql
@@ -0,0 +1,18 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+  id: int32, 
+  dblpid: string,
+  title: string,
+  authors: string,
+  misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+create dataset DBLP1(DBLPType) primary key id;
+
+create index ngram_index on DBLP(title) type ngram(3);
+create index ngram_index1 on DBLP1(title) type ngram(3);
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/load-with-ngram-index/load-with-ngram-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-ngram-index/load-with-ngram-index.2.update.aql
new file mode 100644
index 0000000..b3f20ad
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-ngram-index/load-with-ngram-index.2.update.aql
@@ -0,0 +1,4 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-id.txt"),("format"="delimited-text"),("delimiter"=":")) pre-sorted;
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/load-with-ngram-index/load-with-ngram-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-ngram-index/load-with-ngram-index.3.query.aql
new file mode 100644
index 0000000..866b045
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-ngram-index/load-with-ngram-index.3.query.aql
@@ -0,0 +1,6 @@
+use dataverse test;
+
+for $o in dataset('DBLP')
+where contains($o.title, "Multimedia")
+order by $o.id
+return $o
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/load-with-rtree-index/load-with-rtree-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-rtree-index/load-with-rtree-index.1.ddl.aql
new file mode 100644
index 0000000..6a21ae8
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-rtree-index/load-with-rtree-index.1.ddl.aql
@@ -0,0 +1,22 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type MyRecord as closed {
+  id: int32,
+  point: point,
+  kwds: string,
+  line1: line,
+  line2: line,
+  poly1: polygon,
+  poly2: polygon,
+  rec: rectangle,
+  circle: circle
+}
+
+create dataset MyData(MyRecord)
+  primary key id;
+
+create index rtree_index_point on MyData(point) type rtree;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/load-with-rtree-index/load-with-rtree-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-rtree-index/load-with-rtree-index.2.update.aql
new file mode 100644
index 0000000..2c4c9ed
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-rtree-index/load-with-rtree-index.2.update.aql
@@ -0,0 +1,6 @@
+use dataverse test;
+
+load dataset MyData 
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/spatial/spatialData.json"),("format"="adm")) pre-sorted;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/load-with-rtree-index/load-with-rtree-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-rtree-index/load-with-rtree-index.3.query.aql
new file mode 100644
index 0000000..a8d43ff
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-rtree-index/load-with-rtree-index.3.query.aql
@@ -0,0 +1,6 @@
+use dataverse test;
+
+for $o in dataset('MyData')
+where spatial-intersect($o.point, create-polygon([0.0,1.0,0.0,4.0,12.0,4.0,12.0,1.0]))
+order by $o.id
+return {"id":$o.id}
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/load-with-word-index/load-with-word-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-word-index/load-with-word-index.1.ddl.aql
new file mode 100644
index 0000000..db60934
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-word-index/load-with-word-index.1.ddl.aql
@@ -0,0 +1,18 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+  id: int32, 
+  dblpid: string,
+  title: string,
+  authors: string,
+  misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+create dataset DBLP1(DBLPType) primary key id;
+
+create index keyword_index on DBLP(title) type keyword;
+create index keyword_index1 on DBLP1(title) type keyword;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/load-with-word-index/load-with-word-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-word-index/load-with-word-index.2.update.aql
new file mode 100644
index 0000000..441e8a4
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-word-index/load-with-word-index.2.update.aql
@@ -0,0 +1,6 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-id.txt"),("format"="delimited-text"),("delimiter"=":")) pre-sorted;
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/load-with-word-index/load-with-word-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-word-index/load-with-word-index.3.query.aql
new file mode 100644
index 0000000..92d3667
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/load-with-word-index/load-with-word-index.3.query.aql
@@ -0,0 +1,6 @@
+use dataverse test;
+
+for $o in dataset('DBLP')
+let $jacc := similarity-jaccard-check(word-tokens($o.title), word-tokens("Transactions for Cooperative Environments"), 0.5f)
+where $jacc[0]
+return $o
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/load-with-ngram-index/load-with-ngram-index.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/load-with-ngram-index/load-with-ngram-index.1.adm
new file mode 100644
index 0000000..8a99b26
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/load-with-ngram-index/load-with-ngram-index.1.adm
@@ -0,0 +1,3 @@
+{ "id": 4, "dblpid": "books/acm/kim95/ChristodoulakisK95", "title": "Multimedia Information Systems  Issues and Approaches.", "authors": "Stavros Christodoulakis Leonidas Koveos", "misc": "2002-01-03 318-337 1995 Modern Database Systems db/books/collections/kim95.html#ChristodoulakisK95" }
+{ "id": 89, "dblpid": "conf/icip/SchonfeldL98", "title": "VORTEX  Video Retrieval and Tracking from Compressed Multimedia Databases.", "authors": "Dan Schonfeld Dan Lelescu", "misc": "2002-11-05 123-127 1998 ICIP (3) db/conf/icip/icip1998-3.html#SchonfeldL98" }
+{ "id": 90, "dblpid": "conf/hicss/SchonfeldL99", "title": "VORTEX  Video Retrieval and Tracking from Compressed Multimedia Databases ¾ Visual Search Engine.", "authors": "Dan Schonfeld Dan Lelescu", "misc": "2002-01-03 1999 HICSS http //computer.org/proceedings/hicss/0001/00013/00013006abs.htm db/conf/hicss/hicss1999-3.html#SchonfeldL99" }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/load-with-rtree-index/load-with-rtree-index.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/load-with-rtree-index/load-with-rtree-index.1.adm
new file mode 100644
index 0000000..5f5be7c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/load-with-rtree-index/load-with-rtree-index.1.adm
@@ -0,0 +1,3 @@
+{ "id": 10 }
+{ "id": 12 }
+{ "id": 20 }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/load-with-word-index/load-with-word-index.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/load-with-word-index/load-with-word-index.1.adm
new file mode 100644
index 0000000..5bf6ae0
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/load-with-word-index/load-with-word-index.1.adm
@@ -0,0 +1 @@
+{ "id": 9, "dblpid": "books/acm/kim95/Kaiser95", "title": "Cooperative Transactions for Multiuser Environments.", "authors": "Gail E. Kaiser", "misc": "2002-01-03 409-433 1995 Modern Database Systems db/books/collections/kim95.html#Kaiser95" }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index aaaefc1..a621f25 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -1104,6 +1104,22 @@
       </compilation-unit>
     </test-case>
     <test-case FilePath="dml">
+      <compilation-unit name="load-with-ngram-index">
+        <output-dir compare="Text">load-with-index</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="dml">
+      <compilation-unit name="load-with-rtree-index">
+        <output-dir compare="Text">load-with-index</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="dml">
+      <compilation-unit name="load-with-word-index">
+        <output-dir compare="Text">load-with-index</output-dir>
+      </compilation-unit>
+    </test-case>
+    
+    <test-case FilePath="dml">
       <compilation-unit name="opentype-c2o-recursive">
         <output-dir compare="Text">opentype-c2o-recursive</output-dir>
       </compilation-unit>
diff --git a/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java b/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java
index 30452e5..9643e1d 100644
--- a/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java
+++ b/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java
@@ -356,7 +356,8 @@
         return s.toString();
     }
 
-    public static void executeTest(String actualPath, TestCaseContext testCaseCtx, ProcessBuilder pb) throws Exception {
+    public static void executeTest(String actualPath, TestCaseContext testCaseCtx, ProcessBuilder pb,
+            boolean isDmlRecoveryTest) throws Exception {
 
         File testFile;
         File expectedResultFile;
@@ -382,10 +383,26 @@
                             TestsUtils.executeDDL(statement);
                             break;
                         case "update":
+
+                            //isDmlRecoveryTest: set IP address
+                            if (isDmlRecoveryTest && statement.contains("nc1://")) {
+                                statement = statement
+                                        .replaceAll("nc1://", "127.0.0.1://../../../../../../asterix-app/");
+
+                            }
+
                             TestsUtils.executeUpdate(statement);
                             break;
                         case "query":
                             try {
+                                // isDmlRecoveryTest: insert Crash and Recovery
+                                if (isDmlRecoveryTest) {
+                                    executeScript(pb, pb.environment().get("SCRIPT_HOME") + File.separator
+                                            + "dml_recovery" + File.separator + "kill_cc_and_nc.sh");
+                                    executeScript(pb, pb.environment().get("SCRIPT_HOME") + File.separator
+                                            + "dml_recovery" + File.separator + "stop_and_start.sh");
+                                }
+
                                 InputStream resultStream = executeQuery(statement);
                                 expectedResultFile = expectedResultFileCtxs.get(queryCount).getFile();
 
diff --git a/asterix-installer/pom.xml b/asterix-installer/pom.xml
index 373503d1..fcf0e87 100644
--- a/asterix-installer/pom.xml
+++ b/asterix-installer/pom.xml
@@ -17,6 +17,7 @@
 	<artifactId>asterix-installer</artifactId>
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+		<test.excludes>**/DmlRecoveryIT.java</test.excludes>
 	</properties>
 
 	<build>
@@ -106,6 +107,9 @@
 				<configuration>
 					<runOrder>alphabetical</runOrder>
 					<forkMode>pertest</forkMode>
+					<excludes>
+						<exclude>${test.excludes}</exclude>
+					</excludes>
 				</configuration>
 				<executions>
 					<execution>
diff --git a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixExternalLibraryIT.java b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixExternalLibraryIT.java
index a6be981..a1226c3 100644
--- a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixExternalLibraryIT.java
+++ b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixExternalLibraryIT.java
@@ -59,7 +59,7 @@
     @Test
     public void test() throws Exception {
         for (TestCaseContext testCaseCtx : testCaseCollection) {
-            TestsUtils.executeTest(PATH_ACTUAL, testCaseCtx, null);
+            TestsUtils.executeTest(PATH_ACTUAL, testCaseCtx, null, false);
         }
     }
 
diff --git a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixFaultToleranceIT.java b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixFaultToleranceIT.java
index 62b79f1..c532de5 100644
--- a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixFaultToleranceIT.java
+++ b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixFaultToleranceIT.java
@@ -63,7 +63,7 @@
     @Test
     public void test() throws Exception {
         for (TestCaseContext testCaseCtx : testCaseCollection) {
-            TestsUtils.executeTest(PATH_ACTUAL, testCaseCtx, null);
+            TestsUtils.executeTest(PATH_ACTUAL, testCaseCtx, null, false);
         }
     }
 
diff --git a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixLifecycleIT.java b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixLifecycleIT.java
index 2fd0146..d2dd016 100644
--- a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixLifecycleIT.java
+++ b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixLifecycleIT.java
@@ -121,7 +121,7 @@
     @Test
     public void test() throws Exception {
         for (TestCaseContext testCaseCtx : testCaseCollection) {
-            TestsUtils.executeTest(PATH_ACTUAL, testCaseCtx, null);
+            TestsUtils.executeTest(PATH_ACTUAL, testCaseCtx, null, false);
         }
     }
 
diff --git a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/transaction/DmlRecoveryIT.java b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/transaction/DmlRecoveryIT.java
new file mode 100644
index 0000000..ee41ddb
--- /dev/null
+++ b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/transaction/DmlRecoveryIT.java
@@ -0,0 +1,116 @@
+package edu.uci.ics.asterix.installer.transaction;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.InputStream;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Logger;
+
+import org.apache.commons.io.FileUtils;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import edu.uci.ics.asterix.test.aql.TestsUtils;
+import edu.uci.ics.asterix.testframework.context.TestCaseContext;
+import edu.uci.ics.asterix.testframework.context.TestFileContext;
+import edu.uci.ics.asterix.testframework.xml.TestCase.CompilationUnit;
+
+@RunWith(Parameterized.class)
+public class DmlRecoveryIT {
+
+    // variable to indicate whether this test will be executed
+
+    private static final Logger LOGGER = Logger.getLogger(RecoveryIT.class.getName());
+    private static final String PATH_ACTUAL = "rttest/";
+
+    private static final String TESTSUITE_PATH_BASE = "../asterix-app/src/test/resources/runtimets/";
+
+    private TestCaseContext tcCtx;
+    private static File asterixInstallerPath;
+    private static File asterixAppPath;
+    private static File asterixDBPath;
+    private static File installerTargetPath;
+    private static String managixHomeDirName;
+    private static String managixHomePath;
+    private static String scriptHomePath;
+    private static ProcessBuilder pb;
+    private static Map<String, String> env;
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        File outdir = new File(PATH_ACTUAL);
+        outdir.mkdirs();
+
+        asterixInstallerPath = new File(System.getProperty("user.dir"));
+        asterixDBPath = new File(asterixInstallerPath.getParent());
+        asterixAppPath = new File(asterixDBPath.getAbsolutePath() + File.separator + "asterix-app");
+        installerTargetPath = new File(asterixInstallerPath, "target");
+        managixHomeDirName = installerTargetPath.list(new FilenameFilter() {
+            @Override
+            public boolean accept(File dir, String name) {
+                return new File(dir, name).isDirectory() && name.startsWith("asterix-installer")
+                        && name.endsWith("binary-assembly");
+            }
+        })[0];
+        managixHomePath = new File(installerTargetPath, managixHomeDirName).getAbsolutePath();
+        LOGGER.info("MANAGIX_HOME=" + managixHomePath);
+
+        pb = new ProcessBuilder();
+        env = pb.environment();
+        env.put("MANAGIX_HOME", managixHomePath);
+        scriptHomePath = asterixInstallerPath + File.separator + "src" + File.separator + "test" + File.separator
+                + "resources" + File.separator + "transactionts" + File.separator + "scripts";
+        env.put("SCRIPT_HOME", scriptHomePath);
+
+        TestsUtils.executeScript(pb, scriptHomePath + File.separator + "dml_recovery" + File.separator
+                + "configure_and_validate.sh");
+        TestsUtils.executeScript(pb, scriptHomePath + File.separator + "dml_recovery" + File.separator
+                + "stop_and_delete.sh");
+
+        TestsUtils.executeScript(pb, scriptHomePath + File.separator + "dml_recovery" + File.separator
+                + "create_and_start.sh");
+
+    }
+
+    @AfterClass
+    public static void tearDown() throws Exception {
+        File outdir = new File(PATH_ACTUAL);
+        FileUtils.deleteDirectory(outdir);
+        TestsUtils.executeScript(pb, scriptHomePath + File.separator + "dml_recovery" + File.separator
+                + "stop_and_delete.sh");
+        TestsUtils.executeScript(pb, scriptHomePath + File.separator + "dml_recovery" + File.separator + "shutdown.sh");
+
+    }
+
+    @Parameters
+    public static Collection<Object[]> tests() throws Exception {
+
+        Collection<Object[]> testArgs = new ArrayList<Object[]>();
+        TestCaseContext.Builder b = new TestCaseContext.Builder();
+        for (TestCaseContext ctx : b.build(new File(TESTSUITE_PATH_BASE))) {
+            if (ctx.getTestCase().getFilePath().equals("dml"))
+                testArgs.add(new Object[] { ctx });
+        }
+        return testArgs;
+    }
+
+    public DmlRecoveryIT(TestCaseContext tcCtx) {
+        this.tcCtx = tcCtx;
+    }
+
+    @Test
+    public void test() throws Exception {
+
+        TestsUtils.executeTest(PATH_ACTUAL, tcCtx, pb, true);
+
+    }
+}
diff --git a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/transaction/RecoveryIT.java b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/transaction/RecoveryIT.java
index 39bd0a2..44ca366 100644
--- a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/transaction/RecoveryIT.java
+++ b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/transaction/RecoveryIT.java
@@ -114,7 +114,7 @@
 
     @Test
     public void test() throws Exception {
-        TestsUtils.executeTest(PATH_ACTUAL, tcCtx, pb);
+        TestsUtils.executeTest(PATH_ACTUAL, tcCtx, pb, false);
     }
 
 }
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.5.txnqbc.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.5.txnqbc.aql
index e40e389..f069443 100644
--- a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.5.txnqbc.aql
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.5.txnqbc.aql
@@ -1,10 +1,5 @@
-/* 
- * Test case Name  : primary_plus_keyword_secondary_index.aql
- * Description     : Check that abort from duplicate key exception works and crash recovery works after the abort.   
- * Expected Result : Success
- * Date            : Otc 15 2013
- */
-
 use dataverse recovery;
 
-count (for $x in dataset Fragile where edit-distance($x.text, $x.text)=0 return $x);
\ No newline at end of file
+for $x in dataset Fragile 
+where word-tokens($x.text) ~= word-tokens(" 1 20130417 1")
+return $x.text;
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.8.txnqar.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.8.txnqar.aql
index b72cb5f..f069443 100644
--- a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.8.txnqar.aql
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_keyword_secondary_index/primary_plus_keyword_secondary_index.8.txnqar.aql
@@ -1,10 +1,5 @@
-/* 
- * Test case Name  : primary_plus_keyword_secondary_index.aql
- * Description     : Check that abort from duplicate key exception works and crash recovery works after the abort.   
- * Expected Result : Success
- * Date            : Oct 15 2013
- */
-
 use dataverse recovery;
 
-count (for $x in dataset Fragile where edit-distance($x.text, $x.text)=0 return $x);
\ No newline at end of file
+for $x in dataset Fragile 
+where word-tokens($x.text) ~= word-tokens(" 1 20130417 1")
+return $x.text;
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.5.txnqbc.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.5.txnqbc.aql
index 28ea155..97935fa 100644
--- a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.5.txnqbc.aql
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.5.txnqbc.aql
@@ -5,6 +5,8 @@
  * Date            : Otc 15 2013
  */
 
+
 use dataverse recovery;
 
-count (for $x in dataset Fragile where edit-distance($x.text2, $x.text2)=0 return $x);
\ No newline at end of file
+count (for $x in dataset Fragile 
+where contains($x.text2, "location") return $x);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.8.txnqar.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.8.txnqar.aql
index 33551e6..5e18be6 100644
--- a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.8.txnqar.aql
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_multiple_secondary_indices/primary_plus_multiple_secondary_indices.8.txnqar.aql
@@ -1,10 +1,5 @@
-/* 
- * Test case Name  : primary_plus_keyword_secondary_index.aql
- * Description     : Check that abort from duplicate key exception works and crash recovery works after the abort.   
- * Expected Result : Success
- * Date            : Oct 15 2013
- */
 
 use dataverse recovery;
 
-count (for $x in dataset Fragile where edit-distance($x.text2, $x.text2)=0 return $x);
\ No newline at end of file
+count (for $x in dataset Fragile 
+where contains($x.text2, "location") return $x);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_ngram_index/primary_plus_ngram_index.5.txnqbc.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_ngram_index/primary_plus_ngram_index.5.txnqbc.aql
index 1383cc6..434e8b0 100644
--- a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_ngram_index/primary_plus_ngram_index.5.txnqbc.aql
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_ngram_index/primary_plus_ngram_index.5.txnqbc.aql
@@ -7,4 +7,5 @@
 
 use dataverse recovery;
 
-count (for $x in dataset Fragile where edit-distance($x.text2, $x.text2)=0 return $x);
\ No newline at end of file
+count (for $x in dataset Fragile 
+where contains($x.text2, "location") return $x);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_ngram_index/primary_plus_ngram_index.8.txnqar.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_ngram_index/primary_plus_ngram_index.8.txnqar.aql
index 0dbbf71..21a5d28 100644
--- a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_ngram_index/primary_plus_ngram_index.8.txnqar.aql
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_ngram_index/primary_plus_ngram_index.8.txnqar.aql
@@ -7,4 +7,5 @@
 
 use dataverse recovery;
 
-count (for $x in dataset Fragile where edit-distance($x.text2, $x.text2)=0 return $x);
\ No newline at end of file
+count (for $x in dataset Fragile 
+where contains($x.text2, "location") return $x);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index/primary_plus_rtree_index.5.txnqbc.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index/primary_plus_rtree_index.5.txnqbc.aql
index 1d8bc54..9189b28 100644
--- a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index/primary_plus_rtree_index.5.txnqbc.aql
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index/primary_plus_rtree_index.5.txnqbc.aql
@@ -1,10 +1,5 @@
-/* 
- * Test case Name  : primary_plus_keyword_secondary_index.aql
- * Description     : Check that abort from duplicate key exception works and crash recovery works after the abort.   
- * Expected Result : Success
- * Date            : Otc 15 2013
- */
-
 use dataverse recovery;
 
-count (for $x in dataset Fragile where $x.location=$x.location return $x);
\ No newline at end of file
+count (for $x in dataset Fragile where 
+spatial-intersect($x.location, create-polygon([0.0,0.0, 2.0,2.0, 0.0,2.0, 2.0,0.0]))
+return $x);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index/primary_plus_rtree_index.8.txnqar.aql b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index/primary_plus_rtree_index.8.txnqar.aql
index 4a94174..9189b28 100644
--- a/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index/primary_plus_rtree_index.8.txnqar.aql
+++ b/asterix-installer/src/test/resources/transactionts/queries/recover_after_abort/primary_plus_rtree_index/primary_plus_rtree_index.8.txnqar.aql
@@ -1,10 +1,5 @@
-/* 
- * Test case Name  : primary_plus_keyword_secondary_index.aql
- * Description     : Check that abort from duplicate key exception works and crash recovery works after the abort.   
- * Expected Result : Success
- * Date            : Oct 15 2013
- */
-
 use dataverse recovery;
 
-count (for $x in dataset Fragile where $x.location=$x.location return $x);
\ No newline at end of file
+count (for $x in dataset Fragile where 
+spatial-intersect($x.location, create-polygon([0.0,0.0, 2.0,2.0, 0.0,2.0, 2.0,0.0]))
+return $x);
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/configure_and_validate.sh b/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/configure_and_validate.sh
new file mode 100755
index 0000000..643e9ad
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/configure_and_validate.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix configure;
+$MANAGIX_HOME/bin/managix validate;
+$MANAGIX_HOME/bin/managix validate -c $MANAGIX_HOME/clusters/local/local.xml;
\ No newline at end of file
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/create_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/create_and_start.sh
new file mode 100755
index 0000000..945f01d
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/create_and_start.sh
@@ -0,0 +1 @@
+$MANAGIX_HOME/bin/managix create -n nc1 -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/kill_cc_and_nc.sh b/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/kill_cc_and_nc.sh
new file mode 100755
index 0000000..096d7df
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/kill_cc_and_nc.sh
@@ -0,0 +1 @@
+jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/shutdown.sh b/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/shutdown.sh
new file mode 100755
index 0000000..4df5a05
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/shutdown.sh
@@ -0,0 +1 @@
+$MANAGIX_HOME/bin/managix shutdown;
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/stop_and_delete.sh b/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/stop_and_delete.sh
new file mode 100755
index 0000000..d7deea3
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/stop_and_delete.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix delete -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/stop_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/stop_and_start.sh
new file mode 100755
index 0000000..7855938
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/dml_recovery/stop_and_start.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix start -n nc1;
+
diff --git a/asterix-installer/src/test/resources/transactionts/testsuite.xml b/asterix-installer/src/test/resources/transactionts/testsuite.xml
index 211564a..f2bd2b5 100644
--- a/asterix-installer/src/test/resources/transactionts/testsuite.xml
+++ b/asterix-installer/src/test/resources/transactionts/testsuite.xml
@@ -13,8 +13,9 @@
  ! limitations under the License.
  !-->
 <test-suite xmlns="urn:xml.testframework.asterix.ics.uci.edu" ResultOffsetPath="results" QueryOffsetPath="queries" QueryFileExtension=".aql">
+ 
   <test-group name="recover_after_abort">
-  
+
     <test-case FilePath="recover_after_abort">
       <compilation-unit name="primary_index_only">
         <output-dir compare="Text">primary_index_only</output-dir>
@@ -26,13 +27,13 @@
         <output-dir compare="Text">primary_plus_default_secondary_index</output-dir>
       </compilation-unit>
     </test-case>
-     
+
     <test-case FilePath="recover_after_abort">
       <compilation-unit name="primary_plus_rtree_index">
         <output-dir compare="Text">primary_plus_rtree_index</output-dir>
       </compilation-unit>
     </test-case>
-   
+     
     <test-case FilePath="recover_after_abort">
       <compilation-unit name="primary_plus_keyword_secondary_index">
         <output-dir compare="Text">primary_plus_keyword_secondary_index</output-dir>
@@ -50,7 +51,11 @@
         <output-dir compare="Text">primary_plus_multiple_secondary_indices</output-dir>
       </compilation-unit>
     </test-case>
-<!-- 
+  
+ </test-group>
+
+ <test-group name="recovery_ddl">
+    
     <test-case FilePath="recovery_ddl">
       <compilation-unit name="dataverse_recovery">
         <output-dir compare="Text">dataverse_recovery</output-dir>
@@ -80,7 +85,7 @@
         <output-dir compare="Text">load_after_recovery</output-dir>
       </compilation-unit>
     </test-case>
-
+    
     <test-case FilePath="recovery_ddl">
       <compilation-unit name="insert_after_recovery">
         <output-dir compare="Text">insert_after_recovery</output-dir>
@@ -92,12 +97,14 @@
         <output-dir compare="Text">delete_after_recovery</output-dir>
       </compilation-unit>
     </test-case>
-
+    
     <test-case FilePath="recovery_ddl">
       <compilation-unit name="function_recovery">
         <output-dir compare="Text">function_recovery</output-dir>
       </compilation-unit>
     </test-case>
---> 
+ 
   </test-group>
+
+
 </test-suite>