Retain Server Logs On IT Test Failures

Server logs are saved to target/failsafe-reports/ in case of test failure.

Change-Id: I667111c03e7394f75cb5c4cd24b4db1c944e68b7
Reviewed-on: https://asterix-gerrit.ics.uci.edu/954
Reviewed-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java
index e7e7b61..adb0d7a 100644
--- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java
@@ -20,11 +20,16 @@
 
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileFilter;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
 import java.util.ArrayList;
 
+import org.apache.commons.io.FileUtils;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+
 public class AsterixTestHelper {
 
     public static String extToResExt(String fname, String resultExt) {
@@ -77,4 +82,54 @@
         }
         path.delete();
     }
+
+    public static void deepSelectiveCopy(File srcDir, File destDir, FileFilter filter) throws IOException {
+        if (!srcDir.isDirectory()) {
+            throw new IllegalArgumentException("Not a directory: " + srcDir);
+        }
+        if (destDir.exists() && !destDir.isDirectory()) {
+            throw new IllegalArgumentException("Exists and not a directory: " + destDir);
+        }
+        for (File child : srcDir.listFiles()) {
+            if (child.isDirectory()) {
+                deepSelectiveCopy(child, new File(destDir, child.getName()), filter);
+            } else if (filter.accept(child)) {
+                destDir.mkdirs();
+                FileUtils.copyFile(child, new File(destDir, child.getName()));
+            }
+        }
+    }
+
+    public static class RetainLogsRule extends TestWatcher {
+        private final File baseDir;
+        private final File destDir;
+        private long startTime;
+
+        public RetainLogsRule(File baseDir, File destDir) {
+            this.baseDir = baseDir;
+            this.destDir = destDir;
+        }
+
+        public RetainLogsRule(String baseDir, String destDir) {
+            this(new File(baseDir), new File(destDir));
+        }
+
+        @Override
+        protected void starting(Description description) {
+            startTime = System.currentTimeMillis();
+        }
+
+        @Override
+        protected void failed(Throwable e, Description description) {
+            File reportDir = new File(destDir, description.getTestClass().getName() + "." + description.getMethodName());
+            reportDir.mkdirs();
+            try {
+                AsterixTestHelper.deepSelectiveCopy(baseDir, reportDir,
+                        pathname -> pathname.getName().endsWith("log") &&
+                                pathname.lastModified() > startTime);
+            } catch (Exception e1) {
+                e1.printStackTrace();
+            }
+        }
+    }
 }
diff --git a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AbstractExecutionIT.java b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AbstractExecutionIT.java
index 1c3c3da..e901584 100644
--- a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AbstractExecutionIT.java
+++ b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AbstractExecutionIT.java
@@ -24,6 +24,7 @@
 import org.apache.asterix.external.util.ExternalDataConstants;
 import org.apache.asterix.external.util.IdentitiyResolverFactory;
 import org.apache.asterix.test.aql.TestExecutor;
+import org.apache.asterix.test.base.AsterixTestHelper;
 import org.apache.asterix.test.runtime.HDFSCluster;
 import org.apache.asterix.testframework.context.TestCaseContext;
 import org.apache.asterix.testframework.context.TestFileContext;
@@ -32,7 +33,9 @@
 import org.codehaus.plexus.util.FileUtils;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TestRule;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
@@ -51,10 +54,17 @@
 
     protected static final String HDFS_BASE = "../asterix-app/";
 
-    protected final static TestExecutor testExecutor = new TestExecutor();
+    protected static final TestExecutor testExecutor = new TestExecutor();
 
     private static final String EXTERNAL_LIBRARY_TEST_GROUP = "lib";
 
+    private static String reportPath =
+            new File(StringUtils.join(new String[] { "target", "failsafe-reports" }, File.separator)).getAbsolutePath();
+
+    @Rule
+    public TestRule retainLogs = new AsterixTestHelper.RetainLogsRule(
+            AsterixInstallerIntegrationUtil.getManagixHome(), reportPath);
+
     @BeforeClass
     public static void setUp() throws Exception {
         System.out.println("Starting setup");
@@ -103,6 +113,9 @@
         // to be node controller ids; a valid assumption in test environment.
         System.setProperty(ExternalDataConstants.NODE_RESOLVER_FACTORY_PROPERTY,
                 IdentitiyResolverFactory.class.getName());
+
+        reportPath = new File(StringUtils.join(new String[]{"target", "failsafe-reports"}, File.separator))
+                .getAbsolutePath();
     }
 
     @AfterClass
diff --git a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
index 506e31f..3d3d564 100644
--- a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
+++ b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
@@ -24,10 +24,14 @@
 
 import org.apache.asterix.event.model.AsterixInstance.State;
 import org.apache.asterix.test.aql.TestExecutor;
+import org.apache.asterix.test.base.AsterixTestHelper;
 import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.commons.lang3.StringUtils;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TestRule;
 
 public class AsterixExternalLibraryIT {
 
@@ -39,8 +43,15 @@
             + File.separator + "testlib-zip-binary-assembly.zip";
     private static final Logger LOGGER = Logger.getLogger(AsterixExternalLibraryIT.class.getName());
     private static List<TestCaseContext> testCaseCollection;
+    private static String reportPath =
+            new File(StringUtils.join(new String[] { "target", "failsafe-reports" }, File.separator)).getAbsolutePath();
+
     private final TestExecutor testExecutor = new TestExecutor();
 
+    @Rule
+    public TestRule retainLogs = new AsterixTestHelper.RetainLogsRule(
+            AsterixInstallerIntegrationUtil.getManagixHome(), reportPath);
+
     @BeforeClass
     public static void setUp() throws Exception {
         try {
diff --git a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/transaction/DmlRecoveryIT.java b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/transaction/DmlRecoveryIT.java
index fd10e21..ecac44e 100644
--- a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/transaction/DmlRecoveryIT.java
+++ b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/transaction/DmlRecoveryIT.java
@@ -26,11 +26,14 @@
 import java.util.logging.Logger;
 
 import org.apache.asterix.test.aql.TestExecutor;
+import org.apache.asterix.test.base.AsterixTestHelper;
 import org.apache.asterix.testframework.context.TestCaseContext;
 import org.apache.commons.io.FileUtils;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TestRule;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
@@ -51,10 +54,14 @@
     private static String managixHomeDirName;
     private static String managixHomePath;
     private static String scriptHomePath;
+    private static String reportPath;
     private static ProcessBuilder pb;
     private static Map<String, String> env;
     private final TestExecutor testExecutor = new TestExecutor();
 
+    @Rule
+    public TestRule retainLogs = new AsterixTestHelper.RetainLogsRule(managixHomePath, reportPath);
+
     @BeforeClass
     public static void setUp() throws Exception {
         File outdir = new File(PATH_ACTUAL);
@@ -62,6 +69,7 @@
 
         asterixInstallerPath = new File(System.getProperty("user.dir"));
         installerTargetPath = new File(asterixInstallerPath, "target");
+        reportPath = new File(installerTargetPath, "failsafe-reports").getAbsolutePath();
         managixHomeDirName = installerTargetPath.list(new FilenameFilter() {
             @Override
             public boolean accept(File dir, String name) {
diff --git a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/transaction/RecoveryIT.java b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/transaction/RecoveryIT.java
index 5fe09d0..25c93af 100644
--- a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/transaction/RecoveryIT.java
+++ b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/transaction/RecoveryIT.java
@@ -26,17 +26,18 @@
 import java.util.logging.Logger;
 
 import org.apache.asterix.test.aql.TestExecutor;
+import org.apache.asterix.test.base.AsterixTestHelper;
+import org.apache.asterix.test.runtime.HDFSCluster;
 import org.apache.asterix.testframework.context.TestCaseContext;
 import org.apache.commons.io.FileUtils;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TestRule;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
-
-import org.apache.asterix.test.runtime.HDFSCluster;
-import org.apache.asterix.testframework.context.TestCaseContext;
 @RunWith(Parameterized.class)
 public class RecoveryIT {
 
@@ -50,10 +51,14 @@
     private static String managixHomeDirName;
     private static String managixHomePath;
     private static String scriptHomePath;
+    private static String reportPath;
     private static ProcessBuilder pb;
     private static Map<String, String> env;
     private final TestExecutor testExecutor = new TestExecutor();
 
+    @Rule
+    public TestRule retainLogs = new AsterixTestHelper.RetainLogsRule(managixHomePath, reportPath);
+
     @BeforeClass
     public static void setUp() throws Exception {
         File outdir = new File(PATH_ACTUAL);
@@ -61,6 +66,7 @@
 
         asterixInstallerPath = new File(System.getProperty("user.dir"));
         installerTargetPath = new File(asterixInstallerPath, "target");
+        reportPath = new File(installerTargetPath, "failsafe-reports").getAbsolutePath();
         managixHomeDirName = installerTargetPath.list(new FilenameFilter() {
             @Override
             public boolean accept(File dir, String name) {
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh
index d3c75b2..2f6e71c 100755
--- a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh
@@ -1,4 +1,3 @@
-jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9;
 $MANAGIX_HOME/bin/managix stop -n asterix 1>/dev/null 2>&1;
 $MANAGIX_HOME/bin/managix delete -n asterix 1>/dev/null 2>&1;
 $MANAGIX_HOME/bin/managix create -n asterix -c $MANAGIX_HOME/clusters/local/local.xml;