[NO ISSUE][TEST] Retry on file copy, ignore failures after 3 retries

- for NCServiceExecutionIT based tests, print JVM final flags, help ensure
  max gc wait time is being configured properly by NCService

Change-Id: I7127ee0dcba6ead1a19a373b62e4d6aa22467909
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2454
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
diff --git a/asterixdb/asterix-common/pom.xml b/asterixdb/asterix-common/pom.xml
index 79af3b6..ee7fe5c 100644
--- a/asterixdb/asterix-common/pom.xml
+++ b/asterixdb/asterix-common/pom.xml
@@ -180,10 +180,6 @@
   </build>
   <dependencies>
     <dependency>
-      <groupId>commons-io</groupId>
-      <artifactId>commons-io</artifactId>
-    </dependency>
-    <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-lang3</artifactId>
     </dependency>
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java
index 1ef5abd..3840a00 100644
--- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java
@@ -25,9 +25,15 @@
 import java.io.IOException;
 import java.util.ArrayList;
 
-import org.apache.commons.io.FileUtils;
+import org.apache.hyracks.util.file.FileUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 public class AsterixTestHelper {
+    private static final Logger LOGGER = LogManager.getLogger();
+
+    private AsterixTestHelper() {
+    }
 
     public static String extToResExt(String fname, String resultExt) {
         int dot = fname.lastIndexOf('.');
@@ -77,11 +83,12 @@
             throw new IllegalArgumentException("Exists and not a directory: " + destDir);
         }
         for (File child : srcDir.listFiles()) {
+            File destChild = new File(destDir, child.getName());
             if (child.isDirectory()) {
-                deepSelectiveCopy(child, new File(destDir, child.getName()), filter);
+                deepSelectiveCopy(child, destChild, filter);
             } else if (filter.accept(child)) {
-                destDir.mkdirs();
-                FileUtils.copyFile(child, new File(destDir, child.getName()));
+                FileUtil.safeCopyFile(child, destChild);
+                return;
             }
         }
     }
diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java
similarity index 89%
rename from asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionIT.java
rename to asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java
index f1738fe..88ada80 100644
--- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionIT.java
+++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java
@@ -27,9 +27,9 @@
  * Runs the runtime test cases under 'asterix-app/src/test/resources/runtimets'.
  */
 @RunWith(Parameterized.class)
-public class SqlppExecutionIT extends AbstractExecutionIT {
+public class SqlppExecutionNCServiceIT extends AbstractExecutionIT {
 
-    @Parameters(name = "SqlppExecutionIT {index}: {0}")
+    @Parameters(name = "SqlppExecutionNCServiceIT {index}: {0}")
     public static Collection<Object[]> tests() throws Exception {
         Collection<Object[]> testArgs = buildTestsInXml("only_sqlpp.xml");
         if (testArgs.size() == 0) {
@@ -48,7 +48,7 @@
 
     }
 
-    public SqlppExecutionIT(TestCaseContext tcCtx) {
+    public SqlppExecutionNCServiceIT(TestCaseContext tcCtx) {
         super(tcCtx);
     }
 }
diff --git a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf
index 4020ca1..4658f1b 100644
--- a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf
+++ b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf
@@ -20,7 +20,7 @@
 core.dump.dir=../asterix-server/target/tmp/asterix_nc1/coredump
 iodevices=../asterix-server/target/tmp/asterix_nc1/iodevice1,../asterix-server/target/tmp/asterix_nc1/iodevice2
 nc.api.port=19004
-jvm.args=-Xloggc:/tmp/asterix-server-failsafe/asterix_nc1-%p-gc.log -XX:+PrintGC -XX:+PrintGCDateStamps -XX:GCLogFileSize=10M
+jvm.args=-Xloggc:/tmp/asterix-server-failsafe/asterix_nc1-%p-gc.log -XX:+PrintGC -XX:+PrintGCDateStamps -XX:GCLogFileSize=10M -XX:+PrintFlagsFinal
 #jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006
 
 [nc/asterix_nc2]
@@ -29,7 +29,7 @@
 core.dump.dir=../asterix-server/target/tmp/asterix_nc2/coredump
 iodevices=../asterix-server/target/tmp/asterix_nc2/iodevice1,../asterix-server/target/tmp/asterix_nc2/iodevice2
 nc.api.port=19005
-jvm.args=-Xloggc:/tmp/asterix-server-failsafe/asterix_nc2-%p-gc.log -XX:+PrintGC -XX:+PrintGCDateStamps -XX:GCLogFileSize=10M
+jvm.args=-Xloggc:/tmp/asterix-server-failsafe/asterix_nc2-%p-gc.log -XX:+PrintGC -XX:+PrintGCDateStamps -XX:GCLogFileSize=10M -XX:+PrintFlagsFinal
 #jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007
 
 [nc]
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
index 1b0093d..443640a 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
@@ -30,6 +30,7 @@
 
     private static final Logger LOGGER = LogManager.getLogger();
     private static final Object LOCK = new Object();
+    private static final int MAX_COPY_ATTEMPTS = 3;
 
     private FileUtil() {
     }
@@ -63,4 +64,26 @@
             return joined.replaceAll("(" + escapedSeparator + ")+", "$1").replaceAll(escapedSeparator + "$", "");
         }
     }
+
+    public static void safeCopyFile(File child, File destChild) throws IOException {
+        forceMkdirs(destChild.getParentFile());
+        IOException ioException = null;
+        while (true) {
+            try {
+                FileUtils.copyFile(child, destChild);
+                return;
+            } catch (IOException e) {
+                if (ioException == null) {
+                    ioException = e;
+                } else {
+                    ioException.addSuppressed(e);
+                }
+                if (ioException.getSuppressed().length >= MAX_COPY_ATTEMPTS) {
+                    LOGGER.warn("Unable to copy {} to {} after " + MAX_COPY_ATTEMPTS + " attempts; skipping file",
+                            child, destChild, e);
+                    return;
+                }
+            }
+        }
+    }
 }