[NO ISSUE][TEST] Retry on file copy, ignore failures after 3 retries
- for NCServiceExecutionIT based tests, print JVM final flags, help ensure
max gc wait time is being configured properly by NCService
Change-Id: I7127ee0dcba6ead1a19a373b62e4d6aa22467909
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2454
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
diff --git a/asterixdb/asterix-common/pom.xml b/asterixdb/asterix-common/pom.xml
index 79af3b6..ee7fe5c 100644
--- a/asterixdb/asterix-common/pom.xml
+++ b/asterixdb/asterix-common/pom.xml
@@ -180,10 +180,6 @@
</build>
<dependencies>
<dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java
index 1ef5abd..3840a00 100644
--- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java
@@ -25,9 +25,15 @@
import java.io.IOException;
import java.util.ArrayList;
-import org.apache.commons.io.FileUtils;
+import org.apache.hyracks.util.file.FileUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class AsterixTestHelper {
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private AsterixTestHelper() {
+ }
public static String extToResExt(String fname, String resultExt) {
int dot = fname.lastIndexOf('.');
@@ -77,11 +83,12 @@
throw new IllegalArgumentException("Exists and not a directory: " + destDir);
}
for (File child : srcDir.listFiles()) {
+ File destChild = new File(destDir, child.getName());
if (child.isDirectory()) {
- deepSelectiveCopy(child, new File(destDir, child.getName()), filter);
+ deepSelectiveCopy(child, destChild, filter);
} else if (filter.accept(child)) {
- destDir.mkdirs();
- FileUtils.copyFile(child, new File(destDir, child.getName()));
+ FileUtil.safeCopyFile(child, destChild);
+ return;
}
}
}
diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java
similarity index 89%
rename from asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionIT.java
rename to asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java
index f1738fe..88ada80 100644
--- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionIT.java
+++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java
@@ -27,9 +27,9 @@
* Runs the runtime test cases under 'asterix-app/src/test/resources/runtimets'.
*/
@RunWith(Parameterized.class)
-public class SqlppExecutionIT extends AbstractExecutionIT {
+public class SqlppExecutionNCServiceIT extends AbstractExecutionIT {
- @Parameters(name = "SqlppExecutionIT {index}: {0}")
+ @Parameters(name = "SqlppExecutionNCServiceIT {index}: {0}")
public static Collection<Object[]> tests() throws Exception {
Collection<Object[]> testArgs = buildTestsInXml("only_sqlpp.xml");
if (testArgs.size() == 0) {
@@ -48,7 +48,7 @@
}
- public SqlppExecutionIT(TestCaseContext tcCtx) {
+ public SqlppExecutionNCServiceIT(TestCaseContext tcCtx) {
super(tcCtx);
}
}
diff --git a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf
index 4020ca1..4658f1b 100644
--- a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf
+++ b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf
@@ -20,7 +20,7 @@
core.dump.dir=../asterix-server/target/tmp/asterix_nc1/coredump
iodevices=../asterix-server/target/tmp/asterix_nc1/iodevice1,../asterix-server/target/tmp/asterix_nc1/iodevice2
nc.api.port=19004
-jvm.args=-Xloggc:/tmp/asterix-server-failsafe/asterix_nc1-%p-gc.log -XX:+PrintGC -XX:+PrintGCDateStamps -XX:GCLogFileSize=10M
+jvm.args=-Xloggc:/tmp/asterix-server-failsafe/asterix_nc1-%p-gc.log -XX:+PrintGC -XX:+PrintGCDateStamps -XX:GCLogFileSize=10M -XX:+PrintFlagsFinal
#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006
[nc/asterix_nc2]
@@ -29,7 +29,7 @@
core.dump.dir=../asterix-server/target/tmp/asterix_nc2/coredump
iodevices=../asterix-server/target/tmp/asterix_nc2/iodevice1,../asterix-server/target/tmp/asterix_nc2/iodevice2
nc.api.port=19005
-jvm.args=-Xloggc:/tmp/asterix-server-failsafe/asterix_nc2-%p-gc.log -XX:+PrintGC -XX:+PrintGCDateStamps -XX:GCLogFileSize=10M
+jvm.args=-Xloggc:/tmp/asterix-server-failsafe/asterix_nc2-%p-gc.log -XX:+PrintGC -XX:+PrintGCDateStamps -XX:GCLogFileSize=10M -XX:+PrintFlagsFinal
#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007
[nc]
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
index 1b0093d..443640a 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
@@ -30,6 +30,7 @@
private static final Logger LOGGER = LogManager.getLogger();
private static final Object LOCK = new Object();
+ private static final int MAX_COPY_ATTEMPTS = 3;
private FileUtil() {
}
@@ -63,4 +64,26 @@
return joined.replaceAll("(" + escapedSeparator + ")+", "$1").replaceAll(escapedSeparator + "$", "");
}
}
+
+ public static void safeCopyFile(File child, File destChild) throws IOException {
+ forceMkdirs(destChild.getParentFile());
+ IOException ioException = null;
+ while (true) {
+ try {
+ FileUtils.copyFile(child, destChild);
+ return;
+ } catch (IOException e) {
+ if (ioException == null) {
+ ioException = e;
+ } else {
+ ioException.addSuppressed(e);
+ }
+ if (ioException.getSuppressed().length >= MAX_COPY_ATTEMPTS) {
+ LOGGER.warn("Unable to copy {} to {} after " + MAX_COPY_ATTEMPTS + " attempts; skipping file",
+ child, destChild, e);
+ return;
+ }
+ }
+ }
+ }
}