Merge branch 'gerrit/stabilization-f69489'
Change-Id: I5b75e79a5dbe74314fec8b9ef7734a792ca5731c
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 71ef783..a10dc54 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -19,6 +19,7 @@
package org.apache.asterix.test.common;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hyracks.util.file.FileUtil.canonicalize;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
@@ -237,7 +238,7 @@
public void runScriptAndCompareWithResult(File scriptFile, File expectedFile, File actualFile,
ComparisonEnum compare, Charset actualEncoding) throws Exception {
- LOGGER.info("Expected results file: {} ", expectedFile);
+ LOGGER.info("Expected results file: {} ", canonicalize(expectedFile));
boolean regex = false;
if (expectedFile.getName().endsWith(".ignore")) {
return; //skip the comparison
@@ -317,7 +318,11 @@
throw createLineChangedException(scriptFile, "<EOF>", lineActual, num);
}
} catch (Exception e) {
- LOGGER.info("Actual results file: {} encoding: {}", actualFile, actualEncoding);
+ if (!actualEncoding.equals(UTF_8)) {
+ LOGGER.info("Actual results file: {} encoding: {}", canonicalize(actualFile), actualEncoding);
+ } else {
+ LOGGER.info("Actual results file: {}", canonicalize(actualFile));
+ }
throw e;
}
@@ -325,8 +330,8 @@
private ComparisonException createLineChangedException(File scriptFile, String lineExpected, String lineActual,
int num) {
- return new ComparisonException("Result for " + scriptFile + " changed at line " + num + ":\nexpected < "
- + truncateIfLong(lineExpected) + "\nactual > " + truncateIfLong(lineActual));
+ return new ComparisonException("Result for " + canonicalize(scriptFile) + " changed at line " + num
+ + ":\nexpected < " + truncateIfLong(lineExpected) + "\nactual > " + truncateIfLong(lineActual));
}
private String truncateIfLong(String string) {
@@ -468,7 +473,7 @@
if (match && !negate || negate && !match) {
continue;
}
- throw new Exception("Result for " + scriptFile + ": expected pattern '" + expression
+ throw new Exception("Result for " + canonicalize(scriptFile) + ": expected pattern '" + expression
+ "' not found in result: " + actual);
}
}
@@ -497,7 +502,8 @@
}
endOfMatch = matcher.end();
}
- throw new Exception("Result for " + scriptFile + ": actual file did not match expected result");
+ throw new Exception(
+ "Result for " + canonicalize(scriptFile) + ": actual file did not match expected result");
}
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java
index 611d5ea..c71cf31 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.test.sqlpp;
+import static org.apache.hyracks.util.file.FileUtil.canonicalize;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -82,7 +83,7 @@
try {
if (queryCount >= expectedResultFileCtxs.size()
&& !cUnit.getOutputDir().getValue().equals("none")) {
- throw new ComparisonException("no result file for " + testFile.toString() + "; queryCount: "
+ throw new ComparisonException("no result file for " + canonicalize(testFile) + "; queryCount: "
+ queryCount + ", filectxs.size: " + expectedResultFileCtxs.size());
}
@@ -96,21 +97,21 @@
"[TEST]: " + testCaseCtx.getTestCase().getFilePath() + "/" + cUnit.getName() + " PASSED ");
queryCount++;
} catch (Exception e) {
- System.err.println("testFile " + testFile.toString() + " raised an exception: " + e);
+ System.err.println("testFile " + canonicalize(testFile) + " raised an exception: " + e);
if (cUnit.getExpectedError().isEmpty()) {
e.printStackTrace();
System.err.println("...Unexpected!");
if (failedGroup != null) {
failedGroup.getTestCase().add(testCaseCtx.getTestCase());
}
- throw new Exception("Test \"" + testFile + "\" FAILED!", e);
+ throw new Exception("Test \"" + canonicalize(testFile) + "\" FAILED!", e);
} else {
// must compare with the expected failure message
if (e instanceof ComparisonException) {
throw e;
}
- LOGGER.info("[TEST]: " + testCaseCtx.getTestCase().getFilePath() + "/" + cUnit.getName()
- + " failed as expected: " + e.getMessage());
+ LOGGER.info("[TEST]: " + canonicalize(testCaseCtx.getTestCase().getFilePath()) + "/"
+ + cUnit.getName() + " failed as expected: " + e.getMessage());
System.err.println("...but that was expected.");
}
}
@@ -165,7 +166,7 @@
runScriptAndCompareWithResult(queryFile, expectedFile, actualResultFile, ComparisonEnum.TEXT,
StandardCharsets.UTF_8);
} catch (Exception e) {
- GlobalConfig.ASTERIX_LOGGER.warn("Failed while testing file " + queryFile);
+ GlobalConfig.ASTERIX_LOGGER.warn("Failed while testing file " + canonicalize(queryFile));
throw e;
} finally {
writer.close();
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/comm/NetworkAddress.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/comm/NetworkAddress.java
index b70e0c7..4fad59a 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/comm/NetworkAddress.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/comm/NetworkAddress.java
@@ -23,6 +23,7 @@
import java.io.IOException;
import java.io.Serializable;
import java.net.InetAddress;
+import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import org.apache.hyracks.api.io.IWritable;
@@ -52,6 +53,14 @@
ipAddress = null;
}
+ public NetworkAddress(InetSocketAddress socketAddress) {
+ this.address = socketAddress.getHostString();
+ this.port = socketAddress.getPort();
+ if (!socketAddress.isUnresolved()) {
+ ipAddress = socketAddress.getAddress().getAddress();
+ }
+ }
+
public String getAddress() {
return address;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java
index 474bc0a..aa1a01f 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java
@@ -21,6 +21,7 @@
import static org.apache.hyracks.util.MXHelper.osMXBean;
import static org.apache.hyracks.util.MXHelper.runtimeMXBean;
+import java.io.IOException;
import java.io.Serializable;
import java.net.InetSocketAddress;
import java.util.HashMap;
@@ -35,11 +36,17 @@
import org.apache.hyracks.control.common.heartbeat.HeartbeatSchema;
import org.apache.hyracks.util.MXHelper;
import org.apache.hyracks.util.PidHelper;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public final class NodeRegistration implements Serializable {
private static final long serialVersionUID = 1L;
- private final InetSocketAddress ncAddress;
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private InetSocketAddress ncAddress;
+
+ private NetworkAddress ncPort;
private final String nodeId;
@@ -81,9 +88,9 @@
private final HashMap<SerializedOption, Object> config;
- public NodeRegistration(InetSocketAddress ncAddress, String nodeId, NCConfig ncConfig, NetworkAddress dataPort,
+ public NodeRegistration(NetworkAddress ncPort, String nodeId, NCConfig ncConfig, NetworkAddress dataPort,
NetworkAddress resultPort, HeartbeatSchema hbSchema, NetworkAddress messagingPort, NodeCapacity capacity) {
- this.ncAddress = ncAddress;
+ this.ncPort = ncPort;
this.nodeId = nodeId;
this.dataPort = dataPort;
this.resultPort = resultPort;
@@ -193,4 +200,21 @@
public long getPid() {
return pid;
}
+
+ private void readObject(java.io.ObjectInputStream ois) throws IOException, ClassNotFoundException {
+ ois.defaultReadObject();
+ if (ncPort == null) {
+ // writer was old; we need to create a NetworkAddress from the InetSocketAddress
+ LOGGER.warn(
+ "deserializing old NodeRegistration for {}; address may be incorrectly resolved as it was resolved on the NC: {}",
+ nodeId, ncAddress);
+ ncPort = new NetworkAddress(ncAddress.getHostString(), ncAddress.getPort());
+ }
+ }
+
+ private void writeObject(java.io.ObjectOutputStream oos) throws IOException {
+ // we need to write a resolved InetSocketAddress to keep old readers happy
+ ncAddress = new InetSocketAddress(ncPort.getAddress(), ncPort.getPort());
+ oos.defaultWriteObject();
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
index bf07c20..6b0933e 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
@@ -319,11 +319,11 @@
private void initNodeControllerState() {
// Use "public" versions of network addresses and ports, if defined
- InetSocketAddress ncAddress;
+ NetworkAddress ncAddress;
if (ncConfig.getClusterPublicPort() == 0) {
- ncAddress = ipc.getSocketAddress();
+ ncAddress = new NetworkAddress(ipc.getSocketAddress());
} else {
- ncAddress = new InetSocketAddress(ncConfig.getClusterPublicAddress(), ncConfig.getClusterPublicPort());
+ ncAddress = new NetworkAddress(ncConfig.getClusterPublicAddress(), ncConfig.getClusterPublicPort());
}
HeartbeatSchema.GarbageCollectorInfo[] gcInfos = new HeartbeatSchema.GarbageCollectorInfo[gcMXBeans.size()];
for (int i = 0; i < gcInfos.length; ++i) {
diff --git a/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelControlBlock.java b/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
index 4ca7f1a..0151dbf 100644
--- a/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
+++ b/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
@@ -151,7 +151,7 @@
@Override
public void addPendingCredits(int credit) {
- cSet.addPendingCredits(channelId, credit);
+ cSet.addPendingCredits(this, credit);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelSet.java b/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelSet.java
index 179f42c..20dc8f4 100644
--- a/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelSet.java
+++ b/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/ChannelSet.java
@@ -143,13 +143,15 @@
}
}
- void addPendingCredits(int channelId, int delta) {
+ void addPendingCredits(ChannelControlBlock targetCcb, int delta) {
if (delta <= 0) {
return;
}
synchronized (mConn) {
+ final int channelId = targetCcb.getChannelId();
ChannelControlBlock ccb = ccbArray[channelId];
- if (ccb != null) {
+ // ensure the channel slot id was not recycled and used for a diffierent channel
+ if (ccb == targetCcb) {
if (ccb.getRemoteEOS()) {
return;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
index fe60653..7d162ee 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
@@ -22,6 +22,7 @@
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.file.Path;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
@@ -31,6 +32,7 @@
public class FileUtil {
private static final Logger LOGGER = LogManager.getLogger();
+ private static final Pattern PARENT_DIR = Pattern.compile("([/\\\\]|^)[^./\\\\]+[/\\\\]\\.\\.([/\\\\]|$)");
private static final Object LOCK = new Object();
private static final int MAX_COPY_ATTEMPTS = 3;
@@ -95,4 +97,22 @@
raf.getChannel().force(true);
}
}
+
+ public static String canonicalize(CharSequence path) {
+ String newPath = path.toString();
+ Matcher matcher = PARENT_DIR.matcher(newPath);
+ while (matcher.find()) {
+ // TODO(mblow): use StringBuilder once Java 8 is no longer supported (requires >=9)
+ StringBuffer sb = new StringBuffer();
+ matcher.appendReplacement(sb, matcher.group(2).isEmpty() ? "" : matcher.group(1).replace("\\", "\\\\"));
+ matcher.appendTail(sb);
+ newPath = sb.toString();
+ matcher.reset(newPath);
+ }
+ return newPath;
+ }
+
+ public static File canonicalize(File file) {
+ return new File(canonicalize(file.getPath()));
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/file/FileUtilTest.java b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/file/FileUtilTest.java
index 8d6c631..87c730c 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/file/FileUtilTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/file/FileUtilTest.java
@@ -41,4 +41,27 @@
joinPath('\\', "\\\\myserver\\tmp\\\\far\\baz\\\\\\\\lala"));
Assert.assertEquals("C:\\temp\\far\\baz\\lala", joinPath('\\', "C:\\temp\\\\far\\baz\\\\\\\\lala\\"));
}
+
+ @Test
+ public void testCanonicalize() {
+ Assert.assertEquals("bat.txt", FileUtil.canonicalize("foo/../bat.txt"));
+ Assert.assertEquals("bat.txt", FileUtil.canonicalize("foo/bar/../../bat.txt"));
+ Assert.assertEquals("foo/", FileUtil.canonicalize("foo/bar/../"));
+ Assert.assertEquals("foo", FileUtil.canonicalize("foo/bar/.."));
+ Assert.assertEquals("../bat.txt", FileUtil.canonicalize("../bat.txt"));
+ Assert.assertEquals("/bat.txt", FileUtil.canonicalize("/foo/bar/../../bat.txt"));
+ Assert.assertEquals("/bar/bat.txt", FileUtil.canonicalize("/foo/../bar/bat.txt"));
+ }
+
+ @Test
+ public void testCanonicalizeWindoze() {
+ Assert.assertEquals("bat.txt", FileUtil.canonicalize("foo\\..\\bat.txt"));
+ Assert.assertEquals("bat.txt", FileUtil.canonicalize("foo\\bar\\..\\..\\bat.txt"));
+ Assert.assertEquals("foo\\", FileUtil.canonicalize("foo\\bar\\..\\"));
+ Assert.assertEquals("foo", FileUtil.canonicalize("foo\\bar\\.."));
+ Assert.assertEquals("..\\bat.txt", FileUtil.canonicalize("..\\bat.txt"));
+ Assert.assertEquals("\\bat.txt", FileUtil.canonicalize("\\foo\\bar\\..\\..\\bat.txt"));
+ Assert.assertEquals("\\bar\\bat.txt", FileUtil.canonicalize("\\foo\\..\\bar\\bat.txt"));
+ Assert.assertEquals("C:\\bar\\bat.txt", FileUtil.canonicalize("C:\\foo\\..\\bar\\bat.txt"));
+ }
}