Currently FileSplit stores java.io.file as a field and the host housing the file as a string. This is appropriate for a local file system but not for HDFS. The path contained inside the file type is incorrectly normalized. FileSplit should be constructed just from the node and the file path. Added field path (String) to FileSplit.
git-svn-id: https://hyracks.googlecode.com/svn/trunk/hyracks@183 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java
index 76fef14..9ee7411 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java
@@ -45,7 +45,7 @@
public void open() throws HyracksDataException {
FileSplit split = splits[index];
try {
- writer = createRecordWriter(split.getLocalFile(), index);
+ writer = createRecordWriter(split, index);
} catch (Exception e) {
throw new HyracksDataException(e);
}
@@ -84,7 +84,7 @@
this.splits = splits;
}
- protected abstract IRecordWriter createRecordWriter(File file, int index) throws Exception;
+ protected abstract IRecordWriter createRecordWriter(FileSplit fileSplit, int index) throws Exception;
@Override
public IOperatorNodePushable createPushRuntime(IHyracksContext ctx, IOperatorEnvironment env,
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileSplit.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileSplit.java
index 004499a..254e8b0 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileSplit.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileSplit.java
@@ -23,12 +23,21 @@
private final String nodeName;
private final File file;
+
+ private final String path;
public FileSplit(String nodeName, File file) {
this.nodeName = nodeName;
this.file = file;
+ this.path = file.getAbsolutePath();
}
+ public FileSplit(String nodeName, String path) {
+ this.nodeName = nodeName;
+ this.path = path;
+ this.file = new File(path);
+ }
+
public String getNodeName() {
return nodeName;
}
@@ -36,4 +45,8 @@
public File getLocalFile() {
return file;
}
+
+ public String getPath(){
+ return path;
+ }
}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java
index 54fce9d..6a11fdf 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java
@@ -65,7 +65,7 @@
@Override
- protected IRecordWriter createRecordWriter(File file,int index) throws Exception {
- return new LineWriterImpl(file,columns,separator);
+ protected IRecordWriter createRecordWriter(FileSplit fileSplit,int index) throws Exception {
+ return new LineWriterImpl(fileSplit.getLocalFile(),columns,separator);
}
}
\ No newline at end of file