added fix for issue 604 and 605
diff --git a/asterix-app/data/csv/sample_01.csv b/asterix-app/data/csv/sample_01.csv
index 5a2a4f5..285a30d 100644
--- a/asterix-app/data/csv/sample_01.csv
+++ b/asterix-app/data/csv/sample_01.csv
@@ -1,4 +1,4 @@
-1,0.899682764,5.6256,2013-08-07,7:22:35,1979-02-25T23:48:27.000,"3.485, 94.493","-4.0,2.0 2.0,2.0","1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0","0.0,0.0 4.0,4.0","11.0,14.0 15.0"
-2,0.669052398,0.446192,-1923-3-29,19:33:34,-1979-02-25T23:48:27.000,"-3.485, 94.493","-4.0,2.0 2.0,2.0","1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0","0.0,0.0 4.0,4.0","11.0,14.0 15.0"
+1,0.899682764,5.6256,2013-08-07,07:22:35,1979-02-25T23:48:27.034
+2,0.669052398,0.446192,-1923-03-29,19:33:34,-1979-02-25T23:48:27.002
,,,,,,,,,,
-3,0.572733058,192674,-1923-3-29,19:33:34,-1979-02-25T23:48:27.000,"-3.485, 94.493","-4.0,2.0 2.0,2.0","1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0","0.0,0.0 4.0,4.0","11.0,14.0 15.0"
\ No newline at end of file
+3,0.572733058,192674,-1923-03-29,19:33:34,-1979-02-25T23:48:27.001
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/load/csv_01/csv_01.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/load/csv_01/csv_01.1.ddl.aql
index aabc9d8..4a9b7af 100644
--- a/asterix-app/src/test/resources/runtimets/queries/load/csv_01/csv_01.1.ddl.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/load/csv_01/csv_01.1.ddl.aql
@@ -15,12 +15,7 @@
double: double,
date: string,
time: string,
- datetime: string,
- point: string,
- line: string,
- polygon: string,
- rectangle: string,
- circle: string
+ datetime: string
};
create dataset testds (test)
diff --git a/asterix-app/src/test/resources/runtimets/queries/load/csv_01/csv_01.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/load/csv_01/csv_01.3.query.aql
index 3f03ab9..f798b20 100644
--- a/asterix-app/src/test/resources/runtimets/queries/load/csv_01/csv_01.3.query.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/load/csv_01/csv_01.3.query.aql
@@ -13,9 +13,5 @@
"double": $i.double,
"date": date($i.date),
"time": time($i.time),
- "datetime": datetime($i.datetime),
- "point": point($i.point),
- "line": line($i.line),
- "polygon": polygon($i.polygon),
- "rectangle": rectangle($i.rectangle),
- "circle": circle($i.circle)}
\ No newline at end of file
+ "datetime": datetime($i.datetime)
+ }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/load/csv_01/csv_01.1.adm b/asterix-app/src/test/resources/runtimets/results/load/csv_01/csv_01.1.adm
index aabc9d8..8287a4a 100644
--- a/asterix-app/src/test/resources/runtimets/results/load/csv_01/csv_01.1.adm
+++ b/asterix-app/src/test/resources/runtimets/results/load/csv_01/csv_01.1.adm
@@ -1,27 +1,3 @@
-/**
- *
- * CSV file loading test
- * Expected result: success
- *
- */
-
-drop dataverse temp if exists;
-create dataverse temp
-use dataverse temp;
-
-create type test as closed {
- id: int32,
- float: float,
- double: double,
- date: string,
- time: string,
- datetime: string,
- point: string,
- line: string,
- polygon: string,
- rectangle: string,
- circle: string
-};
-
-create dataset testds (test)
-primary key id;
+{ "id": 1, "float": 0.89968276f, "double": 5.6256d, "date": date("2013-08-07"), "time": time("07:22:35.000Z"), "datetime": datetime("1979-02-25T23:48:27.034Z") }
+{ "id": 2, "float": 0.6690524f, "double": 0.446192d, "date": date("-1923-03-29"), "time": time("19:33:34.000Z"), "datetime": datetime("-1979-02-25T23:48:27.002Z") }
+{ "id": 3, "float": 0.57273304f, "double": 192674.0d, "date": date("-1923-03-29"), "time": time("19:33:34.000Z"), "datetime": datetime("-1979-02-25T23:48:27.001Z") }
\ No newline at end of file
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/DelimitedDataParser.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/DelimitedDataParser.java
index 921dbf3..0e8724e 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/DelimitedDataParser.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/DelimitedDataParser.java
@@ -47,6 +47,8 @@
private int[] fldIds;
private ArrayBackedValueStorage[] nameBuffers;
+ private boolean areAllNullFields;
+
public DelimitedDataParser(ARecordType recordType, IValueParserFactory[] valueParserFactories, char fieldDelimter) {
this.recordType = recordType;
this.valueParserFactories = valueParserFactories;
@@ -97,29 +99,36 @@
@Override
public boolean parse(DataOutput out) throws AsterixException, IOException {
-
- if (cursor.nextRecord()) {
+ while (cursor.nextRecord()) {
recBuilder.reset(recordType);
recBuilder.init();
+ areAllNullFields = true;
for (int i = 0; i < valueParsers.length; ++i) {
if (!cursor.nextField()) {
break;
}
fieldValueBuffer.reset();
- fieldValueBufferOutput.writeByte(fieldTypeTags[i]);
- valueParsers[i]
- .parse(cursor.buffer, cursor.fStart, cursor.fEnd - cursor.fStart, fieldValueBufferOutput);
+
+ if (cursor.fStart == cursor.fEnd) {
+ fieldValueBufferOutput.writeByte(ATypeTag.NULL.serialize());
+ } else {
+ fieldValueBufferOutput.writeByte(fieldTypeTags[i]);
+ valueParsers[i].parse(cursor.buffer, cursor.fStart, cursor.fEnd - cursor.fStart,
+ fieldValueBufferOutput);
+ areAllNullFields = false;
+ }
if (fldIds[i] < 0) {
recBuilder.addField(nameBuffers[i], fieldValueBuffer);
} else {
recBuilder.addField(fldIds[i], fieldValueBuffer);
}
}
- recBuilder.write(out, true);
- return true;
- } else {
- return false;
+ if (!areAllNullFields) {
+ recBuilder.write(out, true);
+ return true;
+ }
}
+ return false;
}
protected void fieldNameToBytes(String fieldName, AMutableString str, ArrayBackedValueStorage buffer)
@@ -254,6 +263,8 @@
eof = !readMore();
if (eof) {
state = State.EOF;
+ fStart = start;
+ fEnd = p - (s - start);
return true;
}
p -= (s - start);