[ASTERIXDB-2719][EXT] Add .gz compressed test cases for CSV and TSV
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Added test cases for compressed data for CSV and TSV.
- Addressed a comment to ensure closing the S3 stream in
case of a failure during streaming the compressed data.
Change-Id: I6160ee5f5aefa4ca60e5a88c3922bde40f4c65ec
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6123
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
Reviewed-by: Dmitry Lychagin <dmitry.lychagin@couchbase.com>
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index c76a7ca..5e2d2de 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -208,15 +208,13 @@
LOGGER.info("Adding TSV files to the bucket");
loadTsvFiles();
LOGGER.info("TSV Files added successfully");
-
- LOGGER.info("Files added successfully");
}
private static void loadJsonFiles() {
String dataBasePath = JSON_DATA_PATH;
String definition = S3_MOCK_SERVER_BUCKET_JSON_DEFINITION;
- // Json data
+ // Normal format
String definitionSegment = "json";
loadData(dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
loadData(dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
@@ -224,7 +222,7 @@
loadData(dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition, definitionSegment,
false);
- // Json gz compressed data
+ // gz compressed format
definitionSegment = "gz";
loadGzData(dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
loadGzData(dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
@@ -232,7 +230,7 @@
loadGzData(dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition, definitionSegment,
false);
- // Mixed json and json gz compressed data
+ // Mixed normal and gz compressed format
definitionSegment = "mixed";
loadData(dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
loadData(dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
@@ -246,6 +244,50 @@
false);
}
+ private static void loadCsvFiles() {
+ String dataBasePath = CSV_DATA_PATH;
+ String definition = S3_MOCK_SERVER_BUCKET_CSV_DEFINITION;
+
+ // Normal format
+ String definitionSegment = "csv";
+ loadData(dataBasePath, "", "01.csv", definition, definitionSegment, false);
+ loadData(dataBasePath, "", "02.csv", definition, definitionSegment, false);
+
+ // gz compressed format
+ definitionSegment = "gz";
+ loadGzData(dataBasePath, "", "01.csv", definition, definitionSegment, false);
+ loadGzData(dataBasePath, "", "02.csv", definition, definitionSegment, false);
+
+ // Mixed normal and gz compressed format
+ definitionSegment = "mixed";
+ loadData(dataBasePath, "", "01.csv", definition, definitionSegment, false);
+ loadData(dataBasePath, "", "02.csv", definition, definitionSegment, false);
+ loadGzData(dataBasePath, "", "01.csv", definition, definitionSegment, false);
+ loadGzData(dataBasePath, "", "02.csv", definition, definitionSegment, false);
+ }
+
+ private static void loadTsvFiles() {
+ String dataBasePath = TSV_DATA_PATH;
+ String definition = S3_MOCK_SERVER_BUCKET_TSV_DEFINITION;
+
+ // Normal format
+ String definitionSegment = "tsv";
+ loadData(dataBasePath, "", "01.tsv", definition, definitionSegment, false);
+ loadData(dataBasePath, "", "02.tsv", definition, definitionSegment, false);
+
+ // gz compressed format
+ definitionSegment = "gz";
+ loadGzData(dataBasePath, "", "01.tsv", definition, definitionSegment, false);
+ loadGzData(dataBasePath, "", "02.tsv", definition, definitionSegment, false);
+
+ // Mixed normal and gz compressed format
+ definitionSegment = "mixed";
+ loadData(dataBasePath, "", "01.tsv", definition, definitionSegment, false);
+ loadData(dataBasePath, "", "02.tsv", definition, definitionSegment, false);
+ loadGzData(dataBasePath, "", "01.tsv", definition, definitionSegment, false);
+ loadGzData(dataBasePath, "", "02.tsv", definition, definitionSegment, false);
+ }
+
private static void loadData(String fileBasePath, String filePathSegment, String filename, String definition,
String definitionSegment, boolean removeExtension) {
// Files data
@@ -262,7 +304,9 @@
}
// Files base definition
- String basePath = definition + filePathSegment + "/" + definitionSegment + "/";
+ filePathSegment = filePathSegment.isEmpty() ? "" : filePathSegment + "/";
+ definitionSegment = definitionSegment.isEmpty() ? "" : definitionSegment + "/";
+ String basePath = definition + filePathSegment + definitionSegment;
// Load the data
client.putObject(builder.key(basePath + finalFileName).build(), requestBody);
@@ -297,7 +341,9 @@
finalFileName += ".gz";
// Files base definition
- String basePath = definition + filePathSegment + "/" + definitionSegment + "/";
+ filePathSegment = filePathSegment.isEmpty() ? "" : filePathSegment + "/";
+ definitionSegment = definitionSegment.isEmpty() ? "" : definitionSegment + "/";
+ String basePath = definition + filePathSegment + definitionSegment;
// Load the data
client.putObject(builder.key(basePath + finalFileName).build(), requestBody);
@@ -310,31 +356,6 @@
}
}
- private static void loadCsvFiles() {
- LOGGER.info("Adding CSV files to the bucket");
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_CSV_DEFINITION + "01.csv").build(),
- RequestBody.fromFile(Paths.get(CSV_DATA_PATH, "01.csv")));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_CSV_DEFINITION + "2018/01.csv").build(),
- RequestBody.fromFile(Paths.get(CSV_DATA_PATH, "02.csv")));
- }
-
- private static void loadTsvFiles() {
- LOGGER.info("Adding TSV files to the bucket");
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_TSV_DEFINITION + "01.tsv").build(),
- RequestBody.fromFile(Paths.get(TSV_DATA_PATH, "01.tsv")));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_TSV_DEFINITION + "2018/01.tsv").build(),
- RequestBody.fromFile(Paths.get(TSV_DATA_PATH, "02.tsv")));
- LOGGER.info("Files added successfully");
- }
-
static class AwsTestExecutor extends TestExecutor {
public void executeTestFile(TestCaseContext testCaseCtx, TestFileContext ctx, Map<String, Object> variableCtx,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/csv/query-dataset.000.ddl.sqlpp
similarity index 96%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.000.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/csv/query-dataset.000.ddl.sqlpp
index 15ba6a8..5929d2d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/csv/query-dataset.000.ddl.sqlpp
@@ -31,7 +31,7 @@
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
("container"="playground"),
-("definition"="csv-data/reviews"),
+("definition"="csv-data/reviews/csv"),
("format"="Csv"),
("header"="false"),
("null"="")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/csv/query-dataset.002.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.002.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/csv/query-dataset.002.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.003.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/csv/query-dataset.003.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.003.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/csv/query-dataset.003.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/gz/query-dataset.000.ddl.sqlpp
similarity index 96%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/gz/query-dataset.000.ddl.sqlpp
index 15ba6a8..9683b18 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/gz/query-dataset.000.ddl.sqlpp
@@ -31,7 +31,7 @@
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
("container"="playground"),
-("definition"="csv-data/reviews"),
+("definition"="csv-data/reviews/gz"),
("format"="Csv"),
("header"="false"),
("null"="")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/gz/query-dataset.002.query.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.002.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/gz/query-dataset.002.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.003.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/gz/query-dataset.003.ddl.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.003.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/gz/query-dataset.003.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/mixed/query-dataset.000.ddl.sqlpp
similarity index 96%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/mixed/query-dataset.000.ddl.sqlpp
index 15ba6a8..7fe8fda 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/mixed/query-dataset.000.ddl.sqlpp
@@ -31,7 +31,7 @@
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
("container"="playground"),
-("definition"="csv-data/reviews"),
+("definition"="csv-data/reviews/mixed"),
("format"="Csv"),
("header"="false"),
("null"="")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/mixed/query-dataset.002.query.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.002.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/mixed/query-dataset.002.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.003.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/mixed/query-dataset.003.ddl.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.003.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/mixed/query-dataset.003.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/gz/query-dataset.000.ddl.sqlpp
similarity index 96%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/gz/query-dataset.000.ddl.sqlpp
index 3c6ad92..ac33d92 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/gz/query-dataset.000.ddl.sqlpp
@@ -31,7 +31,7 @@
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
("container"="playground"),
-("definition"="tsv-data/reviews"),
+("definition"="tsv-data/reviews/gz"),
("format"="TSV"),
("header"="False"),
("null"="")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/gz/query-dataset.002.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.002.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/gz/query-dataset.002.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.003.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/gz/query-dataset.003.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.003.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/gz/query-dataset.003.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/mixed/query-dataset.000.ddl.sqlpp
similarity index 96%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.000.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/mixed/query-dataset.000.ddl.sqlpp
index 3c6ad92..a392c61 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/mixed/query-dataset.000.ddl.sqlpp
@@ -31,7 +31,7 @@
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
("container"="playground"),
-("definition"="tsv-data/reviews"),
+("definition"="tsv-data/reviews/mixed"),
("format"="TSV"),
("header"="False"),
("null"="")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/mixed/query-dataset.002.query.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.002.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/mixed/query-dataset.002.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.003.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/mixed/query-dataset.003.ddl.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.003.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/mixed/query-dataset.003.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/tsv/query-dataset.000.ddl.sqlpp
similarity index 96%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/tsv/query-dataset.000.ddl.sqlpp
index 3c6ad92..f9b82d4 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/tsv/query-dataset.000.ddl.sqlpp
@@ -31,7 +31,7 @@
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
("container"="playground"),
-("definition"="tsv-data/reviews"),
+("definition"="tsv-data/reviews/tsv"),
("format"="TSV"),
("header"="False"),
("null"="")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/tsv/query-dataset.002.query.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.002.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/tsv/query-dataset.002.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.003.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/tsv/query-dataset.003.ddl.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.003.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/tsv/query-dataset.003.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/csv/000/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/csv/000/external_dataset.001.adm
deleted file mode 100644
index 93d1b57..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/csv/000/external_dataset.001.adm
+++ /dev/null
@@ -1,6 +0,0 @@
-{ "id": 1, "year": null, "review": "good", "details": "recommend" }
-{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
-{ "id": 3, "year": null, "review": "good", "details": null }
-{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
-{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
-{ "id": 6, "year": 2018, "review": "good", "details": null }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/csv/csv/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/csv/csv/external_dataset.001.adm
new file mode 100644
index 0000000..4dbd63a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/csv/csv/external_dataset.001.adm
@@ -0,0 +1,30 @@
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/csv/gz/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/csv/gz/external_dataset.001.adm
new file mode 100644
index 0000000..4dbd63a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/csv/gz/external_dataset.001.adm
@@ -0,0 +1,30 @@
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/csv/mixed/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/csv/mixed/external_dataset.001.adm
new file mode 100644
index 0000000..7aec747
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/csv/mixed/external_dataset.001.adm
@@ -0,0 +1,60 @@
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 1, "year": null, "review": "good", "details": "recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 2, "year": null, "review": "bad", "details": "not recommend" }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 3, "year": null, "review": "good", "details": null }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 4, "year": 2018, "review": "good", "details": "recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 5, "year": 2018, "review": "", "details": "not recommend" }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
+{ "id": 6, "year": 2018, "review": "good", "details": null }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/tsv/000/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/tsv/000/external_dataset.001.adm
deleted file mode 100644
index 1954b05..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/tsv/000/external_dataset.001.adm
+++ /dev/null
@@ -1,6 +0,0 @@
-{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
-{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
-{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
-{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
-{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
-{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/tsv/gz/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/tsv/gz/external_dataset.001.adm
new file mode 100644
index 0000000..e04a6b5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/tsv/gz/external_dataset.001.adm
@@ -0,0 +1,30 @@
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/tsv/mixed/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/tsv/mixed/external_dataset.001.adm
new file mode 100644
index 0000000..4823286
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/tsv/mixed/external_dataset.001.adm
@@ -0,0 +1,60 @@
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/tsv/tsv/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/tsv/tsv/external_dataset.001.adm
new file mode 100644
index 0000000..e04a6b5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/tsv/tsv/external_dataset.001.adm
@@ -0,0 +1,30 @@
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 1, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 2, "year": null, "review": "\"bad\"", "details": "\"not recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 3, "year": null, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 4, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 5, "year": 2018, "review": "", "details": "\"not recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
+{ "id": 6, "year": 2018, "review": "\"good\"", "details": "\"recommend\"" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
index 007f194..02846d1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
@@ -35,15 +35,31 @@
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/csv/000">
- <output-dir compare="Text">aws/s3/csv/000</output-dir>
+ <compilation-unit name="aws/s3/csv/csv">
+ <output-dir compare="Text">aws/s3/csv/csv</output-dir>
+ </compilation-unit>
+ </test-case><test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/csv/gz">
+ <output-dir compare="Text">aws/s3/csv/gz</output-dir>
+ </compilation-unit>
+ </test-case><test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/csv/mixed">
+ <output-dir compare="Text">aws/s3/csv/mixed</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/tsv/000">
- <output-dir compare="Text">aws/s3/tsv/000</output-dir>
+ <compilation-unit name="aws/s3/tsv/tsv">
+ <output-dir compare="Text">aws/s3/tsv/tsv</output-dir>
</compilation-unit>
- </test-case>
+ </test-case><test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/tsv/gz">
+ <output-dir compare="Text">aws/s3/tsv/gz</output-dir>
+ </compilation-unit>
+ </test-case><test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/tsv/mixed">
+ <output-dir compare="Text">aws/s3/tsv/mixed</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-dataset">
<compilation-unit name="aws/s3/negative">
<output-dir compare="Text">aws/s3/negative</output-dir>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
index 315327f..d725687 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
@@ -78,13 +78,16 @@
GetObjectRequest.Builder getObjectBuilder = GetObjectRequest.builder();
GetObjectRequest getObjectRequest = getObjectBuilder.bucket(bucket).key(filePaths.get(nextFileIndex)).build();
- // Use the proper input stream
+ // Have a reference to the S3 stream to ensure that if GZipInputStream causes an IOException because of reading
+ // the header, then the S3 stream gets closed in the close method
+ in = s3Client.getObject(getObjectRequest);
+
+ // Use gzip stream if needed
String filename = filePaths.get(nextFileIndex).toLowerCase();
if (filename.endsWith(".gz") || filename.endsWith(".gzip")) {
in = new GZIPInputStream(s3Client.getObject(getObjectRequest), ExternalDataConstants.DEFAULT_BUFFER_SIZE);
- } else {
- in = s3Client.getObject(getObjectRequest);
}
+
if (notificationHandler != null) {
notificationHandler.notifyNewSource();
}