[ASTERIXDB-2718][EXT] Add more test cases to cover complex JSON formats
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Added test cases for JSON single line, multiple lines, arrays, nested
objects, ... etc.
Change-Id: I3cdcd2a3f417b9a045067c2fb50249c3bdf02211
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/5923
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
Reviewed-by: Dmitry Lychagin <dmitry.lychagin@couchbase.com>
diff --git a/asterixdb/asterix-app/data/json/multi-lines-with-arrays/5-records.json b/asterixdb/asterix-app/data/json/multi-lines-with-arrays/5-records.json
new file mode 100644
index 0000000..fd8526b
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/multi-lines-with-arrays/5-records.json
@@ -0,0 +1,19 @@
+{"id": 1, "year": null, "quarter": null, "review": "good", "array": [1, 2, 3]}
+
+
+{"id": 2, "year": null, "quarter": null, "review": "good", "array": [1, [1, 2], [1]]}
+{"id": 3, "year": 2018, "quarter": null, "review": "good", "array": [1,
+2, 3]}
+
+{"id": 4,
+ "year": 2018,
+ "quarter": null,
+ "review": "bad",
+ "array": [
+ 1,
+ 2,
+ 3
+ ]
+}
+{"id": 5,
+ "year": 2018, "quarter": 1, "review": "good", "array": [1, 2, 3]}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/multi-lines-with-nested-objects/5-records.json b/asterixdb/asterix-app/data/json/multi-lines-with-nested-objects/5-records.json
new file mode 100644
index 0000000..b89e81e
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/multi-lines-with-nested-objects/5-records.json
@@ -0,0 +1,32 @@
+{"id": 1, "year": null, "quarter": null, "review": "good", "array": [1, 2, 3], "nested": { "id" : 1}}
+
+
+{"id": 2, "year": null, "quarter": null, "review": "good", "array": [1, [1, 2], [1]], "nested": { "id" : 1}, "nested2": [ {"id":1 }]}
+{"id":3,"year":2018,"quarter":null,"review":"good","array":[1,2,3,{"nested":{"array":[{"nested":{"array":[1,2]}}]}}]}
+
+{
+ "id": 4,
+ "year": 2018,
+ "quarter": null,
+ "review": "bad",
+ "array": [
+ 1,
+ 2,
+ 3,
+ {
+ "nested1": {
+ "id": 1,
+ "nested2": {
+ "id": 2,
+ "nested3": [
+ {
+ "nested4": null
+ }
+ ]
+ }
+ }
+ }
+ ]
+}
+{"id": 5,
+ "year": 2018, "quarter": 1, "review": "good", "array": [1, 2, 3, {"nested": { "array": [1, 2]}}]}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/multi-lines/20-records.json b/asterixdb/asterix-app/data/json/multi-lines/20-records.json
new file mode 100644
index 0000000..0591790
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/multi-lines/20-records.json
@@ -0,0 +1,34 @@
+{"id": 1, "year": null, "quarter": null, "review": "good"}
+
+
+{"id": 2, "year": null, "quarter": null, "review": "good"}
+{"id": 3, "year": 2018, "quarter": null, "review": "good"}
+
+{"id": 4,
+ "year": 2018,
+ "quarter": null,
+ "review": "bad"
+}
+{"id": 5,
+ "year": 2018, "quarter": 1, "review": "good"}
+{"id": 6, "year": 2018, "quarter": 1, "review": "bad"
+}
+{"id": 7, "year": 2018, "quarter": 2, "review": "good"}
+{"id": 8, "year": 2018, "quarter": 2, "review": "bad"}
+{"id": 9, "year": 2019, "quarter": null,
+
+ "review": "good"}
+{"id": 10, "year": 2019,
+
+ "quarter": null,
+ "review": "bad"}
+{"id": 11, "year": 2019, "quarter": 1, "review": "good"}
+{"id": 12, "year": 2019, "quarter": 1, "review": "bad"}
+{"id": 13, "year": 2019, "quarter": 2, "review": "good"}
+{"id": 14, "year": 2019, "quarter": 2, "review": "bad"}
+{"id": 15, "year": 2019, "quarter": null, "review": "good"}
+{"id": 16, "year": 2019, "quarter": null, "review": "bad"}
+{"id": 17, "year": 2019, "quarter": 1, "review": "good"}
+{"id": 18, "year": 2019, "quarter": 1, "review": "bad"}
+{"id": 19, "year": 2019, "quarter": 2, "review": "good"}
+{"id": 20, "year": 2019, "quarter": 2, "review": "bad"}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/single-line/20-records.json b/asterixdb/asterix-app/data/json/single-line/20-records.json
new file mode 100644
index 0000000..01176da
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/single-line/20-records.json
@@ -0,0 +1,20 @@
+{"id": 1, "year": null, "quarter": null, "review": "good"}
+{"id": 2, "year": null, "quarter": null, "review": "good"}
+{"id": 3, "year": 2018, "quarter": null, "review": "good"}
+{"id": 4, "year": 2018, "quarter": null, "review": "bad"}
+{"id": 5, "year": 2018, "quarter": 1, "review": "good"}
+{"id": 6, "year": 2018, "quarter": 1, "review": "bad"}
+{"id": 7, "year": 2018, "quarter": 2, "review": "good"}
+{"id": 8, "year": 2018, "quarter": 2, "review": "bad"}
+{"id": 9, "year": 2019, "quarter": null, "review": "good"}
+{"id": 10, "year": 2019, "quarter": null, "review": "bad"}
+{"id": 11, "year": 2019, "quarter": 1, "review": "good"}
+{"id": 12, "year": 2019, "quarter": 1, "review": "bad"}
+{"id": 13, "year": 2019, "quarter": 2, "review": "good"}
+{"id": 14, "year": 2019, "quarter": 2, "review": "bad"}
+{"id": 15, "year": 2019, "quarter": null, "review": "good"}
+{"id": 16, "year": 2019, "quarter": null, "review": "bad"}
+{"id": 17, "year": 2019, "quarter": 1, "review": "good"}
+{"id": 18, "year": 2019, "quarter": 1, "review": "bad"}
+{"id": 19, "year": 2019, "quarter": 2, "review": "good"}
+{"id": 20, "year": 2019, "quarter": 2, "review": "bad"}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
index 6ac1259..88cd6b5 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
@@ -22,8 +22,6 @@
import org.apache.asterix.test.runtime.LangExecutionUtil;
import org.apache.asterix.testframework.context.TestCaseContext;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
import org.junit.FixMethodOrder;
import org.junit.runner.RunWith;
import org.junit.runners.MethodSorters;
@@ -36,20 +34,19 @@
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class AwsS3ExternalDatasetOnePartitionTest extends AwsS3ExternalDatasetTest {
- private static final Logger LOGGER = LogManager.getLogger();
- private static final String SUITE_PATH = "testsuite_external_dataset_one_partition.xml";
-
- @Parameterized.Parameters(name = "SqlppExecutionTest {index}: {0}")
- public static Collection<Object[]> tests() throws Exception {
- TEST_CONFIG_FILE_NAME = "src/test/resources/cc-single.conf";
- PREPARE_S3_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareS3Bucket;
- return LangExecutionUtil.tests("only_external_dataset.xml", SUITE_PATH);
- }
-
public AwsS3ExternalDatasetOnePartitionTest(TestCaseContext tcCtx) {
super(tcCtx);
}
+ @Parameterized.Parameters(name = "AwsS3ExternalDatasetOnePartitionTest {index}: {0}")
+ public static Collection<Object[]> tests() throws Exception {
+ SUITE_TESTS = "testsuite_external_dataset_one_partition.xml";
+ ONLY_TESTS = "only_external_dataset.xml";
+ TEST_CONFIG_FILE_NAME = "src/test/resources/cc-single.conf";
+ PREPARE_S3_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareS3Bucket;
+ return LangExecutionUtil.tests(ONLY_TESTS, SUITE_TESTS);
+ }
+
private static void prepareS3Bucket() {
}
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index b3fff7a..6ff59ee 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -24,6 +24,7 @@
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URI;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collection;
import java.util.HashMap;
@@ -73,28 +74,43 @@
private static final Logger LOGGER = LogManager.getLogger();
- protected static String TEST_CONFIG_FILE_NAME;
+ // subclasses of this class MUST instantiate these variables before using them to avoid unexpected behavior
+ static String SUITE_TESTS;
+ static String ONLY_TESTS;
+ static String TEST_CONFIG_FILE_NAME;
static Runnable PREPARE_S3_BUCKET;
- // S3 mock server
- private static S3Mock s3MockServer;
-
- // IMPORTANT: The following values must be used in the AWS S3 test case
- private static S3Client client;
- private static final String S3_MOCK_SERVER_BUCKET = "playground";
- private static final String S3_MOCK_SERVER_BUCKET_DEFINITION = "json-data/reviews/"; // data resides here
- private static final String S3_MOCK_SERVER_BUCKET_CSV_DEFINITION = "csv-data/reviews/"; // data resides here
- private static final String S3_MOCK_SERVER_BUCKET_TSV_DEFINITION = "tsv-data/reviews/"; // data resides here
- private static final String S3_MOCK_SERVER_REGION = "us-west-2";
- private static final int S3_MOCK_SERVER_PORT = 8001;
- private static final String S3_MOCK_SERVER_HOSTNAME = "http://localhost:" + S3_MOCK_SERVER_PORT;
+ // Base directory paths for data files
+ private static final String JSON_DATA_PATH = joinPath("data", "json");
private static final String CSV_DATA_PATH = joinPath("data", "csv");
private static final String TSV_DATA_PATH = joinPath("data", "tsv");
+
+ // IMPORTANT: The following values must be used in the AWS S3 test case
+ private static S3Mock s3MockServer;
+ private static S3Client client;
+
+ // Service endpoint
+ private static final int S3_MOCK_SERVER_PORT = 8001;
+ private static final String S3_MOCK_SERVER_HOSTNAME = "http://localhost:" + S3_MOCK_SERVER_PORT;
+
+ // Region, bucket and definitions
+ private static final String S3_MOCK_SERVER_REGION = "us-west-2";
+ private static final String S3_MOCK_SERVER_BUCKET = "playground";
+ private static final String S3_MOCK_SERVER_BUCKET_JSON_DEFINITION = "json-data/reviews/"; // data resides here
+ private static final String S3_MOCK_SERVER_BUCKET_CSV_DEFINITION = "csv-data/reviews/"; // data resides here
+ private static final String S3_MOCK_SERVER_BUCKET_TSV_DEFINITION = "tsv-data/reviews/"; // data resides here
+
private static final Set<String> fileNames = new HashSet<>();
private static final CreateBucketRequest.Builder CREATE_BUCKET_BUILDER = CreateBucketRequest.builder();
private static final DeleteBucketRequest.Builder DELETE_BUCKET_BUILDER = DeleteBucketRequest.builder();
private static final PutObjectRequest.Builder PUT_OBJECT_BUILDER = PutObjectRequest.builder();
+ protected TestCaseContext tcCtx;
+
+ public AwsS3ExternalDatasetTest(TestCaseContext tcCtx) {
+ this.tcCtx = tcCtx;
+ }
+
@BeforeClass
public static void setUp() throws Exception {
final TestExecutor testExecutor = new AwsTestExecutor();
@@ -118,17 +134,13 @@
LOGGER.info("S3 mock down and client shut down successfully");
}
- @Parameters(name = "SqlppExecutionTest {index}: {0}")
+ @Parameters(name = "AwsS3ExternalDatasetTest {index}: {0}")
public static Collection<Object[]> tests() throws Exception {
+ SUITE_TESTS = "testsuite_external_dataset.xml";
+ ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
PREPARE_S3_BUCKET = AwsS3ExternalDatasetTest::prepareS3Bucket;
- return LangExecutionUtil.tests("only_external_dataset.xml", "testsuite_external_dataset.xml");
- }
-
- protected TestCaseContext tcCtx;
-
- public AwsS3ExternalDatasetTest(TestCaseContext tcCtx) {
- this.tcCtx = tcCtx;
+ return LangExecutionUtil.tests(ONLY_TESTS, SUITE_TESTS);
}
@Test
@@ -180,64 +192,58 @@
client.createBucket(CreateBucketRequest.builder().bucket(S3_MOCK_SERVER_BUCKET).build());
LOGGER.info("bucket created successfully");
- LOGGER.info("Adding JSON files to the bucket");
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "0.json").build(),
- RequestBody.fromString("{\"id\": 1, \"year\": null, \"quarter\": null, \"review\": \"good\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "1.json").build(),
- RequestBody.fromString("{\"id\": 2, \"year\": null, \"quarter\": null, \"review\": \"good\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2018/1.json").build(),
- RequestBody.fromString("{\"id\": 3, \"year\": 2018, \"quarter\": null, \"review\": \"good\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2018/2.json").build(),
- RequestBody.fromString("{\"id\": 4, \"year\": 2018, \"quarter\": null, \"review\": \"bad\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2018/q1/1.json").build(),
- RequestBody.fromString("{\"id\": 5, \"year\": 2018, \"quarter\": 1, \"review\": \"good\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2018/q1/2.json").build(),
- RequestBody.fromString("{\"id\": 6, \"year\": 2018, \"quarter\": 1, \"review\": \"bad\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2018/q2/1.json").build(),
- RequestBody.fromString("{\"id\": 7, \"year\": 2018, \"quarter\": 2, \"review\": \"good\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2018/q2/2.json").build(),
- RequestBody.fromString("{\"id\": 8, \"year\": 2018, \"quarter\": 2, \"review\": \"bad\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2019/1.json").build(),
- RequestBody.fromString("{\"id\": 9, \"year\": 2019, \"quarter\": null, \"review\": \"good\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2019/2.json").build(),
- RequestBody.fromString("{\"id\": 10, \"year\": 2019, \"quarter\": null, \"review\": \"bad\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2019/q1/1.json").build(),
- RequestBody.fromString("{\"id\": 11, \"year\": 2019, \"quarter\": 1, \"review\": \"good\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2019/q1/2.json").build(),
- RequestBody.fromString("{\"id\": 12, \"year\": 2019, \"quarter\": 1, \"review\": \"bad\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2019/q2/1.json").build(),
- RequestBody.fromString("{\"id\": 13, \"year\": 2019, \"quarter\": 2, \"review\": \"good\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2019/q2/2.json").build(),
- RequestBody.fromString("{\"id\": 14, \"year\": 2019, \"quarter\": 2, \"review\": \"bad\"}"));
+ // Load JSON files
+ loadJsonFiles();
+ loadCsvFiles();
+ loadTsvFiles();
+ LOGGER.info("Files added successfully");
+ }
+
+ private static void loadJsonFiles() {
+ LOGGER.info("Adding JSON files to the bucket");
+
+ // Set the bucket
+ PutObjectRequest.Builder builder1 = PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET);
+
+ // load multi-level single line JSON files
+ String singleLineBasePath = S3_MOCK_SERVER_BUCKET_JSON_DEFINITION + "single-line/";
+ Path filePath1 = Paths.get(JSON_DATA_PATH, "single-line", "20-records.json");
+ RequestBody reqBody1 = RequestBody.fromFile(filePath1);
+ client.putObject(builder1.key(singleLineBasePath + "20-records.json").build(), reqBody1);
+ client.putObject(builder1.key(singleLineBasePath + "level1a/" + "20-records.json").build(), reqBody1);
+ client.putObject(builder1.key(singleLineBasePath + "level1b/" + "20-records.json").build(), reqBody1);
+ client.putObject(builder1.key(singleLineBasePath + "level1a/level2a/" + "20-records.json").build(), reqBody1);
+ client.putObject(builder1.key(singleLineBasePath + "level1a/level2b/" + "20-records.json").build(), reqBody1);
+
+ // Load multi-level multi-lines JSON files
+ String multiLinesBasePath = S3_MOCK_SERVER_BUCKET_JSON_DEFINITION + "multi-lines/";
+ Path filePath2 = Paths.get(JSON_DATA_PATH, "multi-lines", "20-records.json");
+ RequestBody reqBody2 = RequestBody.fromFile(filePath2);
+ client.putObject(builder1.key(multiLinesBasePath + "20-records.json").build(), reqBody2);
+ client.putObject(builder1.key(multiLinesBasePath + "level1a/" + "20-records.json").build(), reqBody2);
+ client.putObject(builder1.key(multiLinesBasePath + "level1b/" + "20-records.json").build(), reqBody2);
+ client.putObject(builder1.key(multiLinesBasePath + "level1a/level2a/" + "20-records.json").build(), reqBody2);
+ client.putObject(builder1.key(multiLinesBasePath + "level1a/level2b/" + "20-records.json").build(), reqBody2);
+
+ // Load multi-level multi-lines with array JSON files
+ String multiLinesWithArraysBasePath = S3_MOCK_SERVER_BUCKET_JSON_DEFINITION + "multi-lines-with-arrays/";
+ Path filePath3 = Paths.get(JSON_DATA_PATH, "multi-lines-with-arrays", "5-records.json");
+ RequestBody reqBody3 = RequestBody.fromFile(filePath3);
+ client.putObject(builder1.key(multiLinesWithArraysBasePath + "5-records.json").build(), reqBody3);
+ client.putObject(builder1.key(multiLinesWithArraysBasePath + "level1a/" + "5-records.json").build(), reqBody3);
+
+ // Load multi-level multi-lines with nested objects JSON files
+ String multiLinesWithNestedObjectsBasePath =
+ S3_MOCK_SERVER_BUCKET_JSON_DEFINITION + "multi-lines-with-nested-objects/";
+ Path filePath4 = Paths.get(JSON_DATA_PATH, "multi-lines-with-nested-objects", "5-records.json");
+ RequestBody reqBody4 = RequestBody.fromFile(filePath4);
+ client.putObject(builder1.key(multiLinesWithNestedObjectsBasePath + "5-records.json").build(), reqBody4);
+ client.putObject(builder1.key(multiLinesWithNestedObjectsBasePath + "level1a/" + "5-records.json").build(),
+ reqBody4);
+ }
+
+ private static void loadCsvFiles() {
LOGGER.info("Adding CSV files to the bucket");
client.putObject(
PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
@@ -247,7 +253,9 @@
PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
.key(S3_MOCK_SERVER_BUCKET_CSV_DEFINITION + "2018/01.csv").build(),
RequestBody.fromFile(Paths.get(CSV_DATA_PATH, "02.csv")));
+ }
+ private static void loadTsvFiles() {
LOGGER.info("Adding TSV files to the bucket");
client.putObject(
PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.000.ddl.sqlpp
deleted file mode 100644
index 9c6a994..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.000.ddl.sqlpp
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKey"="dummyAccessKey"),
-("secretKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="playground"),
-("definition"="json-data/reviews"),
-("format"="json")
-);
-
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/002/query-dataset.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/002/query-dataset.002.query.sqlpp
deleted file mode 100644
index 6e31eb3..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/002/query-dataset.002.query.sqlpp
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-USE test;
-
-FROM test SELECT VALUE test ORDER BY id ASC;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/002/query-dataset.003.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/002/query-dataset.003.ddl.sqlpp
deleted file mode 100644
index 0ff713d..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/002/query-dataset.003.ddl.sqlpp
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-DROP DATASET test IF EXISTS;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/001/query-dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.000.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/001/query-dataset.000.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.000.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/001/query-dataset.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.002.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/001/query-dataset.002.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.002.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/001/query-dataset.003.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.003.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/001/query-dataset.003.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.003.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.000.ddl.sqlpp
new file mode 100644
index 0000000..8d084a1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.000.ddl.sqlpp
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test1 if exists;
+create external dataset test1(test) using S3 (
+("accessKey"="dummyAccessKey"),
+("secretKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/single-line"),
+("format"="json")
+);
+
+drop dataset test2 if exists;
+create external dataset test2(test) using S3 (
+("accessKey"="dummyAccessKey"),
+("secretKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/multi-lines"),
+("format"="json")
+);
+
+drop dataset test3 if exists;
+create external dataset test3(test) using S3 (
+("accessKey"="dummyAccessKey"),
+("secretKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/multi-lines-with-arrays"),
+("format"="json")
+);
+
+drop dataset test4 if exists;
+create external dataset test4(test) using S3 (
+("accessKey"="dummyAccessKey"),
+("secretKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/multi-lines-with-nested-objects"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.001.query.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.001.query.sqlpp
index 2dd9cc5..b72e741 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.001.query.sqlpp
@@ -19,7 +19,5 @@
use test;
-from test
-select value test
-order by id asc;
+select count(*) `count` from test1;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.002.query.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.002.query.sqlpp
index 2dd9cc5..c3e883e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.002.query.sqlpp
@@ -19,7 +19,5 @@
use test;
-from test
-select value test
-order by id asc;
+select count(*) `count` from test2;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.003.query.sqlpp
similarity index 94%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.003.query.sqlpp
index 2dd9cc5..e3d64ff 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.003.query.sqlpp
@@ -19,7 +19,5 @@
use test;
-from test
-select value test
-order by id asc;
+select count(*) `count` from test3;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.004.query.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.004.query.sqlpp
index 2dd9cc5..1fda76a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.004.query.sqlpp
@@ -19,7 +19,5 @@
use test;
-from test
-select value test
-order by id asc;
+select value test3 from test3 order by id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.005.query.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.005.query.sqlpp
index 2dd9cc5..2b9b5cb 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.005.query.sqlpp
@@ -19,7 +19,5 @@
use test;
-from test
-select value test
-order by id asc;
+select count(*) `count` from test4;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.006.query.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.006.query.sqlpp
index 2dd9cc5..2cd184d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.006.query.sqlpp
@@ -19,7 +19,5 @@
use test;
-from test
-select value test
-order by id asc;
+select value test4 from test4 order by id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.002.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.007.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.002.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.007.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/002/query-dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.000.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/002/query-dataset.000.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.000.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/001/query-dataset.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.002.query.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/001/query-dataset.002.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.002.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/001/query-dataset.003.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.003.ddl.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/001/query-dataset.003.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.003.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/000/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/000/external_dataset.001.adm
deleted file mode 100644
index a7ce908..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/000/external_dataset.001.adm
+++ /dev/null
@@ -1,14 +0,0 @@
-{ "id": 1, "year": null, "quarter": null, "review": "good" }
-{ "id": 2, "year": null, "quarter": null, "review": "good" }
-{ "id": 3, "year": 2018, "quarter": null, "review": "good" }
-{ "id": 4, "year": 2018, "quarter": null, "review": "bad" }
-{ "id": 5, "year": 2018, "quarter": 1, "review": "good" }
-{ "id": 6, "year": 2018, "quarter": 1, "review": "bad" }
-{ "id": 7, "year": 2018, "quarter": 2, "review": "good" }
-{ "id": 8, "year": 2018, "quarter": 2, "review": "bad" }
-{ "id": 9, "year": 2019, "quarter": null, "review": "good" }
-{ "id": 10, "year": 2019, "quarter": null, "review": "bad" }
-{ "id": 11, "year": 2019, "quarter": 1, "review": "good" }
-{ "id": 12, "year": 2019, "quarter": 1, "review": "bad" }
-{ "id": 13, "year": 2019, "quarter": 2, "review": "good" }
-{ "id": 14, "year": 2019, "quarter": 2, "review": "bad" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/001/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/csv/000/external_dataset.001.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/001/external_dataset.001.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/csv/000/external_dataset.001.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.002.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.002.adm
new file mode 100644
index 0000000..187a8cb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.002.adm
@@ -0,0 +1 @@
+{ "count": 100 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.003.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.003.adm
new file mode 100644
index 0000000..187a8cb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.003.adm
@@ -0,0 +1 @@
+{ "count": 100 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.004.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.004.adm
new file mode 100644
index 0000000..d10a16c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.004.adm
@@ -0,0 +1 @@
+{ "count": 10 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.005.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.005.adm
new file mode 100644
index 0000000..a1881c3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.005.adm
@@ -0,0 +1,10 @@
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3 ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3 ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.006.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.006.adm
new file mode 100644
index 0000000..d10a16c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.006.adm
@@ -0,0 +1 @@
+{ "count": 10 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.007.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.007.adm
new file mode 100644
index 0000000..d24a08b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.007.adm
@@ -0,0 +1,10 @@
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ], "nested": { "id": 1 } }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ], "nested": { "id": 1 } }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ], "nested": { "id": 1 }, "nested2": [ { "id": 1 } ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ], "nested": { "id": 1 }, "nested2": [ { "id": 1 } ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ { "nested": { "array": [ 1, 2 ] } } ] } } ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ { "nested": { "array": [ 1, 2 ] } } ] } } ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3, { "nested1": { "id": 1, "nested2": { "id": 2, "nested3": [ { "nested4": null } ] } } } ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3, { "nested1": { "id": 1, "nested2": { "id": 2, "nested3": [ { "nested4": null } ] } } } ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ 1, 2 ] } } ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ 1, 2 ] } } ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/002/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/tsv/000/external_dataset.001.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/002/external_dataset.001.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/tsv/000/external_dataset.001.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
index 2456f13..a6774fa 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
@@ -20,18 +20,18 @@
<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
<test-group name="external-dataset">
<test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/000">
- <output-dir compare="Text">aws/s3/000</output-dir>
+ <compilation-unit name="aws/s3/json/000">
+ <output-dir compare="Text">aws/s3/json/000</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/001">
- <output-dir compare="Text">aws/s3/001</output-dir>
+ <compilation-unit name="aws/s3/csv/000">
+ <output-dir compare="Text">aws/s3/csv/000</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/002">
- <output-dir compare="Text">aws/s3/002</output-dir>
+ <compilation-unit name="aws/s3/tsv/000">
+ <output-dir compare="Text">aws/s3/tsv/000</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset">