[ASTERIXDB-3255][RT] Embed filter values in CSV/TSV
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Add the support to embed filter values when
parsing CSV files..
Change-Id: I611e2117cc68d7f1f2c5c5bcf1c97947f20043cb
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17779
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/1/maintenance-report/year-2001-month-01-day-01-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/1/maintenance-report/year-2001-month-01-day-01-date/0.csv
new file mode 100644
index 0000000..f8fd00a
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/1/maintenance-report/year-2001-month-01-day-01-date/0.csv
@@ -0,0 +1,3 @@
+1,2001,1,1
+1,2001,1,1
+1,2001,1,1
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/1/maintenance-report/year-2002-month-02-day-02-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/1/maintenance-report/year-2002-month-02-day-02-date/0.csv
new file mode 100644
index 0000000..bff3930
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/1/maintenance-report/year-2002-month-02-day-02-date/0.csv
@@ -0,0 +1,3 @@
+1,2002,2,2
+1,2002,2,2
+1,2002,2,2
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/1/maintenance-report/year-2003-month-03-day-03-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/1/maintenance-report/year-2003-month-03-day-03-date/0.csv
new file mode 100644
index 0000000..e0e1ef6
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/1/maintenance-report/year-2003-month-03-day-03-date/0.csv
@@ -0,0 +1,3 @@
+1,2003,3,3
+1,2003,3,3
+1,2003,3,3
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/2/maintenance-report/year-2001-month-01-day-01-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/2/maintenance-report/year-2001-month-01-day-01-date/0.csv
new file mode 100644
index 0000000..b70858d
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/2/maintenance-report/year-2001-month-01-day-01-date/0.csv
@@ -0,0 +1,3 @@
+2,2001,1,1
+2,2001,1,1
+2,2001,1,1
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/2/maintenance-report/year-2002-month-02-day-02-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/2/maintenance-report/year-2002-month-02-day-02-date/0.csv
new file mode 100644
index 0000000..583c101
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/2/maintenance-report/year-2002-month-02-day-02-date/0.csv
@@ -0,0 +1,3 @@
+2,2002,2,2
+2,2002,2,2
+2,2002,2,2
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/2/maintenance-report/year-2003-month-03-day-03-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/2/maintenance-report/year-2003-month-03-day-03-date/0.csv
new file mode 100644
index 0000000..0151182
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/2/maintenance-report/year-2003-month-03-day-03-date/0.csv
@@ -0,0 +1,3 @@
+2,2003,3,3
+2,2003,3,3
+2,2003,3,3
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/3/maintenance-report/year-2001-month-01-day-01-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/3/maintenance-report/year-2001-month-01-day-01-date/0.csv
new file mode 100644
index 0000000..7db333d
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/3/maintenance-report/year-2001-month-01-day-01-date/0.csv
@@ -0,0 +1,3 @@
+3,2001,1,1
+3,2001,1,1
+3,2001,1,1
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/3/maintenance-report/year-2002-month-02-day-02-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/3/maintenance-report/year-2002-month-02-day-02-date/0.csv
new file mode 100644
index 0000000..cdc768d
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/3/maintenance-report/year-2002-month-02-day-02-date/0.csv
@@ -0,0 +1,3 @@
+3,2002,2,2
+3,2002,2,2
+3,2002,2,2
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/3/maintenance-report/year-2003-month-03-day-03-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/3/maintenance-report/year-2003-month-03-day-03-date/0.csv
new file mode 100644
index 0000000..ddd7c28
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/ford/customer/3/maintenance-report/year-2003-month-03-day-03-date/0.csv
@@ -0,0 +1,3 @@
+3,2003,3,3
+3,2003,3,3
+3,2003,3,3
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/1/maintenance-report/year-2001-month-01-day-01-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/1/maintenance-report/year-2001-month-01-day-01-date/0.csv
new file mode 100644
index 0000000..f8fd00a
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/1/maintenance-report/year-2001-month-01-day-01-date/0.csv
@@ -0,0 +1,3 @@
+1,2001,1,1
+1,2001,1,1
+1,2001,1,1
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/1/maintenance-report/year-2002-month-02-day-02-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/1/maintenance-report/year-2002-month-02-day-02-date/0.csv
new file mode 100644
index 0000000..bff3930
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/1/maintenance-report/year-2002-month-02-day-02-date/0.csv
@@ -0,0 +1,3 @@
+1,2002,2,2
+1,2002,2,2
+1,2002,2,2
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/1/maintenance-report/year-2003-month-03-day-03-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/1/maintenance-report/year-2003-month-03-day-03-date/0.csv
new file mode 100644
index 0000000..e0e1ef6
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/1/maintenance-report/year-2003-month-03-day-03-date/0.csv
@@ -0,0 +1,3 @@
+1,2003,3,3
+1,2003,3,3
+1,2003,3,3
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/2/maintenance-report/year-2001-month-01-day-01-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/2/maintenance-report/year-2001-month-01-day-01-date/0.csv
new file mode 100644
index 0000000..b70858d
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/2/maintenance-report/year-2001-month-01-day-01-date/0.csv
@@ -0,0 +1,3 @@
+2,2001,1,1
+2,2001,1,1
+2,2001,1,1
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/2/maintenance-report/year-2002-month-02-day-02-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/2/maintenance-report/year-2002-month-02-day-02-date/0.csv
new file mode 100644
index 0000000..583c101
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/2/maintenance-report/year-2002-month-02-day-02-date/0.csv
@@ -0,0 +1,3 @@
+2,2002,2,2
+2,2002,2,2
+2,2002,2,2
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/2/maintenance-report/year-2003-month-03-day-03-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/2/maintenance-report/year-2003-month-03-day-03-date/0.csv
new file mode 100644
index 0000000..0151182
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/2/maintenance-report/year-2003-month-03-day-03-date/0.csv
@@ -0,0 +1,3 @@
+2,2003,3,3
+2,2003,3,3
+2,2003,3,3
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/3/maintenance-report/year-2001-month-01-day-01-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/3/maintenance-report/year-2001-month-01-day-01-date/0.csv
new file mode 100644
index 0000000..7db333d
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/3/maintenance-report/year-2001-month-01-day-01-date/0.csv
@@ -0,0 +1,3 @@
+3,2001,1,1
+3,2001,1,1
+3,2001,1,1
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/3/maintenance-report/year-2002-month-02-day-02-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/3/maintenance-report/year-2002-month-02-day-02-date/0.csv
new file mode 100644
index 0000000..cdc768d
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/3/maintenance-report/year-2002-month-02-day-02-date/0.csv
@@ -0,0 +1,3 @@
+3,2002,2,2
+3,2002,2,2
+3,2002,2,2
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/3/maintenance-report/year-2003-month-03-day-03-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/3/maintenance-report/year-2003-month-03-day-03-date/0.csv
new file mode 100644
index 0000000..ddd7c28
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/lexus/customer/3/maintenance-report/year-2003-month-03-day-03-date/0.csv
@@ -0,0 +1,3 @@
+3,2003,3,3
+3,2003,3,3
+3,2003,3,3
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/1/maintenance-report/year-2001-month-01-day-01-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/1/maintenance-report/year-2001-month-01-day-01-date/0.csv
new file mode 100644
index 0000000..f8fd00a
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/1/maintenance-report/year-2001-month-01-day-01-date/0.csv
@@ -0,0 +1,3 @@
+1,2001,1,1
+1,2001,1,1
+1,2001,1,1
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/1/maintenance-report/year-2002-month-02-day-02-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/1/maintenance-report/year-2002-month-02-day-02-date/0.csv
new file mode 100644
index 0000000..bff3930
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/1/maintenance-report/year-2002-month-02-day-02-date/0.csv
@@ -0,0 +1,3 @@
+1,2002,2,2
+1,2002,2,2
+1,2002,2,2
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/1/maintenance-report/year-2003-month-03-day-03-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/1/maintenance-report/year-2003-month-03-day-03-date/0.csv
new file mode 100644
index 0000000..e0e1ef6
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/1/maintenance-report/year-2003-month-03-day-03-date/0.csv
@@ -0,0 +1,3 @@
+1,2003,3,3
+1,2003,3,3
+1,2003,3,3
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/2/maintenance-report/year-2001-month-01-day-01-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/2/maintenance-report/year-2001-month-01-day-01-date/0.csv
new file mode 100644
index 0000000..b70858d
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/2/maintenance-report/year-2001-month-01-day-01-date/0.csv
@@ -0,0 +1,3 @@
+2,2001,1,1
+2,2001,1,1
+2,2001,1,1
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/2/maintenance-report/year-2002-month-02-day-02-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/2/maintenance-report/year-2002-month-02-day-02-date/0.csv
new file mode 100644
index 0000000..583c101
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/2/maintenance-report/year-2002-month-02-day-02-date/0.csv
@@ -0,0 +1,3 @@
+2,2002,2,2
+2,2002,2,2
+2,2002,2,2
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/2/maintenance-report/year-2003-month-03-day-03-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/2/maintenance-report/year-2003-month-03-day-03-date/0.csv
new file mode 100644
index 0000000..0151182
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/2/maintenance-report/year-2003-month-03-day-03-date/0.csv
@@ -0,0 +1,3 @@
+2,2003,3,3
+2,2003,3,3
+2,2003,3,3
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/3/maintenance-report/year-2001-month-01-day-01-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/3/maintenance-report/year-2001-month-01-day-01-date/0.csv
new file mode 100644
index 0000000..7db333d
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/3/maintenance-report/year-2001-month-01-day-01-date/0.csv
@@ -0,0 +1,3 @@
+3,2001,1,1
+3,2001,1,1
+3,2001,1,1
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/3/maintenance-report/year-2002-month-02-day-02-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/3/maintenance-report/year-2002-month-02-day-02-date/0.csv
new file mode 100644
index 0000000..cdc768d
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/3/maintenance-report/year-2002-month-02-day-02-date/0.csv
@@ -0,0 +1,3 @@
+3,2002,2,2
+3,2002,2,2
+3,2002,2,2
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/3/maintenance-report/year-2003-month-03-day-03-date/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/3/maintenance-report/year-2003-month-03-day-03-date/0.csv
new file mode 100644
index 0000000..ddd7c28
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/car/toyota/customer/3/maintenance-report/year-2003-month-03-day-03-date/0.csv
@@ -0,0 +1,3 @@
+3,2003,3,3
+3,2003,3,3
+3,2003,3,3
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/department/accounting/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/department/accounting/0.csv
new file mode 100644
index 0000000..bbfab7b
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/department/accounting/0.csv
@@ -0,0 +1,3 @@
+1,"John","Smith"
+2,"Mike","Jones"
+3,"Alex","Miller"
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/department/engineering/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/department/engineering/0.csv
new file mode 100644
index 0000000..092cdf5
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/department/engineering/0.csv
@@ -0,0 +1,3 @@
+4,"Tom","Smith"
+5,"Alice","Jones"
+6,"Sheila","Miller"
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/csv/external-filter/embed-csv/department/hr/0.csv b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/department/hr/0.csv
new file mode 100644
index 0000000..18dc1cc
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/external-filter/embed-csv/department/hr/0.csv
@@ -0,0 +1,3 @@
+7,"James","Smith"
+8,"David","Jones"
+9,"Noah","Miller"
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
index f61ccbe..0db9a2a 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
@@ -48,6 +48,7 @@
protected static final Logger LOGGER = LogManager.getLogger();
// Extension filters
private static final FilenameFilter JSON_FILTER = ((dir, name) -> name.endsWith(".json"));
+ private static final FilenameFilter CSV_FILTER = ((dir, name) -> name.endsWith(".csv"));
private static final FilenameFilter PARQUET_FILTER = ((dir, name) -> name.endsWith(".parquet"));
// Base directory paths for data files
@@ -256,6 +257,9 @@
loadData(dataBasePath, "", "02.csv", definition, definitionSegment, false);
loadGzData(dataBasePath, "", "01.csv", definition, definitionSegment, false);
loadGzData(dataBasePath, "", "02.csv", definition, definitionSegment, false);
+
+ // Load external filter directories and files
+ loadDirectory(dataBasePath, "external-filter", CSV_FILTER);
}
private static void loadTsvFiles() {
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-multiple-values/embed-multiple-values.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-multiple-values/embed-multiple-values.000.ddl.sqlpp
new file mode 100644
index 0000000..cf867f0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-multiple-values/embed-multiple-values.000.ddl.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE test AS {
+ customer_id:int,
+ year:int,
+ month:int,
+ day:int
+};
+
+CREATE EXTERNAL DATASET maintenance(test) USING %adapter% (
+ %template%,
+ ("container"="playground"),
+ ("definition"="external-filter/embed-csv/car/{company:string}/customer/{customer_id:int}/maintenance-report/year-{year:int}-month-{month:int}-day-{day:int}-date"),
+ ("embed-filter-values" = "true"),
+ ("header"="false"),
+ ("format"="csv")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-multiple-values/embed-multiple-values.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-multiple-values/embed-multiple-values.001.query.sqlpp
new file mode 100644
index 0000000..2907baa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-multiple-values/embed-multiple-values.001.query.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT value t
+FROM maintenance t
+ORDER BY t.customer_id,
+ t.company,
+ t.year,
+ t.month,
+ t.day
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-multiple-values/embed-multiple-values.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-multiple-values/embed-multiple-values.002.query.sqlpp
new file mode 100644
index 0000000..3c99576
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-multiple-values/embed-multiple-values.002.query.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT VALUE t
+FROM maintenance t
+WHERE t.company = "ford"
+ AND t.customer_id = 1
+ AND t.year = 2001
+ AND t.month = 01
+ AND t.day = 01;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.000.ddl.sqlpp
new file mode 100644
index 0000000..a143f60
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.000.ddl.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE TestType AS {
+ id: string,
+ first_name: string,
+ last_name:string
+};
+
+
+CREATE EXTERNAL DATASET Department(TestType) USING %adapter% (
+ %template%,
+ ("container"="playground"),
+ ("definition"="external-filter/embed-csv/department/{department:string}"),
+ ("embed-filter-values" = "true"),
+ ("header"="false"),
+ ("format"="csv")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.010.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.010.query.sqlpp
new file mode 100644
index 0000000..0034525
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.010.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.011.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.011.query.sqlpp
new file mode 100644
index 0000000..b5856ac
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.011.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.020.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.020.query.sqlpp
new file mode 100644
index 0000000..b1d2220
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.020.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ AND d.last_name = "Smith"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.021.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.021.query.sqlpp
new file mode 100644
index 0000000..9a8b1dc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.021.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ AND d.last_name = "Smith"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.030.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.030.query.sqlpp
new file mode 100644
index 0000000..f429f50
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.030.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ OR d.last_name = "Smith"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.031.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.031.query.sqlpp
new file mode 100644
index 0000000..bae5485
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.031.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ OR d.last_name = "Smith"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.040.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.040.query.sqlpp
new file mode 100644
index 0000000..e3e6ee5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.040.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM Department d
+WHERE uppercase(d.department) = "H" || "R"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.041.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.041.query.sqlpp
new file mode 100644
index 0000000..bfa65b5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.041.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM Department d
+WHERE uppercase(d.department) = "H" || "R"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.050.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.050.query.sqlpp
new file mode 100644
index 0000000..fb4ad2b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.050.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM Department d
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.051.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.051.query.sqlpp
new file mode 100644
index 0000000..fb4ad2b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/csv/embed-one-value/embed-one-value.051.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM Department d
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-multiple-values/embed-multiple-values.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-multiple-values/embed-multiple-values.001.adm
new file mode 100644
index 0000000..1fcdcfa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-multiple-values/embed-multiple-values.001.adm
@@ -0,0 +1,81 @@
+{ "customer_id": 1, "year": 2001, "month": 1, "day": 1, "company": "ford" }
+{ "customer_id": 1, "year": 2001, "month": 1, "day": 1, "company": "ford" }
+{ "customer_id": 1, "year": 2001, "month": 1, "day": 1, "company": "ford" }
+{ "customer_id": 1, "year": 2002, "month": 2, "day": 2, "company": "ford" }
+{ "customer_id": 1, "year": 2002, "month": 2, "day": 2, "company": "ford" }
+{ "customer_id": 1, "year": 2002, "month": 2, "day": 2, "company": "ford" }
+{ "customer_id": 1, "year": 2003, "month": 3, "day": 3, "company": "ford" }
+{ "customer_id": 1, "year": 2003, "month": 3, "day": 3, "company": "ford" }
+{ "customer_id": 1, "year": 2003, "month": 3, "day": 3, "company": "ford" }
+{ "customer_id": 1, "year": 2001, "month": 1, "day": 1, "company": "lexus" }
+{ "customer_id": 1, "year": 2001, "month": 1, "day": 1, "company": "lexus" }
+{ "customer_id": 1, "year": 2001, "month": 1, "day": 1, "company": "lexus" }
+{ "customer_id": 1, "year": 2002, "month": 2, "day": 2, "company": "lexus" }
+{ "customer_id": 1, "year": 2002, "month": 2, "day": 2, "company": "lexus" }
+{ "customer_id": 1, "year": 2002, "month": 2, "day": 2, "company": "lexus" }
+{ "customer_id": 1, "year": 2003, "month": 3, "day": 3, "company": "lexus" }
+{ "customer_id": 1, "year": 2003, "month": 3, "day": 3, "company": "lexus" }
+{ "customer_id": 1, "year": 2003, "month": 3, "day": 3, "company": "lexus" }
+{ "customer_id": 1, "year": 2001, "month": 1, "day": 1, "company": "toyota" }
+{ "customer_id": 1, "year": 2001, "month": 1, "day": 1, "company": "toyota" }
+{ "customer_id": 1, "year": 2001, "month": 1, "day": 1, "company": "toyota" }
+{ "customer_id": 1, "year": 2002, "month": 2, "day": 2, "company": "toyota" }
+{ "customer_id": 1, "year": 2002, "month": 2, "day": 2, "company": "toyota" }
+{ "customer_id": 1, "year": 2002, "month": 2, "day": 2, "company": "toyota" }
+{ "customer_id": 1, "year": 2003, "month": 3, "day": 3, "company": "toyota" }
+{ "customer_id": 1, "year": 2003, "month": 3, "day": 3, "company": "toyota" }
+{ "customer_id": 1, "year": 2003, "month": 3, "day": 3, "company": "toyota" }
+{ "customer_id": 2, "year": 2001, "month": 1, "day": 1, "company": "ford" }
+{ "customer_id": 2, "year": 2001, "month": 1, "day": 1, "company": "ford" }
+{ "customer_id": 2, "year": 2001, "month": 1, "day": 1, "company": "ford" }
+{ "customer_id": 2, "year": 2002, "month": 2, "day": 2, "company": "ford" }
+{ "customer_id": 2, "year": 2002, "month": 2, "day": 2, "company": "ford" }
+{ "customer_id": 2, "year": 2002, "month": 2, "day": 2, "company": "ford" }
+{ "customer_id": 2, "year": 2003, "month": 3, "day": 3, "company": "ford" }
+{ "customer_id": 2, "year": 2003, "month": 3, "day": 3, "company": "ford" }
+{ "customer_id": 2, "year": 2003, "month": 3, "day": 3, "company": "ford" }
+{ "customer_id": 2, "year": 2001, "month": 1, "day": 1, "company": "lexus" }
+{ "customer_id": 2, "year": 2001, "month": 1, "day": 1, "company": "lexus" }
+{ "customer_id": 2, "year": 2001, "month": 1, "day": 1, "company": "lexus" }
+{ "customer_id": 2, "year": 2002, "month": 2, "day": 2, "company": "lexus" }
+{ "customer_id": 2, "year": 2002, "month": 2, "day": 2, "company": "lexus" }
+{ "customer_id": 2, "year": 2002, "month": 2, "day": 2, "company": "lexus" }
+{ "customer_id": 2, "year": 2003, "month": 3, "day": 3, "company": "lexus" }
+{ "customer_id": 2, "year": 2003, "month": 3, "day": 3, "company": "lexus" }
+{ "customer_id": 2, "year": 2003, "month": 3, "day": 3, "company": "lexus" }
+{ "customer_id": 2, "year": 2001, "month": 1, "day": 1, "company": "toyota" }
+{ "customer_id": 2, "year": 2001, "month": 1, "day": 1, "company": "toyota" }
+{ "customer_id": 2, "year": 2001, "month": 1, "day": 1, "company": "toyota" }
+{ "customer_id": 2, "year": 2002, "month": 2, "day": 2, "company": "toyota" }
+{ "customer_id": 2, "year": 2002, "month": 2, "day": 2, "company": "toyota" }
+{ "customer_id": 2, "year": 2002, "month": 2, "day": 2, "company": "toyota" }
+{ "customer_id": 2, "year": 2003, "month": 3, "day": 3, "company": "toyota" }
+{ "customer_id": 2, "year": 2003, "month": 3, "day": 3, "company": "toyota" }
+{ "customer_id": 2, "year": 2003, "month": 3, "day": 3, "company": "toyota" }
+{ "customer_id": 3, "year": 2001, "month": 1, "day": 1, "company": "ford" }
+{ "customer_id": 3, "year": 2001, "month": 1, "day": 1, "company": "ford" }
+{ "customer_id": 3, "year": 2001, "month": 1, "day": 1, "company": "ford" }
+{ "customer_id": 3, "year": 2002, "month": 2, "day": 2, "company": "ford" }
+{ "customer_id": 3, "year": 2002, "month": 2, "day": 2, "company": "ford" }
+{ "customer_id": 3, "year": 2002, "month": 2, "day": 2, "company": "ford" }
+{ "customer_id": 3, "year": 2003, "month": 3, "day": 3, "company": "ford" }
+{ "customer_id": 3, "year": 2003, "month": 3, "day": 3, "company": "ford" }
+{ "customer_id": 3, "year": 2003, "month": 3, "day": 3, "company": "ford" }
+{ "customer_id": 3, "year": 2001, "month": 1, "day": 1, "company": "lexus" }
+{ "customer_id": 3, "year": 2001, "month": 1, "day": 1, "company": "lexus" }
+{ "customer_id": 3, "year": 2001, "month": 1, "day": 1, "company": "lexus" }
+{ "customer_id": 3, "year": 2002, "month": 2, "day": 2, "company": "lexus" }
+{ "customer_id": 3, "year": 2002, "month": 2, "day": 2, "company": "lexus" }
+{ "customer_id": 3, "year": 2002, "month": 2, "day": 2, "company": "lexus" }
+{ "customer_id": 3, "year": 2003, "month": 3, "day": 3, "company": "lexus" }
+{ "customer_id": 3, "year": 2003, "month": 3, "day": 3, "company": "lexus" }
+{ "customer_id": 3, "year": 2003, "month": 3, "day": 3, "company": "lexus" }
+{ "customer_id": 3, "year": 2001, "month": 1, "day": 1, "company": "toyota" }
+{ "customer_id": 3, "year": 2001, "month": 1, "day": 1, "company": "toyota" }
+{ "customer_id": 3, "year": 2001, "month": 1, "day": 1, "company": "toyota" }
+{ "customer_id": 3, "year": 2002, "month": 2, "day": 2, "company": "toyota" }
+{ "customer_id": 3, "year": 2002, "month": 2, "day": 2, "company": "toyota" }
+{ "customer_id": 3, "year": 2002, "month": 2, "day": 2, "company": "toyota" }
+{ "customer_id": 3, "year": 2003, "month": 3, "day": 3, "company": "toyota" }
+{ "customer_id": 3, "year": 2003, "month": 3, "day": 3, "company": "toyota" }
+{ "customer_id": 3, "year": 2003, "month": 3, "day": 3, "company": "toyota" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-multiple-values/embed-multiple-values.002.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-multiple-values/embed-multiple-values.002.adm
new file mode 100644
index 0000000..eff14d1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-multiple-values/embed-multiple-values.002.adm
@@ -0,0 +1,3 @@
+{ "customer_id": 1, "year": 2001, "month": 1, "day": 1, "company": "ford" }
+{ "customer_id": 1, "year": 2001, "month": 1, "day": 1, "company": "ford" }
+{ "customer_id": 1, "year": 2001, "month": 1, "day": 1, "company": "ford" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.010.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.010.adm
new file mode 100644
index 0000000..02b4c8f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.010.adm
@@ -0,0 +1,3 @@
+{ "id": "1", "first_name": "John", "last_name": "Smith", "department": "accounting" }
+{ "id": "2", "first_name": "Mike", "last_name": "Jones", "department": "accounting" }
+{ "id": "3", "first_name": "Alex", "last_name": "Miller", "department": "accounting" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.011.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.011.plan
new file mode 100644
index 0000000..855cae6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.011.plan
@@ -0,0 +1,24 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$17(ASC) ] |PARTITIONED|
+ order (ASC, $$17) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$17(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (eq($$d.getField("department"), "accounting")) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$17] <- [$$d.getField(0)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.Department prefix-filter on: eq($$d.getField("department"), "accounting") embed-filter-value: true [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.020.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.020.adm
new file mode 100644
index 0000000..e0f50dc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.020.adm
@@ -0,0 +1 @@
+{ "id": "1", "first_name": "John", "last_name": "Smith", "department": "accounting" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.021.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.021.plan
new file mode 100644
index 0000000..0a61174
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.021.plan
@@ -0,0 +1,24 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$21(ASC) ] |PARTITIONED|
+ order (ASC, $$21) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$21(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (and(eq($$d.getField("department"), "accounting"), eq($$d.getField(2), "Smith"))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$21] <- [$$d.getField(0)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.Department prefix-filter on: eq($$d.getField("department"), "accounting") embed-filter-value: true [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.030.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.030.adm
new file mode 100644
index 0000000..0549d5d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.030.adm
@@ -0,0 +1,5 @@
+{ "id": "1", "first_name": "John", "last_name": "Smith", "department": "accounting" }
+{ "id": "2", "first_name": "Mike", "last_name": "Jones", "department": "accounting" }
+{ "id": "3", "first_name": "Alex", "last_name": "Miller", "department": "accounting" }
+{ "id": "4", "first_name": "Tom", "last_name": "Smith", "department": "engineering" }
+{ "id": "7", "first_name": "James", "last_name": "Smith", "department": "hr" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.031.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.031.plan
new file mode 100644
index 0000000..c5a7c10
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.031.plan
@@ -0,0 +1,24 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$21(ASC) ] |PARTITIONED|
+ order (ASC, $$21) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$21(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (or(eq($$d.getField("department"), "accounting"), eq($$d.getField(2), "Smith"))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$21] <- [$$d.getField(0)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.Department embed-filter-value: true [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.040.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.040.adm
new file mode 100644
index 0000000..f0d4195
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.040.adm
@@ -0,0 +1,3 @@
+{ "id": "7", "first_name": "James", "last_name": "Smith", "department": "hr" }
+{ "id": "8", "first_name": "David", "last_name": "Jones", "department": "hr" }
+{ "id": "9", "first_name": "Noah", "last_name": "Miller", "department": "hr" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.041.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.041.plan
new file mode 100644
index 0000000..9e6cdd8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.041.plan
@@ -0,0 +1,24 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$20(ASC) ] |PARTITIONED|
+ order (ASC, $$20) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$20(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (eq(uppercase($$d.getField("department")), "HR")) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$20] <- [$$d.getField(0)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.Department prefix-filter on: eq(uppercase($$d.getField("department")), "HR") embed-filter-value: true [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.050.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.050.adm
new file mode 100644
index 0000000..d315b06
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.050.adm
@@ -0,0 +1,9 @@
+{ "id": "1", "first_name": "John", "last_name": "Smith", "department": "accounting" }
+{ "id": "2", "first_name": "Mike", "last_name": "Jones", "department": "accounting" }
+{ "id": "3", "first_name": "Alex", "last_name": "Miller", "department": "accounting" }
+{ "id": "4", "first_name": "Tom", "last_name": "Smith", "department": "engineering" }
+{ "id": "5", "first_name": "Alice", "last_name": "Jones", "department": "engineering" }
+{ "id": "6", "first_name": "Sheila", "last_name": "Miller", "department": "engineering" }
+{ "id": "7", "first_name": "James", "last_name": "Smith", "department": "hr" }
+{ "id": "8", "first_name": "David", "last_name": "Jones", "department": "hr" }
+{ "id": "9", "first_name": "Noah", "last_name": "Miller", "department": "hr" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.051.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.051.plan
new file mode 100644
index 0000000..d315b06
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/csv/embed-one-value/one-field.051.plan
@@ -0,0 +1,9 @@
+{ "id": "1", "first_name": "John", "last_name": "Smith", "department": "accounting" }
+{ "id": "2", "first_name": "Mike", "last_name": "Jones", "department": "accounting" }
+{ "id": "3", "first_name": "Alex", "last_name": "Miller", "department": "accounting" }
+{ "id": "4", "first_name": "Tom", "last_name": "Smith", "department": "engineering" }
+{ "id": "5", "first_name": "Alice", "last_name": "Jones", "department": "engineering" }
+{ "id": "6", "first_name": "Sheila", "last_name": "Miller", "department": "engineering" }
+{ "id": "7", "first_name": "James", "last_name": "Smith", "department": "hr" }
+{ "id": "8", "first_name": "David", "last_name": "Jones", "department": "hr" }
+{ "id": "9", "first_name": "Noah", "last_name": "Miller", "department": "hr" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 02353f7..b22eed0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -290,6 +290,18 @@
<output-dir compare="Text">embed-multiple-values</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset/common/dynamic-prefixes/csv">
+ <compilation-unit name="embed-one-value">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">embed-one-value</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/common/dynamic-prefixes/csv">
+ <compilation-unit name="embed-multiple-values">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">embed-multiple-values</output-dir>
+ </compilation-unit>
+ </test-case>
<!-- Dynamic prefixes tests end -->
<test-case FilePath="external-dataset">
<compilation-unit name="common/empty-string-definition">
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java
index bc855b4..b8a7480 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java
@@ -38,6 +38,8 @@
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordDataParser;
import org.apache.asterix.external.api.IStreamDataParser;
+import org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
+import org.apache.asterix.external.parser.jackson.ParserContext;
import org.apache.asterix.om.base.AMutableString;
import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
import org.apache.asterix.om.types.ARecordType;
@@ -46,6 +48,7 @@
import org.apache.asterix.om.utils.NonTaggedFormatUtil;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.dataflow.common.data.parsers.IValueParser;
import org.apache.hyracks.dataflow.common.data.parsers.IValueParserFactory;
@@ -69,7 +72,10 @@
private final byte[] fieldTypeTags;
private final int[] fldIds;
private final ArrayBackedValueStorage[] nameBuffers;
+ private final String[] fieldNames;
private final char[] nullChars;
+ private final IExternalFilterValueEmbedder valueEmbedder;
+ private final ParserContext parserContext;
private FieldCursorForDelimitedDataParser cursor;
public DelimitedDataParser(IExternalDataRuntimeContext context, IValueParserFactory[] valueParserFactories,
@@ -78,6 +84,7 @@
this.dataSourceName = context.getDatasourceNameSupplier();
this.lineNumber = context.getLineNumberSupplier();
this.warnings = context.getTaskContext().getWarningCollector();
+ this.valueEmbedder = context.getValueEmbedder();
this.fieldDelimiter = fieldDelimiter;
this.quote = quote;
this.hasHeader = hasHeader;
@@ -102,6 +109,7 @@
fldIds = new int[n];
nameBuffers = new ArrayBackedValueStorage[n];
+ fieldNames = new String[n];
AMutableString str = new AMutableString(null);
for (int i = 0; i < n; i++) {
String name = recordType.getFieldNames()[i];
@@ -116,12 +124,14 @@
IDataParser.toBytes(str, nameBuffers[i], stringSerde);
}
}
+ fieldNames[i] = name;
}
if (!isStreamParser) {
cursor = new FieldCursorForDelimitedDataParser(null, this.fieldDelimiter, quote, warnings,
this::getDataSourceName);
}
this.nullChars = nullString != null ? nullString.toCharArray() : null;
+ this.parserContext = new ParserContext();
}
@Override
@@ -145,7 +155,6 @@
private boolean parseRecord() throws HyracksDataException {
recBuilder.reset(recordType);
recBuilder.init();
-
for (int i = 0; i < valueParsers.length; ++i) {
try {
FieldCursorForDelimitedDataParser.Result result = cursor.nextField();
@@ -190,11 +199,8 @@
return false;
}
}
- if (fldIds[i] < 0) {
- recBuilder.addField(nameBuffers[i], fieldValueBuffer);
- } else {
- recBuilder.addField(fldIds[i], fieldValueBuffer);
- }
+
+ addValue(i);
} catch (IOException e) {
throw HyracksDataException.create(e);
}
@@ -212,7 +218,11 @@
@Override
public boolean parse(IRawRecord<? extends char[]> record, DataOutput out) throws HyracksDataException {
cursor.nextRecord(record.get(), record.size(), lineNumber.getAsLong());
+ valueEmbedder.reset();
+ valueEmbedder.enterObject();
if (parseRecord()) {
+ valueEmbedder.exitObject();
+ finalizeEmbedding();
recBuilder.write(out, true);
return true;
}
@@ -244,6 +254,19 @@
return true;
}
+ private void addValue(int index) throws HyracksDataException {
+ IValueReference value = fieldValueBuffer;
+ if (valueEmbedder.shouldEmbed(fieldNames[index], ATypeTag.VALUE_TYPE_MAPPING[fieldTypeTags[index]])) {
+ value = valueEmbedder.getEmbeddedValue();
+ }
+
+ if (fldIds[index] < 0) {
+ recBuilder.addField(nameBuffers[index], value);
+ } else {
+ recBuilder.addField(fldIds[index], value);
+ }
+ }
+
private String getDataSourceName() {
return dataSourceName.get();
}
@@ -269,4 +292,31 @@
}
return true;
}
+
+ private void finalizeEmbedding() throws HyracksDataException {
+ if (valueEmbedder.isMissingEmbeddedValues()) {
+ String[] embeddedFieldNames = valueEmbedder.getEmbeddedFieldNames();
+ for (int i = 0; i < embeddedFieldNames.length; i++) {
+ String embeddedFieldName = embeddedFieldNames[i];
+ int index = recordType.getFieldIndex(embeddedFieldName);
+ if (valueEmbedder.isMissing(embeddedFieldName)) {
+ IValueReference embeddedValue = valueEmbedder.getEmbeddedValue();
+ if (index < 0) {
+ recBuilder.addField(getSerializedFieldName(embeddedFieldName), embeddedValue);
+ } else {
+ recBuilder.addField(index, embeddedValue);
+ }
+ }
+ }
+ }
+ }
+
+ private IValueReference getSerializedFieldName(String fieldName) throws HyracksDataException {
+ try {
+ return parserContext.getSerializedFieldName(fieldName);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+
+ }
}