[ASTERIXDB-3255][EXT] Part1: Embed external filter values
- user model changes: yes
- storage format changes: no
- interface changes: yes
Details:
- Embed filter values by default
-- Users can disable that by setting 'embed-filter-values'
to false when creating an external dataset.
-- Value embedding is performed at the parser level
- This patch includes the support of value embedding when
querying JSON-based external datasets.
- The support for Parquet, CSV, and TSV will be included
in follow-up patches.
Change-Id: Ia61a62ab1390d9d87105898cf8c17f77c136c434
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17751
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/PushdownProcessorsExecutor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/PushdownProcessorsExecutor.java
index ab6a26c..957e988 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/PushdownProcessorsExecutor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/PushdownProcessorsExecutor.java
@@ -25,7 +25,10 @@
import java.util.Map;
import org.apache.asterix.common.config.DatasetConfig;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.ExternalDataPrefix;
import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
import org.apache.asterix.metadata.utils.DatasetUtil;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.optimizer.rules.pushdown.descriptor.ScanDefineDescriptor;
@@ -112,8 +115,11 @@
scanDefineDescriptor.getDataset().getDatasetName());
}
+ Map<String, String> configuration = ((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties();
+ boolean embedFilterValues = ExternalDataPrefix.containsComputedFields(configuration) && Boolean.parseBoolean(
+ configuration.getOrDefault(ExternalDataConstants.KEY_EMBED_FILTER_VALUES, ExternalDataConstants.TRUE));
return new ExternalDatasetProjectionFiltrationInfo(recordRequestedType, pathLocations,
- scanDefineDescriptor.getFilterPaths(), scanDefineDescriptor.getFilterExpression());
+ scanDefineDescriptor.getFilterPaths(), scanDefineDescriptor.getFilterExpression(), embedFilterValues);
}
private void setInfoToDataScan(AbstractScanOperator scanOp, IProjectionFiltrationInfo info) {
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/ColumnFilterPathBuilderVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/ColumnFilterPathBuilderVisitor.java
index 79bb9b2..6c3027a 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/ColumnFilterPathBuilderVisitor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/ColumnFilterPathBuilderVisitor.java
@@ -18,15 +18,14 @@
*/
package org.apache.asterix.optimizer.rules.pushdown.visitor;
+import static org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil.renameType;
+
import java.util.Map;
-import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.om.types.AOrderedListType;
import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.om.types.IATypeVisitor;
import org.apache.asterix.optimizer.rules.pushdown.schema.AnyExpectedSchemaNode;
import org.apache.asterix.optimizer.rules.pushdown.schema.ArrayExpectedSchemaNode;
import org.apache.asterix.optimizer.rules.pushdown.schema.IExpectedSchemaNode;
@@ -37,8 +36,6 @@
import org.apache.asterix.runtime.projection.FunctionCallInformation;
import org.apache.asterix.runtime.projection.ProjectionFiltrationWarningFactoryProvider;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
public class ColumnFilterPathBuilderVisitor implements IExpectedSchemaNodeVisitor<IAType, IExpectedSchemaNode> {
private IAType type;
@@ -55,7 +52,7 @@
this.sourceInformationMap = sourceInformationMap;
this.type = constType;
if (sourceInformationMap != null) {
- this.type = rename(constType);
+ this.type = renameType(constType, getTypeName());
sourceInformationMap.put(type.getTypeName(), compareFunctionInfo);
}
return (ARecordType) anyNode.accept(this, anyNode);
@@ -115,72 +112,4 @@
return new FunctionCallInformation(node.getFunctionName(), node.getSourceLocation(),
ProjectionFiltrationWarningFactoryProvider.TYPE_MISMATCH_FACTORY);
}
-
- private IAType rename(IAType type) {
- return new RenamedType(type, getTypeName());
- }
-
- private static class RenamedType implements IAType {
- private static final long serialVersionUID = 992690669300951839L;
- private final IAType originalType;
- private final String name;
-
- RenamedType(IAType originalType, String name) {
- this.originalType = originalType;
- this.name = name;
- }
-
- @Override
- public IAType getType() {
- return originalType.getType();
- }
-
- @Override
- public boolean deepEqual(IAObject obj) {
- return originalType.deepEqual(obj);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof RenamedType) {
- return originalType.equals(((RenamedType) obj).originalType);
- }
- return originalType.equals(obj);
- }
-
- @Override
- public int hash() {
- return originalType.hash();
- }
-
- @Override
- public ATypeTag getTypeTag() {
- return originalType.getTypeTag();
- }
-
- @Override
- public String getDisplayName() {
- return originalType.getDisplayName();
- }
-
- @Override
- public String getTypeName() {
- return name;
- }
-
- @Override
- public <R, T> R accept(IATypeVisitor<R, T> visitor, T arg) {
- return visitor.visitFlat(this, arg);
- }
-
- @Override
- public ObjectNode toJSON() {
- return originalType.toJSON();
- }
-
- @Override
- public String toString() {
- return originalType.toString();
- }
- }
}
diff --git a/asterixdb/asterix-app/data/json/external-filter/embed/department/accounting/0.json b/asterixdb/asterix-app/data/json/external-filter/embed/department/accounting/0.json
new file mode 100644
index 0000000..8dca4a7
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/embed/department/accounting/0.json
@@ -0,0 +1,3 @@
+{"id": 1, "name": {"first": "John", "last": "Smith"}}
+{"id": 2, "name": {"first": "Mike", "last": "Jones"}}
+{"id": 3, "name": {"first": "Alex", "last": "Miller"}}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/external-filter/embed/department/engineering/0.json b/asterixdb/asterix-app/data/json/external-filter/embed/department/engineering/0.json
new file mode 100644
index 0000000..d27a58b
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/embed/department/engineering/0.json
@@ -0,0 +1,3 @@
+{"id": 4, "name": {"first": "Tom", "last": "Smith"}}
+{"id": 5, "name": {"first": "Alice", "last": "Jones"}}
+{"id": 6, "name": {"first": "Sheila", "last": "Miller"}}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/external-filter/embed/department/hr/0.json b/asterixdb/asterix-app/data/json/external-filter/embed/department/hr/0.json
new file mode 100644
index 0000000..aba29ab
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/embed/department/hr/0.json
@@ -0,0 +1,3 @@
+{"id": 7, "name": {"first": "James", "last": "Smith"}}
+{"id": 8, "name": {"first": "David", "last": "Jones"}}
+{"id": 9, "name": {"first": "Noah", "last": "Miller"}}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/external-filter/embed/last-name/Jones/0.json b/asterixdb/asterix-app/data/json/external-filter/embed/last-name/Jones/0.json
new file mode 100644
index 0000000..3cec0e4
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/embed/last-name/Jones/0.json
@@ -0,0 +1,3 @@
+{"id": 2, "department": "accounting", "name": {"first": "Mike", "last": "Jones"}}
+{"id": 5, "department": "engineering", "name": {"first": "Alice"}}
+{"id": 8, "department": "hr", "name": null}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/external-filter/embed/last-name/miller/0.json b/asterixdb/asterix-app/data/json/external-filter/embed/last-name/miller/0.json
new file mode 100644
index 0000000..419df09
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/embed/last-name/miller/0.json
@@ -0,0 +1,3 @@
+{"id": 3, "department": "accounting", "name": {"first": "Alex", "last": "Miller"}}
+{"id": 6, "department": "engineering", "name": "Sheila Miller"}
+{"id": 9, "department": "hr"}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/external-filter/embed/last-name/smith/0.json b/asterixdb/asterix-app/data/json/external-filter/embed/last-name/smith/0.json
new file mode 100644
index 0000000..972c531
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/embed/last-name/smith/0.json
@@ -0,0 +1,3 @@
+{"id": 1, "department": "accounting", "name": {"first": "John", "last": "Smith"}}
+{"id": 4, "department": "engineering", "name": {"first": "Tom", "last": "Smith"}}
+{"id": 7, "department": "hr", "name": {"first": "James", "last": "Smith"}}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/PrefixComputedFieldsTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/PrefixComputedFieldsTest.java
index 7b00240..efa4c79 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/PrefixComputedFieldsTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/PrefixComputedFieldsTest.java
@@ -49,7 +49,7 @@
assertTrue(prefix.getIndexToComputedFieldsMap().isEmpty());
String prefix1 = "";
- prefix = new ExternalDataPrefix(prefix1, null);
+ prefix = new ExternalDataPrefix(prefix1);
assertEquals("", prefix.getOriginal());
assertEquals("", prefix.getRoot());
assertFalse(prefix.isEndsWithSlash());
@@ -60,7 +60,7 @@
assertTrue(prefix.getIndexToComputedFieldsMap().isEmpty());
String prefix2 = "hotel";
- prefix = new ExternalDataPrefix(prefix2, null);
+ prefix = new ExternalDataPrefix(prefix2);
assertEquals("hotel", prefix.getOriginal());
assertEquals("hotel", prefix.getRoot());
assertFalse(prefix.isEndsWithSlash());
@@ -71,7 +71,7 @@
assertTrue(prefix.getIndexToComputedFieldsMap().isEmpty());
String prefix3 = "hotel/{hotel-id:inT}/";
- prefix = new ExternalDataPrefix(prefix3, null);
+ prefix = new ExternalDataPrefix(prefix3);
assertEquals("hotel/{hotel-id:inT}/", prefix.getOriginal());
assertEquals("hotel/", prefix.getRoot());
assertTrue(prefix.isEndsWithSlash());
@@ -82,7 +82,7 @@
assertEquals("(.+)", prefix.getIndexToComputedFieldsMap().get(1).getExpression());
String prefix4 = "hotel/{hotel-id:int}-{hotel-name:sTRing}";
- prefix = new ExternalDataPrefix(prefix4, null);
+ prefix = new ExternalDataPrefix(prefix4);
assertEquals("hotel/{hotel-id:int}-{hotel-name:sTRing}", prefix.getOriginal());
assertEquals("hotel", prefix.getRoot());
assertFalse(prefix.isEndsWithSlash());
@@ -93,7 +93,7 @@
assertEquals("(.+)-(.+)", prefix.getIndexToComputedFieldsMap().get(1).getExpression());
String prefix5 = "hotel/something/{hotel-id:int}-{hotel-name:sTRing}/review/{year:int}-{month:int}-{day:int}/";
- prefix = new ExternalDataPrefix(prefix5, null);
+ prefix = new ExternalDataPrefix(prefix5);
assertEquals("hotel/something/{hotel-id:int}-{hotel-name:sTRing}/review/{year:int}-{month:int}-{day:int}/",
prefix.getOriginal());
assertEquals("hotel/something/", prefix.getRoot());
@@ -107,7 +107,7 @@
assertEquals("(.+)-(.+)-(.+)", prefix.getIndexToComputedFieldsMap().get(4).getExpression());
String prefix6 = "hotel/something/{hotel-id:int}-{hotel-name:sTRing}/review/{year:int}/{month:int}/{day:int}";
- prefix = new ExternalDataPrefix(prefix6, null);
+ prefix = new ExternalDataPrefix(prefix6);
assertEquals("hotel/something/{hotel-id:int}-{hotel-name:sTRing}/review/{year:int}/{month:int}/{day:int}",
prefix.getOriginal());
assertEquals("hotel/something", prefix.getRoot());
@@ -123,7 +123,7 @@
assertEquals("(.+)", prefix.getIndexToComputedFieldsMap().get(6).getExpression());
String prefix7 = "hotel/{hotel.details.id:int}-{hotel-name:sTRing}";
- prefix = new ExternalDataPrefix(prefix7, null);
+ prefix = new ExternalDataPrefix(prefix7);
assertEquals("hotel/{hotel.details.id:int}-{hotel-name:sTRing}", prefix.getOriginal());
assertEquals("hotel", prefix.getRoot());
assertFalse(prefix.isEndsWithSlash());
@@ -134,7 +134,7 @@
String prefix8 =
"hotel/hotel-{hotel-id:int}-hotel-{hotel-name:sTRing}/review/year-{year:int}/{month:int}-month/day-{day:int}-day";
- prefix = new ExternalDataPrefix(prefix8, null);
+ prefix = new ExternalDataPrefix(prefix8);
assertEquals(
"hotel/hotel-{hotel-id:int}-hotel-{hotel-name:sTRing}/review/year-{year:int}/{month:int}-month/day-{day:int}-day",
prefix.getOriginal());
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.000.ddl.sqlpp
new file mode 100644
index 0000000..3e98392
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.000.ddl.sqlpp
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE TestType AS {
+};
+
+
+CREATE EXTERNAL DATASET Department(TestType) USING %adapter% (
+ %template%,
+ ("container"="playground"),
+ ("definition"="external-filter/embed/department/{department:string}"),
+ ("embed-filter-values" = "true"),
+ ("format"="json")
+);
+
+CREATE EXTERNAL DATASET LastName(TestType) USING %adapter% (
+ %template%,
+ ("container"="playground"),
+ ("definition"="external-filter/embed/last-name/{name.last:string}"),
+ ("embed-filter-values" = "true"),
+ ("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.010.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.010.query.sqlpp
new file mode 100644
index 0000000..0034525
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.010.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.011.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.011.query.sqlpp
new file mode 100644
index 0000000..b5856ac
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.011.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.020.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.020.query.sqlpp
new file mode 100644
index 0000000..a0919a7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.020.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ AND d.name.last = "Smith"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.021.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.021.query.sqlpp
new file mode 100644
index 0000000..3eb867c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.021.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ AND d.name.last = "Smith"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.030.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.030.query.sqlpp
new file mode 100644
index 0000000..c24bf2c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.030.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ OR d.name.last = "Smith"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.031.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.031.query.sqlpp
new file mode 100644
index 0000000..e1c0b02
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.031.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ OR d.name.last = "Smith"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.040.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.040.query.sqlpp
new file mode 100644
index 0000000..e3e6ee5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.040.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM Department d
+WHERE uppercase(d.department) = "H" || "R"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.041.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.041.query.sqlpp
new file mode 100644
index 0000000..bfa65b5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.041.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM Department d
+WHERE uppercase(d.department) = "H" || "R"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.110.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.110.query.sqlpp
new file mode 100644
index 0000000..349ba83
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.110.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM LastName d
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.111.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.111.query.sqlpp
new file mode 100644
index 0000000..1beac09
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.111.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM LastName d
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.120.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.120.query.sqlpp
new file mode 100644
index 0000000..7ae11e6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.120.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM LastName d
+WHERE d.name.last = "Jones"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.121.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.121.query.sqlpp
new file mode 100644
index 0000000..e26b35a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.121.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM LastName d
+WHERE d.name.last = "Jones"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.130.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.130.query.sqlpp
new file mode 100644
index 0000000..c4d575d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.130.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM LastName d
+WHERE lowercase(d.name.first) = "john"
+ AND "john " || lowercase(d.name.last) = "john smith"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.131.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.131.query.sqlpp
new file mode 100644
index 0000000..63dbd7c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-one-value/embed-one-value.131.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM LastName d
+WHERE lowercase(d.name.first) = "john"
+ AND "john " || lowercase(d.name.last) = "john smith"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/one-field/one-field.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/one-field/one-field.000.ddl.sqlpp
index 27885d2..6036e79 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/one-field/one-field.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/one-field/one-field.000.ddl.sqlpp
@@ -29,13 +29,14 @@
%template%,
("container"="playground"),
("definition"="external-filter/department/{department:string}"),
+ ("embed-filter-values" = "false"),
("format"="json")
);
CREATE EXTERNAL DATASET LastName(TestType) USING %adapter% (
%template%,
("container"="playground"),
- ("container"="playground"),
("definition"="external-filter/last-name/{name.last:string}"),
+ ("embed-filter-values" = "false"),
("format"="json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/one-field/one-field.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/one-field/one-field.000.ddl.sqlpp
index 7c1ac6b..b9c9f6a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/one-field/one-field.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/one-field/one-field.000.ddl.sqlpp
@@ -29,13 +29,14 @@
%template%,
("container"="playground"),
("definition"="parquet-data/external-filter/department/{department:string}"),
+ ("embed-filter-values" = "false"),
("format"="parquet")
);
CREATE EXTERNAL DATASET LastName(TestType) USING %adapter% (
%template%,
("container"="playground"),
- ("container"="playground"),
("definition"="parquet-data/external-filter/last-name/{name.last:string}"),
+ ("embed-filter-values" = "false"),
("format"="parquet")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/query/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/query/test.000.ddl.sqlpp
index 81e7b59..38d121f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/query/test.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/query/test.000.ddl.sqlpp
@@ -29,6 +29,7 @@
%template%,
("container"="playground"),
("definition"="parquet-data/external-filter/car/{company:string}"),
+ ("embed-filter-values" = "false"),
("format"="parquet")
);
@@ -36,6 +37,7 @@
%template%,
("container"="playground"),
("definition"="parquet-data/external-filter/car/{company:string}/customer/{customer_id:int}"),
+ ("embed-filter-values" = "false"),
("format"="parquet")
);
@@ -43,5 +45,6 @@
%template%,
("container"="playground"),
("definition"="parquet-data/external-filter/car/{company:string}/customer/{customer_id:int}/maintenance-report/year-{year:int}-month-{month:int}-day-{day:int}-date"),
+ ("embed-filter-values" = "false"),
("format"="parquet")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/type-mismatch/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/type-mismatch/test.000.ddl.sqlpp
index 7ff8a2b..0e82c4a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/type-mismatch/test.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/type-mismatch/test.000.ddl.sqlpp
@@ -29,5 +29,6 @@
%template%,
("container"="playground"),
("definition"="parquet-data/external-filter/department/{name:bigint}"),
+ ("embed-filter-values" = "false"),
("format"="parquet")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/query/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/query/test.000.ddl.sqlpp
index 4fdae03..bd77e5b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/query/test.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/query/test.000.ddl.sqlpp
@@ -29,6 +29,7 @@
%template%,
("container"="playground"),
("definition"="external-filter/car/{company:string}"),
+ ("embed-filter-values" = "false"),
("format"="json")
);
@@ -36,6 +37,7 @@
%template%,
("container"="playground"),
("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"),
+ ("embed-filter-values" = "false"),
("format"="json")
);
@@ -43,5 +45,6 @@
%template%,
("container"="playground"),
("definition"="external-filter/car/{company:string}/customer/{customer_id:int}/maintenance-report/year-{year:int}-month-{month:int}-day-{day:int}-date"),
+ ("embed-filter-values" = "false"),
("format"="json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/type-mismatch/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/type-mismatch/test.000.ddl.sqlpp
index ee22f55..1da1ded 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/type-mismatch/test.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/type-mismatch/test.000.ddl.sqlpp
@@ -29,5 +29,6 @@
%template%,
("container"="playground"),
("definition"="external-filter/department/{name:bigint}"),
+ ("embed-filter-values" = "false"),
("format"="json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.010.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.010.adm
new file mode 100644
index 0000000..7cb9062
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.010.adm
@@ -0,0 +1,3 @@
+{ "id": 1, "name": { "first": "John", "last": "Smith" }, "department": "accounting" }
+{ "id": 2, "name": { "first": "Mike", "last": "Jones" }, "department": "accounting" }
+{ "id": 3, "name": { "first": "Alex", "last": "Miller" }, "department": "accounting" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.011.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.011.plan
new file mode 100644
index 0000000..b560a5b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.011.plan
@@ -0,0 +1,24 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$17(ASC) ] |PARTITIONED|
+ order (ASC, $$17) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$17(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (eq($$d.getField("department"), "accounting")) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$17] <- [$$d.getField("id")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.Department prefix-filter on: eq($$d.getField("department"), "accounting") embed-filter-value: true [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.020.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.020.adm
new file mode 100644
index 0000000..8bfbbb3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.020.adm
@@ -0,0 +1 @@
+{ "id": 1, "name": { "first": "John", "last": "Smith" }, "department": "accounting" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.021.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.021.plan
new file mode 100644
index 0000000..2bfad06
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.021.plan
@@ -0,0 +1,24 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$22(ASC) ] |PARTITIONED|
+ order (ASC, $$22) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$22(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (and(eq($$d.getField("department"), "accounting"), eq($$d.getField("name").getField("last"), "Smith"))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$22] <- [$$d.getField("id")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.Department prefix-filter on: eq($$d.getField("department"), "accounting") embed-filter-value: true [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.030.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.030.adm
new file mode 100644
index 0000000..7d601d9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.030.adm
@@ -0,0 +1,5 @@
+{ "id": 1, "name": { "first": "John", "last": "Smith" }, "department": "accounting" }
+{ "id": 2, "name": { "first": "Mike", "last": "Jones" }, "department": "accounting" }
+{ "id": 3, "name": { "first": "Alex", "last": "Miller" }, "department": "accounting" }
+{ "id": 4, "name": { "first": "Tom", "last": "Smith" }, "department": "engineering" }
+{ "id": 7, "name": { "first": "James", "last": "Smith" }, "department": "hr" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.031.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.031.plan
new file mode 100644
index 0000000..afb74f1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.031.plan
@@ -0,0 +1,24 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$22(ASC) ] |PARTITIONED|
+ order (ASC, $$22) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$22(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (or(eq($$d.getField("department"), "accounting"), eq($$d.getField("name").getField("last"), "Smith"))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$22] <- [$$d.getField("id")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.Department embed-filter-value: true [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.040.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.040.adm
new file mode 100644
index 0000000..3bfb1ba
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.040.adm
@@ -0,0 +1,3 @@
+{ "id": 7, "name": { "first": "James", "last": "Smith" }, "department": "hr" }
+{ "id": 8, "name": { "first": "David", "last": "Jones" }, "department": "hr" }
+{ "id": 9, "name": { "first": "Noah", "last": "Miller" }, "department": "hr" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.041.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.041.plan
new file mode 100644
index 0000000..6f2ecf3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.041.plan
@@ -0,0 +1,24 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$20(ASC) ] |PARTITIONED|
+ order (ASC, $$20) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$20(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (eq(uppercase($$d.getField("department")), "HR")) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$20] <- [$$d.getField("id")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.Department prefix-filter on: eq(uppercase($$d.getField("department")), "HR") embed-filter-value: true [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.110.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.110.adm
new file mode 100644
index 0000000..de64450
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.110.adm
@@ -0,0 +1,9 @@
+{ "id": 1, "department": "accounting", "name": { "first": "John", "last": "smith" } }
+{ "id": 2, "department": "accounting", "name": { "first": "Mike", "last": "Jones" } }
+{ "id": 3, "department": "accounting", "name": { "first": "Alex", "last": "miller" } }
+{ "id": 4, "department": "engineering", "name": { "first": "Tom", "last": "smith" } }
+{ "id": 5, "department": "engineering", "name": { "first": "Alice", "last": "Jones" } }
+{ "id": 6, "department": "engineering", "name": { "last": "miller" } }
+{ "id": 7, "department": "hr", "name": { "first": "James", "last": "smith" } }
+{ "id": 8, "department": "hr", "name": { "last": "Jones" } }
+{ "id": 9, "department": "hr", "name": { "last": "miller" } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.111.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.111.plan
new file mode 100644
index 0000000..ff27bfe
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.111.plan
@@ -0,0 +1,22 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$14(ASC) ] |PARTITIONED|
+ order (ASC, $$14) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$14(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ assign [$$14] <- [$$d.getField("id")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.LastName embed-filter-value: true [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.120.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.120.adm
new file mode 100644
index 0000000..6d3c45f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.120.adm
@@ -0,0 +1,3 @@
+{ "id": 2, "department": "accounting", "name": { "first": "Mike", "last": "Jones" } }
+{ "id": 5, "department": "engineering", "name": { "first": "Alice", "last": "Jones" } }
+{ "id": 8, "department": "hr", "name": { "last": "Jones" } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.121.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.121.plan
new file mode 100644
index 0000000..fc48056
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.121.plan
@@ -0,0 +1,24 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$18(ASC) ] |PARTITIONED|
+ order (ASC, $$18) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$18(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (eq($$d.getField("name").getField("last"), "Jones")) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$18] <- [$$d.getField("id")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.LastName prefix-filter on: eq($$d.getField("name").getField("last"), "Jones") embed-filter-value: true [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.130.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.130.adm
new file mode 100644
index 0000000..1060822
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.130.adm
@@ -0,0 +1 @@
+{ "id": 1, "department": "accounting", "name": { "first": "John", "last": "smith" } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.131.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.131.plan
new file mode 100644
index 0000000..4edf24e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-one-value/one-field.131.plan
@@ -0,0 +1,26 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$28(ASC) ] |PARTITIONED|
+ order (ASC, $$28) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$28(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d, $$28]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ select (and(eq(lowercase($$25.getField("first")), "john"), eq(string-concat(ordered-list-constructor("john ", lowercase($$25.getField("last")))), "john smith"))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$28, $$25] <- [$$d.getField("id"), $$d.getField("name")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.LastName prefix-filter on: eq(string-concat(ordered-list-constructor("john ", lowercase($$d.getField("name").getField("last")))), "john smith") embed-filter-value: true [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index de91ca5..1b553d3 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -244,6 +244,12 @@
<output-dir compare="Text">query</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset/common/dynamic-prefixes">
+ <compilation-unit name="embed-one-value">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">embed-one-value</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-dataset/common/dynamic-prefixes/parquet">
<compilation-unit name="one-field">
<placeholder name="adapter" value="S3" />
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IExternalFilterEmbeddedValueInformation.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IExternalFilterEmbeddedValueInformation.java
new file mode 100644
index 0000000..9ee37d8
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IExternalFilterEmbeddedValueInformation.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.external;
+
+public interface IExternalFilterEmbeddedValueInformation {
+
+ String[] getEmbeddedFieldNames();
+
+ boolean IsMissingEmbeddedValues();
+
+ boolean isMissing(String fieldName);
+
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IExternalFilterEvaluatorFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IExternalFilterEvaluatorFactory.java
index 38a38a6..d433cf2 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IExternalFilterEvaluatorFactory.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IExternalFilterEvaluatorFactory.java
@@ -27,4 +27,6 @@
public interface IExternalFilterEvaluatorFactory extends Serializable {
IExternalFilterEvaluator create(IServiceContext serviceContext, IWarningCollector warningCollector)
throws HyracksDataException;
+
+ <T extends IExternalFilterEmbeddedValueInformation> T createValueEmbedder(IWarningCollector warningCollector);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/ExternalAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/ExternalAdapterFactory.java
index 9741bb1..2173543 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/ExternalAdapterFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/ExternalAdapterFactory.java
@@ -25,13 +25,13 @@
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.external.IDataSourceAdapter;
import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
-import org.apache.asterix.common.external.NoOpExternalFilterEvaluatorFactory;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.common.library.ILibrary;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.external.api.IExternalDataSourceFactory;
import org.apache.asterix.external.api.ITypedAdapterFactory;
+import org.apache.asterix.external.input.filter.NoOpExternalFilterEvaluatorFactory;
import org.apache.asterix.external.library.JavaLibrary;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
index 89ee37b..7398d02 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
@@ -26,7 +26,6 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.external.IDataSourceAdapter;
import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
-import org.apache.asterix.common.external.NoOpExternalFilterEvaluatorFactory;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IDataParserFactory;
@@ -35,6 +34,7 @@
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.dataset.adapter.FeedAdapter;
import org.apache.asterix.external.dataset.adapter.GenericAdapter;
+import org.apache.asterix.external.input.filter.NoOpExternalFilterEvaluatorFactory;
import org.apache.asterix.external.provider.DataflowControllerProvider;
import org.apache.asterix.external.provider.DatasourceFactoryProvider;
import org.apache.asterix.external.provider.ParserFactoryProvider;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
index 43e822e..8eb4475 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
@@ -21,6 +21,7 @@
import java.io.InputStream;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
+import org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
import org.apache.asterix.external.util.IFeedLogManager;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -29,6 +30,7 @@
protected AbstractFeedDataFlowController controller;
protected IFeedLogManager logManager;
protected IStreamNotificationHandler notificationHandler;
+ protected IExternalFilterValueEmbedder valueEmbedder;
public abstract boolean stop() throws Exception;
@@ -48,6 +50,10 @@
this.notificationHandler = notificationHandler;
}
+ public void setValueEmbedder(IExternalFilterValueEmbedder valueEmbedder) {
+ this.valueEmbedder = valueEmbedder;
+ }
+
public String getStreamName() {
return "";
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataParser.java
index 5dbc383..c1d6797 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataParser.java
@@ -24,6 +24,7 @@
import org.apache.asterix.builders.OrderedListBuilder;
import org.apache.asterix.builders.RecordBuilder;
import org.apache.asterix.builders.UnorderedListBuilder;
+import org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
import org.apache.asterix.om.base.AMutableOrderedList;
import org.apache.asterix.om.base.AMutableRecord;
@@ -38,6 +39,9 @@
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
public interface IDataParser {
+ default void setValueEmbedder(IExternalFilterValueEmbedder valueEmbedder) {
+ // NoOp
+ }
/*
* The following two static methods are expensive. right now, they are used by RSSFeeds and
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java
index d628bc7..058fe89 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java
@@ -28,6 +28,8 @@
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
+import org.apache.asterix.external.input.filter.NoOpFilterValueEmbedder;
+import org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
@@ -69,6 +71,10 @@
void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector,
IExternalFilterEvaluatorFactory filterEvaluatorFactory) throws AlgebricksException, HyracksDataException;
+ default IExternalFilterValueEmbedder createFilterValueEmbedder(IWarningCollector warningCollector) {
+ return NoOpFilterValueEmbedder.INSTANCE;
+ }
+
/**
* returns the passed partition constraints if not null, otherwise returns round robin absolute partition
* constraints that matches the count.
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordDataParser.java
index c4dfdd0..31e1764 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordDataParser.java
@@ -33,8 +33,7 @@
* warning and/or throw an exception in case of failure.
*
* @param record input record to parse
- * @param out output where the parsed record is written into
- *
+ * @param out output where the parsed record is written into
* @return true if the record was parsed successfully and written to out. False, otherwise.
* @throws HyracksDataException HyracksDataException
*/
@@ -44,7 +43,7 @@
* Configures the parser with information suppliers from the {@link IRecordReader} data source.
*
* @param dataSourceName data source name supplier
- * @param lineNumber line number supplier
+ * @param lineNumber line number supplier
*/
default void configure(Supplier<String> dataSourceName, LongSupplier lineNumber) {
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
index 90dc6c4..b7d4c85 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
@@ -25,6 +25,7 @@
import org.apache.asterix.active.message.ActiveManagerMessage;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
+import org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.IFeedLogManager;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -38,8 +39,7 @@
/**
* @return true if the reader has more records remaining, false, otherwise.
- * @throws Exception
- * if an error takes place
+ * @throws Exception if an error takes place
*/
public boolean hasNext() throws Exception;
@@ -58,6 +58,7 @@
public boolean stop();
// TODO: Find a better way to do flushes, this doesn't fit here
+
/**
* set a pointer to the controller of the feed. the controller can be used to flush()
* parsed records when waiting for more records to be pushed
@@ -65,6 +66,7 @@
public void setController(AbstractFeedDataFlowController controller);
// TODO: Find a better way to perform logging. this doesn't fit here
+
/**
* set a pointer to the log manager of the feed. the log manager can be used to log
* progress and errors
@@ -73,6 +75,10 @@
*/
public void setFeedLogManager(IFeedLogManager feedLogManager) throws HyracksDataException;
+ default void setValueEmbedder(IExternalFilterValueEmbedder valueEmbedder) {
+ // NoOp
+ }
+
/**
* gives the record reader a chance to recover from IO errors during feed intake
*/
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterEvaluatorFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterEvaluatorFactory.java
index 0790700..589b382 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterEvaluatorFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterEvaluatorFactory.java
@@ -19,9 +19,15 @@
package org.apache.asterix.external.input.filter;
import java.util.Arrays;
+import java.util.List;
import org.apache.asterix.common.external.IExternalFilterEvaluator;
import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
+import org.apache.asterix.external.input.filter.embedder.ExternalFilterValueEmbedder;
+import org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
+import org.apache.asterix.external.util.ExternalDataPrefix;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -29,21 +35,42 @@
public class ExternalFilterEvaluatorFactory implements IExternalFilterEvaluatorFactory {
private static final long serialVersionUID = -309935877927008746L;
- private final int numberOfComputedFields;
private final IScalarEvaluatorFactory filterEvalFactory;
+ private final ARecordType allPaths;
+ private final List<ProjectionFiltrationTypeUtil.RenamedType> leafs;
+ private final ExternalDataPrefix prefix;
+ private final boolean embedFilterValues;
- public ExternalFilterEvaluatorFactory(int numberOfComputedFields, IScalarEvaluatorFactory filterEvalFactory) {
- this.numberOfComputedFields = numberOfComputedFields;
+ public ExternalFilterEvaluatorFactory(IScalarEvaluatorFactory filterEvalFactory, ARecordType allPaths,
+ List<ProjectionFiltrationTypeUtil.RenamedType> leafs, ExternalDataPrefix prefix,
+ boolean embedFilterValues) {
this.filterEvalFactory = filterEvalFactory;
+ this.allPaths = allPaths;
+ this.leafs = leafs;
+ this.prefix = prefix;
+ this.embedFilterValues = embedFilterValues;
}
@Override
public IExternalFilterEvaluator create(IServiceContext serviceContext, IWarningCollector warningCollector)
throws HyracksDataException {
- IExternalFilterValueEvaluator[] valueEvaluators = new IExternalFilterValueEvaluator[numberOfComputedFields];
+ if (filterEvalFactory == null) {
+ return NoOpExternalFilterEvaluatorFactory.INSTANCE.create(serviceContext, warningCollector);
+ }
+
+ IExternalFilterValueEvaluator[] valueEvaluators = new IExternalFilterValueEvaluator[leafs.size()];
Arrays.fill(valueEvaluators, NoOpExternalFilterValueEvaluator.INSTANCE);
FilterEvaluatorContext filterContext =
new FilterEvaluatorContext(serviceContext, warningCollector, valueEvaluators);
return new ExternalFilterEvaluator(filterEvalFactory.createScalarEvaluator(filterContext), valueEvaluators);
}
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public IExternalFilterValueEmbedder createValueEmbedder(IWarningCollector warningCollector) {
+ if (embedFilterValues) {
+ return new ExternalFilterValueEmbedder(allPaths, leafs, prefix, warningCollector);
+ }
+ return NoOpFilterValueEmbedder.INSTANCE;
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterValueEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterValueEvaluator.java
index 40b10d4..ec5b3e6 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterValueEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterValueEvaluator.java
@@ -32,7 +32,7 @@
import org.apache.hyracks.dataflow.common.data.marshalling.Integer64SerializerDeserializer;
import org.apache.hyracks.util.string.UTF8StringWriter;
-class ExternalFilterValueEvaluator implements IExternalFilterValueEvaluator {
+public class ExternalFilterValueEvaluator implements IExternalFilterValueEvaluator {
private final ATypeTag typeTag;
private final ArrayBackedValueStorage value;
private final AStringSerializerDeserializer stringSerDer;
@@ -47,7 +47,7 @@
public void setValue(String stringValue) throws HyracksDataException {
value.reset();
try {
- writeValue(typeTag, stringValue);
+ writeValue(typeTag, stringValue, value, stringSerDer);
} catch (IOException e) {
throw HyracksDataException.create(e);
}
@@ -58,7 +58,8 @@
result.set(value);
}
- private void writeValue(ATypeTag typeTag, String stringValue) throws HyracksDataException {
+ public static void writeValue(ATypeTag typeTag, String stringValue, ArrayBackedValueStorage value,
+ AStringSerializerDeserializer stringSerDer) throws HyracksDataException {
DataOutput output = value.getDataOutput();
SerializerDeserializerUtil.serializeTag(typeTag, output);
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/NoOpExternalFilterEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/NoOpExternalFilterEvaluator.java
similarity index 91%
rename from asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/NoOpExternalFilterEvaluator.java
rename to asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/NoOpExternalFilterEvaluator.java
index e48ad80..c840cdd 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/NoOpExternalFilterEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/NoOpExternalFilterEvaluator.java
@@ -16,7 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.external;
+package org.apache.asterix.external.input.filter;
+
+import org.apache.asterix.common.external.IExternalFilterEvaluator;
class NoOpExternalFilterEvaluator implements IExternalFilterEvaluator {
static final IExternalFilterEvaluator INSTANCE = new NoOpExternalFilterEvaluator();
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/NoOpExternalFilterEvaluatorFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/NoOpExternalFilterEvaluatorFactory.java
similarity index 74%
rename from asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/NoOpExternalFilterEvaluatorFactory.java
rename to asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/NoOpExternalFilterEvaluatorFactory.java
index 7b8792e..967dee6 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/NoOpExternalFilterEvaluatorFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/NoOpExternalFilterEvaluatorFactory.java
@@ -16,8 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.common.external;
+package org.apache.asterix.external.input.filter;
+import org.apache.asterix.common.external.IExternalFilterEvaluator;
+import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
+import org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.exceptions.IWarningCollector;
@@ -32,4 +35,11 @@
public IExternalFilterEvaluator create(IServiceContext serviceContext, IWarningCollector warningCollector) {
return NoOpExternalFilterEvaluator.INSTANCE;
}
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public IExternalFilterValueEmbedder createValueEmbedder(IWarningCollector warningCollector) {
+ return NoOpFilterValueEmbedder.INSTANCE;
+ }
+
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/NoOpFilterValueEmbedder.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/NoOpFilterValueEmbedder.java
new file mode 100644
index 0000000..fe15387
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/NoOpFilterValueEmbedder.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.filter;
+
+import org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+public final class NoOpFilterValueEmbedder implements IExternalFilterValueEmbedder {
+ public static final IExternalFilterValueEmbedder INSTANCE = new NoOpFilterValueEmbedder();
+
+ private NoOpFilterValueEmbedder() {
+ }
+
+ @Override
+ public void reset() {
+ // NoOp
+ }
+
+ @Override
+ public void setPath(String path) throws HyracksDataException {
+ // NoOp
+ }
+
+ @Override
+ public boolean shouldEmbed(String fieldName, ATypeTag typeTag) {
+ return false;
+ }
+
+ @Override
+ public boolean shouldEmbed(IValueReference fieldName, ATypeTag typeTag) {
+ return false;
+ }
+
+ @Override
+ public IValueReference getEmbeddedValue() {
+ throw new IllegalAccessError("Cannot embed a value to " + this.getClass().getName());
+ }
+
+ @Override
+ public boolean IsMissingEmbeddedValues() {
+ return false;
+ }
+
+ @Override
+ public boolean isMissing(String fieldName) {
+ return false;
+ }
+
+ @Override
+ public String[] getEmbeddedFieldNames() {
+ return null;
+ }
+
+ @Override
+ public void enterObject() {
+ // NoOp
+ }
+
+ @Override
+ public void exitObject() {
+ // NoOp
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/embedder/EmbeddedValueBuilder.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/embedder/EmbeddedValueBuilder.java
new file mode 100644
index 0000000..ca6921b
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/embedder/EmbeddedValueBuilder.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.filter.embedder;
+
+import java.io.IOException;
+import java.util.BitSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+
+import org.apache.asterix.builders.RecordBuilder;
+import org.apache.asterix.dataflow.data.nontagged.serde.AStringSerializerDeserializer;
+import org.apache.asterix.external.input.filter.ExternalFilterValueEvaluator;
+import org.apache.asterix.external.util.ExternalDataPrefix;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.util.string.UTF8StringWriter;
+
+class EmbeddedValueBuilder {
+ private final ARecordType allPaths;
+ private final ExternalDataPrefix prefix;
+ private final Map<IAType, BitSet> setValues;
+ private final Stack<RecordBuilder> recordBuilders;
+ private final Map<IAType, ArrayBackedValueStorage> embeddedValues;
+ private final Map<String, IValueReference> fieldNames;
+ private final AStringSerializerDeserializer stringSerDer;
+
+ EmbeddedValueBuilder(ARecordType allPaths, ExternalDataPrefix prefix, Map<IAType, BitSet> setValues) {
+ this.allPaths = allPaths;
+ this.prefix = prefix;
+ this.setValues = setValues;
+ recordBuilders = new Stack<>();
+ embeddedValues = new HashMap<>();
+ fieldNames = new HashMap<>();
+ stringSerDer = new AStringSerializerDeserializer(new UTF8StringWriter());
+ setValues.put(allPaths, new BitSet(allPaths.getFieldTypes().length));
+ }
+
+ void build(String path) throws HyracksDataException {
+ List<String> values = prefix.getValues(path);
+ build(allPaths, values);
+ }
+
+ IValueReference getValue(IAType type) {
+ return embeddedValues.get(type);
+ }
+
+ private IValueReference build(ARecordType recordType, List<String> values) throws HyracksDataException {
+ RecordBuilder recordBuilder = getOrCreateRecordBuilder(recordType);
+ String[] fieldNames = recordType.getFieldNames();
+ IAType[] fieldTypes = recordType.getFieldTypes();
+
+ for (int i = 0; i < fieldTypes.length; i++) {
+ IValueReference valueRef = getValue(fieldTypes[i], values);
+ IValueReference fieldNameRef = getOrCreateFieldName(fieldNames[i]);
+ recordBuilder.addField(fieldNameRef, valueRef);
+ }
+
+ ArrayBackedValueStorage storage = getOrCreateValueStorage(recordType);
+ recordBuilder.write(storage.getDataOutput(), true);
+ recordBuilders.push(recordBuilder);
+ return storage;
+ }
+
+ private IValueReference build(IAType flatType, List<String> values) throws HyracksDataException {
+ ProjectionFiltrationTypeUtil.RenamedType leaf = (ProjectionFiltrationTypeUtil.RenamedType) flatType;
+ ArrayBackedValueStorage storage = getOrCreateValueStorage(flatType);
+ ATypeTag typeTag = prefix.getComputedFieldTypes().get(leaf.getIndex()).getTypeTag();
+ ExternalFilterValueEvaluator.writeValue(typeTag, values.get(leaf.getIndex()), storage, stringSerDer);
+ return storage;
+ }
+
+ private IValueReference getValue(IAType type, List<String> values) throws HyracksDataException {
+ ATypeTag typeTag = type.getTypeTag();
+ if (typeTag == ATypeTag.OBJECT) {
+ return build((ARecordType) type, values);
+ } else {
+ return build(type, values);
+ }
+ }
+
+ private RecordBuilder getOrCreateRecordBuilder(ARecordType recordType) {
+ RecordBuilder recordBuilder;
+ if (recordBuilders.isEmpty()) {
+ recordBuilder = new RecordBuilder();
+ setValues.put(recordType, new BitSet(recordType.getFieldTypes().length));
+ } else {
+ recordBuilder = recordBuilders.pop();
+ }
+ recordBuilder.reset(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE);
+ return recordBuilder;
+ }
+
+ private ArrayBackedValueStorage getOrCreateValueStorage(IAType type) {
+ ArrayBackedValueStorage storage = embeddedValues.computeIfAbsent(type, k -> new ArrayBackedValueStorage());
+ storage.reset();
+ return storage;
+ }
+
+ private IValueReference getOrCreateFieldName(String fieldName) throws HyracksDataException {
+ IValueReference fieldNameRef = fieldNames.get(fieldName);
+ if (fieldNameRef == null) {
+ ArrayBackedValueStorage storage = new ArrayBackedValueStorage();
+ serializeTag(ATypeTag.STRING, storage);
+ stringSerDer.serialize(fieldName, storage.getDataOutput());
+ fieldNameRef = storage;
+ fieldNames.put(fieldName, fieldNameRef);
+ }
+
+ return fieldNameRef;
+ }
+
+ private static void serializeTag(ATypeTag tag, ArrayBackedValueStorage storage) throws HyracksDataException {
+ try {
+ storage.getDataOutput().writeByte(tag.serialize());
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/embedder/ExternalFilterValueEmbedder.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/embedder/ExternalFilterValueEmbedder.java
new file mode 100644
index 0000000..358cb64
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/embedder/ExternalFilterValueEmbedder.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.filter.embedder;
+
+import java.util.BitSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+
+import org.apache.asterix.external.util.ExternalDataPrefix;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+public final class ExternalFilterValueEmbedder implements IExternalFilterValueEmbedder {
+ private final ARecordType allPaths;
+ private final IWarningCollector warningCollector;
+ private final Map<IAType, BitSet> setValues;
+ private final EmbeddedValueBuilder builder;
+ private final Stack<ARecordType> parents;
+ private final int numberOfValues;
+ private int level;
+ private int numberOfSetValues;
+ private IAType currentType;
+ private BitSet currentSetValues;
+
+ public ExternalFilterValueEmbedder(ARecordType allPaths, List<ProjectionFiltrationTypeUtil.RenamedType> leafs,
+ ExternalDataPrefix prefix, IWarningCollector warningCollector) {
+ this.allPaths = allPaths;
+ this.warningCollector = warningCollector;
+ setValues = new HashMap<>();
+ builder = new EmbeddedValueBuilder(allPaths, prefix, setValues);
+ parents = new Stack<>();
+ numberOfValues = leafs.size();
+ reset();
+ }
+
+ @Override
+ public void reset() {
+ currentType = allPaths;
+ numberOfSetValues = 0;
+ currentSetValues = setValues.get(allPaths);
+ currentSetValues.clear();
+ parents.clear();
+ parents.push(allPaths);
+ level = 0;
+ }
+
+ @Override
+ public void setPath(String path) throws HyracksDataException {
+ builder.build(path);
+ }
+
+ @Override
+ public boolean shouldEmbed(String fieldName, ATypeTag typeTag) {
+ return shouldEmbed(fieldName, typeTag, false);
+ }
+
+ @Override
+ public boolean shouldEmbed(IValueReference fieldName, ATypeTag typeTag) {
+ throw new IllegalAccessError("Should not be invoked");
+ }
+
+ @Override
+ public IValueReference getEmbeddedValue() {
+ IValueReference value = builder.getValue(currentType);
+ if (currentType.getTypeTag() != ATypeTag.OBJECT) {
+ numberOfSetValues++;
+ currentType = parents.peek();
+ } else if (currentType != allPaths) {
+ currentType = parents.pop();
+ }
+ return value;
+ }
+
+ @Override
+ public boolean IsMissingEmbeddedValues() {
+ if (currentType.getTypeTag() == ATypeTag.MISSING) {
+ return false;
+ }
+ int unsetValue = currentSetValues.nextClearBit(0);
+ int numberOfFields = ((ARecordType) currentType).getFieldTypes().length;
+ return unsetValue <= numberOfFields - 1;
+ }
+
+ @Override
+ public boolean isMissing(String fieldName) {
+ ARecordType parent = (ARecordType) currentType;
+ return !currentSetValues.get(parent.getFieldIndex(fieldName)) && shouldEmbed(fieldName, ATypeTag.ANY, true);
+ }
+
+ @Override
+ public String[] getEmbeddedFieldNames() {
+ return ((ARecordType) currentType).getFieldNames();
+ }
+
+ @Override
+ public void enterObject() {
+ level++;
+ if (level != parents.size()) {
+ parents.push((ARecordType) currentType);
+ currentType = BuiltinType.AMISSING;
+ }
+ }
+
+ @Override
+ public void exitObject() {
+ if (currentType != allPaths) {
+ currentType = parents.pop();
+ currentSetValues = setValues.get(currentType);
+ }
+ level--;
+ }
+
+ private boolean shouldEmbed(String fieldName, ATypeTag typeTag, boolean includeObjects) {
+ if (currentType.getTypeTag() == ATypeTag.MISSING || numberOfSetValues == numberOfValues) {
+ return false;
+ }
+
+ ARecordType recordType = (ARecordType) currentType;
+ int index = recordType.getFieldIndex(fieldName);
+
+ if (index < 0 || currentSetValues.get(index)) {
+ return false;
+ }
+
+ currentSetValues.set(index);
+ IAType fieldType = recordType.getFieldTypes()[index];
+ if (fieldType.getTypeTag() == ATypeTag.OBJECT) {
+ ARecordType newParent = (ARecordType) fieldType;
+ parents.push(recordType);
+ currentType = fieldType;
+ currentSetValues = setValues.get(newParent);
+ currentSetValues.clear();
+ return includeObjects || typeTag != ATypeTag.OBJECT;
+ } else {
+ currentType = fieldType;
+ return true;
+ }
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/embedder/IExternalFilterValueEmbedder.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/embedder/IExternalFilterValueEmbedder.java
new file mode 100644
index 0000000..e4623c5
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/embedder/IExternalFilterValueEmbedder.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.filter.embedder;
+
+import org.apache.asterix.common.external.IExternalFilterEmbeddedValueInformation;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+public interface IExternalFilterValueEmbedder extends IExternalFilterEmbeddedValueInformation {
+ void setPath(String path) throws HyracksDataException;
+
+ void reset();
+
+ boolean shouldEmbed(String fieldName, ATypeTag typeTag);
+
+ boolean shouldEmbed(IValueReference fieldName, ATypeTag typeTag);
+
+ void enterObject();
+
+ void exitObject();
+
+ IValueReference getEmbeddedValue();
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStream.java
index b37bce7..8ef19ad 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStream.java
@@ -60,10 +60,13 @@
CleanupUtils.close(in, null);
}
+ String path = filePaths.get(nextFileIndex);
boolean isAvailableStream = getInputStream();
nextFileIndex++; // Always point to next file after getting the current stream
if (!isAvailableStream) {
return advance();
+ } else {
+ valueEmbedder.setPath(path);
}
if (notificationHandler != null) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStreamFactory.java
index 37fd910..34e3d2d 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStreamFactory.java
@@ -29,6 +29,7 @@
import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IInputStreamFactory;
+import org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
@@ -42,6 +43,7 @@
protected Map<String, String> configuration;
protected final List<PartitionWorkLoadBasedOnSize> partitionWorkLoadsBasedOnSize = new ArrayList<>();
+ protected IExternalFilterEvaluatorFactory filterEvaluatorFactory;
protected transient AlgebricksAbsolutePartitionConstraint partitionConstraint;
@Override
@@ -68,6 +70,12 @@
this.configuration = configuration;
this.partitionConstraint =
((ICcApplicationContext) ctx.getApplicationContext()).getClusterStateManager().getClusterLocations();
+ this.filterEvaluatorFactory = filterEvaluatorFactory;
+ }
+
+ @Override
+ public IExternalFilterValueEmbedder createFilterValueEmbedder(IWarningCollector warningCollector) {
+ return filterEvaluatorFactory.createValueEmbedder(warningCollector);
}
public static class PartitionWorkLoadBasedOnSize implements Serializable {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
index e99c0ef..5b4e314 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
@@ -57,7 +57,7 @@
// prepare prefix for computed field calculations
IExternalFilterEvaluator evaluator = filterEvaluatorFactory.create(ctx, warningCollector);
- ExternalDataPrefix externalDataPrefix = new ExternalDataPrefix(configuration, warningCollector);
+ ExternalDataPrefix externalDataPrefix = new ExternalDataPrefix(configuration);
configuration.put(ExternalDataPrefix.PREFIX_ROOT_FIELD_NAME, externalDataPrefix.getRoot());
// get the items
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3ReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3ReaderFactory.java
index 6a21c79..bfdace2 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3ReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3ReaderFactory.java
@@ -22,15 +22,8 @@
import java.util.List;
import java.util.Map;
-import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
import org.apache.asterix.external.input.record.reader.stream.StreamRecordReaderFactory;
-import org.apache.asterix.external.provider.StreamRecordReaderProvider;
import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.api.application.IServiceContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.exceptions.IWarningCollector;
public class AwsS3ReaderFactory extends StreamRecordReaderFactory {
@@ -45,30 +38,7 @@
}
@Override
- public DataSourceType getDataSourceType() {
- return DataSourceType.RECORDS;
- }
-
- @Override
- public Class<?> getRecordClass() {
- return char[].class;
- }
-
- @Override
- public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() throws AlgebricksException {
- return streamFactory.getPartitionConstraint();
- }
-
- @Override
- public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector,
- IExternalFilterEvaluatorFactory filterEvaluatorFactory) throws AlgebricksException, HyracksDataException {
- this.configuration = configuration;
-
- // Stream factory
+ protected void setStreamFactory(Map<String, String> config) {
streamFactory = new AwsS3InputStreamFactory();
- streamFactory.configure(ctx, configuration, warningCollector, filterEvaluatorFactory);
-
- // record reader
- recordReaderClazz = StreamRecordReaderProvider.getRecordReaderClazz(configuration);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java
index 88aeb1b..98b5b06 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java
@@ -69,7 +69,7 @@
// prepare prefix for computed field calculations
IExternalFilterEvaluator evaluator = filterEvaluatorFactory.create(serviceCtx, warningCollector);
- ExternalDataPrefix externalDataPrefix = new ExternalDataPrefix(configuration, warningCollector);
+ ExternalDataPrefix externalDataPrefix = new ExternalDataPrefix(configuration);
configuration.put(ExternalDataPrefix.PREFIX_ROOT_FIELD_NAME, externalDataPrefix.getRoot());
String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
@@ -124,7 +124,6 @@
*
* @param container container
* @param filesOnly files
- *
* @return Comma-delimited paths (e.g., "s3a://bucket/file1.parquet,s3a://bucket/file2.parquet")
*/
private static String buildPathURIs(String container, List<S3Object> filesOnly) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/blob/AzureBlobInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/blob/AzureBlobInputStreamFactory.java
index 173cc15..ada68ac 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/blob/AzureBlobInputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/blob/AzureBlobInputStreamFactory.java
@@ -63,7 +63,7 @@
// prepare prefix for computed field calculations
IExternalFilterEvaluator evaluator = filterEvaluatorFactory.create(ctx, warningCollector);
- ExternalDataPrefix externalDataPrefix = new ExternalDataPrefix(configuration, warningCollector);
+ ExternalDataPrefix externalDataPrefix = new ExternalDataPrefix(configuration);
configuration.put(ExternalDataPrefix.PREFIX_ROOT_FIELD_NAME, externalDataPrefix.getRoot());
List<BlobItem> filesOnly = listBlobItems(appCtx, configuration, includeExcludeMatcher, warningCollector,
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/blob/AzureBlobReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/blob/AzureBlobReaderFactory.java
index 0f4d6ba..0a7d75e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/blob/AzureBlobReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/blob/AzureBlobReaderFactory.java
@@ -22,15 +22,8 @@
import java.util.List;
import java.util.Map;
-import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
import org.apache.asterix.external.input.record.reader.stream.StreamRecordReaderFactory;
-import org.apache.asterix.external.provider.StreamRecordReaderProvider;
import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.api.application.IServiceContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.exceptions.IWarningCollector;
public class AzureBlobReaderFactory extends StreamRecordReaderFactory {
@@ -45,30 +38,7 @@
}
@Override
- public DataSourceType getDataSourceType() {
- return DataSourceType.RECORDS;
- }
-
- @Override
- public Class<?> getRecordClass() {
- return char[].class;
- }
-
- @Override
- public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() throws AlgebricksException {
- return streamFactory.getPartitionConstraint();
- }
-
- @Override
- public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector,
- IExternalFilterEvaluatorFactory filterEvaluatorFactory) throws AlgebricksException, HyracksDataException {
- this.configuration = configuration;
-
- // Stream factory
+ protected void setStreamFactory(Map<String, String> config) {
streamFactory = new AzureBlobInputStreamFactory();
- streamFactory.configure(ctx, configuration, warningCollector, filterEvaluatorFactory);
-
- // record reader
- recordReaderClazz = StreamRecordReaderProvider.getRecordReaderClazz(configuration);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeReaderFactory.java
index 6f4685c..74c5f5e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeReaderFactory.java
@@ -22,15 +22,8 @@
import java.util.List;
import java.util.Map;
-import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
import org.apache.asterix.external.input.record.reader.stream.StreamRecordReaderFactory;
-import org.apache.asterix.external.provider.StreamRecordReaderProvider;
import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.api.application.IServiceContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.exceptions.IWarningCollector;
public class AzureDataLakeReaderFactory extends StreamRecordReaderFactory {
@@ -45,30 +38,7 @@
}
@Override
- public DataSourceType getDataSourceType() {
- return DataSourceType.RECORDS;
- }
-
- @Override
- public Class<?> getRecordClass() {
- return char[].class;
- }
-
- @Override
- public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() throws AlgebricksException {
- return streamFactory.getPartitionConstraint();
- }
-
- @Override
- public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector,
- IExternalFilterEvaluatorFactory filterEvaluatorFactory) throws AlgebricksException, HyracksDataException {
- this.configuration = configuration;
-
- // Stream factory
+ protected void setStreamFactory(Map<String, String> config) {
streamFactory = new AzureDataLakeInputStreamFactory();
- streamFactory.configure(ctx, configuration, warningCollector, filterEvaluatorFactory);
-
- // record reader
- recordReaderClazz = StreamRecordReaderProvider.getRecordReaderClazz(configuration);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/parquet/AzureBlobParquetReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/parquet/AzureBlobParquetReaderFactory.java
index cec29d4..25ac181 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/parquet/AzureBlobParquetReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/parquet/AzureBlobParquetReaderFactory.java
@@ -65,7 +65,7 @@
// prepare prefix for computed field calculations
IExternalFilterEvaluator evaluator = filterEvaluatorFactory.create(serviceCtx, warningCollector);
- ExternalDataPrefix externalDataPrefix = new ExternalDataPrefix(configuration, warningCollector);
+ ExternalDataPrefix externalDataPrefix = new ExternalDataPrefix(configuration);
configuration.put(ExternalDataPrefix.PREFIX_ROOT_FIELD_NAME, externalDataPrefix.getRoot());
List<BlobItem> filesOnly = listBlobItems(blobServiceClient, configuration, includeExcludeMatcher,
@@ -110,8 +110,7 @@
*
* @param container container
* @param filesOnly files
- * @param endPoint endpoint
- *
+ * @param endPoint endpoint
* @return Comma-delimited paths (e.g., "wasbs://container@accountName.blob.core.windows.net/file1.parquet,
* wasbs://container@accountName.blob.core.windows.net/file2.parquet")
*/
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStreamFactory.java
index 89ff24b..74f6428 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStreamFactory.java
@@ -18,8 +18,6 @@
*/
package org.apache.asterix.external.input.record.reader.gcs;
-import static org.apache.asterix.external.util.ExternalDataUtils.getIncludeExcludeMatchers;
-
import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -59,7 +57,7 @@
// prepare prefix for computed field calculations
IExternalFilterEvaluator evaluator = filterEvaluatorFactory.create(ctx, warningCollector);
- ExternalDataPrefix externalDataPrefix = new ExternalDataPrefix(configuration, warningCollector);
+ ExternalDataPrefix externalDataPrefix = new ExternalDataPrefix(configuration);
configuration.put(ExternalDataPrefix.PREFIX_ROOT_FIELD_NAME, externalDataPrefix.getRoot());
// get the items
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSReaderFactory.java
index 981a29d..7c76358 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSReaderFactory.java
@@ -22,15 +22,9 @@
import java.util.List;
import java.util.Map;
-import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
+import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.external.input.record.reader.stream.StreamRecordReaderFactory;
-import org.apache.asterix.external.provider.StreamRecordReaderProvider;
import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.api.application.IServiceContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.exceptions.IWarningCollector;
public class GCSReaderFactory extends StreamRecordReaderFactory {
@@ -45,30 +39,7 @@
}
@Override
- public DataSourceType getDataSourceType() {
- return DataSourceType.RECORDS;
- }
-
- @Override
- public Class<?> getRecordClass() {
- return char[].class;
- }
-
- @Override
- public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() throws AlgebricksException {
- return streamFactory.getPartitionConstraint();
- }
-
- @Override
- public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector,
- IExternalFilterEvaluatorFactory filterEvaluatorFactory) throws AlgebricksException, HyracksDataException {
- this.configuration = configuration;
-
- // Stream factory
+ protected void setStreamFactory(Map<String, String> config) throws CompilationException {
streamFactory = new GCSInputStreamFactory();
- streamFactory.configure(ctx, configuration, warningCollector, filterEvaluatorFactory);
-
- // record reader
- recordReaderClazz = StreamRecordReaderProvider.getRecordReaderClazz(configuration);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/parquet/GCSParquetReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/parquet/GCSParquetReaderFactory.java
index bb98abe..50a1039 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/parquet/GCSParquetReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/parquet/GCSParquetReaderFactory.java
@@ -55,7 +55,7 @@
// prepare prefix for computed field calculations
IExternalFilterEvaluator evaluator = filterEvaluatorFactory.create(serviceCtx, warningCollector);
- ExternalDataPrefix externalDataPrefix = new ExternalDataPrefix(configuration, warningCollector);
+ ExternalDataPrefix externalDataPrefix = new ExternalDataPrefix(configuration);
configuration.put(ExternalDataPrefix.PREFIX_ROOT_FIELD_NAME, externalDataPrefix.getRoot());
String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
@@ -101,7 +101,6 @@
*
* @param container container
* @param filesOnly files
- *
* @return Comma-delimited paths (e.g., "gs://bucket/file1.parquet,gs://bucket/file2.parquet")
*/
private static String buildPathURIs(String container, List<Blob> filesOnly) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReader.java
index 937f3fe..9d12325 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReader.java
@@ -31,6 +31,7 @@
import org.apache.asterix.external.api.IRecordReader;
import org.apache.asterix.external.api.IStreamNotificationHandler;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
+import org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
import org.apache.asterix.external.input.record.CharArrayRecord;
import org.apache.asterix.external.input.stream.AsterixInputStreamReader;
import org.apache.asterix.external.util.ExternalDataUtils;
@@ -48,9 +49,11 @@
protected IFeedLogManager feedLogManager;
private Supplier<String> dataSourceName = EMPTY_STRING;
private Supplier<String> previousDataSourceName = EMPTY_STRING;
+ private AsterixInputStream inputStream;
public void configure(AsterixInputStream inputStream, Map<String, String> config) {
int bufferSize = ExternalDataUtils.getOrDefaultBufferSize(config);
+ this.inputStream = inputStream;
this.reader = new AsterixInputStreamReader(inputStream, bufferSize);
record = new CharArrayRecord();
inputBuffer = new char[bufferSize];
@@ -120,6 +123,11 @@
return dataSourceName;
}
+ @Override
+ public void setValueEmbedder(IExternalFilterValueEmbedder valueEmbedder) {
+ inputStream.setValueEmbedder(valueEmbedder);
+ }
+
String getPreviousStreamName() {
return previousDataSourceName.get();
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
index d9bbd01..c9567c5 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
@@ -30,6 +30,7 @@
import org.apache.asterix.external.api.IInputStreamFactory;
import org.apache.asterix.external.api.IRecordReader;
import org.apache.asterix.external.api.IRecordReaderFactory;
+import org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
import org.apache.asterix.external.input.stream.factory.LocalFSInputStreamFactory;
import org.apache.asterix.external.input.stream.factory.SocketClientInputStreamFactory;
import org.apache.asterix.external.input.stream.factory.SocketServerInputStreamFactory;
@@ -48,52 +49,40 @@
protected IInputStreamFactory streamFactory;
protected Map<String, String> configuration;
protected Class recordReaderClazz;
+ protected IExternalFilterEvaluatorFactory filterEvaluatorFactory;
private static final List<String> recordReaderNames =
Collections.unmodifiableList(Arrays.asList(ExternalDataConstants.ALIAS_LOCALFS_ADAPTER,
ExternalDataConstants.KEY_ALIAS_ADAPTER_NAME_SOCKET, ExternalDataConstants.KEY_ADAPTER_NAME_SOCKET,
ExternalDataConstants.STREAM_SOCKET_CLIENT));
@Override
- public DataSourceType getDataSourceType() {
+ public final DataSourceType getDataSourceType() {
return DataSourceType.RECORDS;
}
@Override
- public Class<?> getRecordClass() {
+ public final Class<?> getRecordClass() {
return char[].class;
}
@Override
- public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() throws AlgebricksException {
+ public final AlgebricksAbsolutePartitionConstraint getPartitionConstraint() throws AlgebricksException {
return streamFactory.getPartitionConstraint();
}
- private void configureInputStreamFactory(Map<String, String> config) throws CompilationException {
- String reader = config.get(ExternalDataConstants.KEY_READER);
- if (reader.equals(ExternalDataConstants.ALIAS_LOCALFS_ADAPTER)) {
- streamFactory = new LocalFSInputStreamFactory();
- } else if (reader.equals(ExternalDataConstants.KEY_ALIAS_ADAPTER_NAME_SOCKET)
- || reader.equals(ExternalDataConstants.KEY_ADAPTER_NAME_SOCKET)) {
- streamFactory = new SocketServerInputStreamFactory();
- } else if (reader.equals(ExternalDataConstants.STREAM_SOCKET_CLIENT)) {
- streamFactory = new SocketClientInputStreamFactory();
- } else {
- throw new CompilationException(ErrorCode.FEED_UNKNOWN_ADAPTER_NAME);
- }
- }
-
@Override
- public void configure(IServiceContext serviceCtx, Map<String, String> configuration,
+ public final void configure(IServiceContext serviceCtx, Map<String, String> configuration,
IWarningCollector warningCollector, IExternalFilterEvaluatorFactory filterEvaluatorFactory)
throws HyracksDataException, AlgebricksException {
this.configuration = configuration;
- configureInputStreamFactory(configuration);
+ setStreamFactory(configuration);
streamFactory.configure(serviceCtx, configuration, warningCollector, filterEvaluatorFactory);
recordReaderClazz = StreamRecordReaderProvider.getRecordReaderClazz(configuration);
+ this.filterEvaluatorFactory = filterEvaluatorFactory;
}
@Override
- public IRecordReader<? extends char[]> createRecordReader(IHyracksTaskContext ctx, int partition)
+ public final IRecordReader<? extends char[]> createRecordReader(IHyracksTaskContext ctx, int partition)
throws HyracksDataException {
try {
StreamRecordReader streamRecordReader =
@@ -107,7 +96,26 @@
}
@Override
+ public final IExternalFilterValueEmbedder createFilterValueEmbedder(IWarningCollector warningCollector) {
+ return filterEvaluatorFactory.createValueEmbedder(warningCollector);
+ }
+
+ @Override
public List<String> getRecordReaderNames() {
return recordReaderNames;
}
+
+ protected void setStreamFactory(Map<String, String> config) throws CompilationException {
+ String reader = config.get(ExternalDataConstants.KEY_READER);
+ if (reader.equals(ExternalDataConstants.ALIAS_LOCALFS_ADAPTER)) {
+ streamFactory = new LocalFSInputStreamFactory();
+ } else if (reader.equals(ExternalDataConstants.KEY_ALIAS_ADAPTER_NAME_SOCKET)
+ || reader.equals(ExternalDataConstants.KEY_ADAPTER_NAME_SOCKET)) {
+ streamFactory = new SocketServerInputStreamFactory();
+ } else if (reader.equals(ExternalDataConstants.STREAM_SOCKET_CLIENT)) {
+ streamFactory = new SocketClientInputStreamFactory();
+ } else {
+ throw new CompilationException(ErrorCode.FEED_UNKNOWN_ADAPTER_NAME);
+ }
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
index c70cff4..0647763 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
@@ -24,7 +24,6 @@
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
-import org.apache.asterix.common.external.NoOpExternalFilterEvaluatorFactory;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.common.library.ILibrary;
import org.apache.asterix.common.library.ILibraryManager;
@@ -32,6 +31,7 @@
import org.apache.asterix.external.api.ITypedAdapterFactory;
import org.apache.asterix.external.feed.api.IFeed;
import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.input.filter.NoOpExternalFilterEvaluatorFactory;
import org.apache.asterix.external.library.JavaLibrary;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.api.context.IHyracksTaskContext;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
index 4f8b628..d9e374f 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
@@ -30,6 +30,8 @@
import org.apache.asterix.builders.IAsterixListBuilder;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
+import org.apache.asterix.external.input.filter.NoOpFilterValueEmbedder;
+import org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
import org.apache.asterix.external.parser.jackson.ADMToken;
import org.apache.asterix.external.parser.jackson.GeometryCoParser;
import org.apache.asterix.external.parser.jackson.ParserContext;
@@ -48,6 +50,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.util.ExceptionUtils;
import org.apache.hyracks.data.std.api.IMutableValueStorage;
+import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.hyracks.util.ParseUtil;
@@ -71,6 +74,8 @@
protected JsonParser jsonParser;
+ protected IExternalFilterValueEmbedder valueEmbedder;
+
/**
* Initialize JSONDataParser with GeometryCoParser
*
@@ -84,6 +89,7 @@
//GeometryCoParser to parse GeoJSON objects to AsterixDB internal spatial types.
geometryCoParser = new GeometryCoParser(jsonParser);
parserContext = new ParserContext();
+ valueEmbedder = NoOpFilterValueEmbedder.INSTANCE;
}
/*
@@ -189,6 +195,7 @@
final IMutableValueStorage valueBuffer = parserContext.enterObject();
final IARecordBuilder objectBuilder = parserContext.getObjectBuilder(recordType);
final BitSet nullBitMap = parserContext.getNullBitmap(recordType.getFieldTypes().length);
+ valueEmbedder.enterObject();
while (nextToken() != ADMToken.OBJECT_END) {
/*
* Jackson parser calls String.intern() for field names (if enabled).
@@ -203,11 +210,17 @@
}
valueBuffer.reset();
nextToken();
-
if (fieldIndex < 0) {
- //field is not defined and the type is open
- parseValue(BuiltinType.ANY, valueBuffer.getDataOutput());
- objectBuilder.addField(parserContext.getSerializedFieldName(fieldName), valueBuffer);
+ IValueReference fieldValue;
+ // field is not defined and the type is open
+ if (valueEmbedder.shouldEmbed(fieldName, currentToken().getTypeTag())) {
+ // It is an embedded value, set it
+ fieldValue = valueEmbedder.getEmbeddedValue();
+ } else {
+ fieldValue = valueBuffer;
+ parseValue(BuiltinType.ANY, valueBuffer.getDataOutput());
+ }
+ objectBuilder.addField(parserContext.getSerializedFieldName(fieldName), fieldValue);
} else {
//field is defined
final IAType fieldType = recordType.getFieldType(fieldName);
@@ -231,6 +244,18 @@
if (nullBitMap != null) {
checkOptionalConstraints(recordType, nullBitMap);
}
+
+ if (valueEmbedder.IsMissingEmbeddedValues()) {
+ String[] embeddedFieldNames = valueEmbedder.getEmbeddedFieldNames();
+ for (int i = 0; i < embeddedFieldNames.length; i++) {
+ String embeddedFieldName = embeddedFieldNames[i];
+ if (valueEmbedder.isMissing(embeddedFieldName)) {
+ IValueReference embeddedValue = valueEmbedder.getEmbeddedValue();
+ objectBuilder.addField(parserContext.getSerializedFieldName(embeddedFieldName), embeddedValue);
+ }
+ }
+ }
+ valueEmbedder.exitObject();
parserContext.exitObject(valueBuffer, nullBitMap, objectBuilder);
objectBuilder.write(out, true);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java
index b2cffa9..20b466b 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java
@@ -31,6 +31,7 @@
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordDataParser;
import org.apache.asterix.external.api.IStreamDataParser;
+import org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
import org.apache.asterix.external.parser.jackson.ADMToken;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.om.types.ARecordType;
@@ -49,10 +50,8 @@
/**
* Initialize JSONDataParser
*
- * @param recordType
- * defined type.
- * @param jsonFactory
- * Jackson JSON parser factory.
+ * @param recordType defined type.
+ * @param jsonFactory Jackson JSON parser factory.
*/
public JSONDataParser(ARecordType recordType, JsonFactory jsonFactory) {
super(recordType, jsonFactory);
@@ -63,6 +62,11 @@
setInput(jsonFactory.createParser(in));
}
+ @Override
+ public void setValueEmbedder(IExternalFilterValueEmbedder valueEmbedder) {
+ this.valueEmbedder = valueEmbedder;
+ }
+
public void setInputNode(JsonNode node) {
setInput(new TreeTraversingParser(node));
}
@@ -81,6 +85,7 @@
if (nextToken() != ADMToken.OBJECT_START) {
throw new ParseException(PARSER_DATA_PARSER_UNEXPECTED_TOKEN, currentToken(), ADMToken.OBJECT_START);
}
+ valueEmbedder.reset();
parseObject(rootType, out);
return true;
} catch (IOException e) {
@@ -100,6 +105,7 @@
if (nextToken() == ADMToken.EOF) {
return false;
}
+ valueEmbedder.reset();
parseObject(rootType, out);
return true;
} catch (IOException e) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
index 8902e82..83ec7e7 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
@@ -44,6 +44,7 @@
import org.apache.asterix.external.dataflow.FeedWithMetaDataFlowController;
import org.apache.asterix.external.dataflow.RecordDataFlowController;
import org.apache.asterix.external.dataflow.StreamDataFlowController;
+import org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.external.util.IFeedLogManager;
import org.apache.asterix.om.types.ARecordType;
@@ -61,15 +62,19 @@
int partition, IExternalDataSourceFactory dataSourceFactory, IDataParserFactory dataParserFactory,
Map<String, String> configuration, boolean isFeed, IFeedLogManager feedLogManager)
throws HyracksDataException {
+ IExternalFilterValueEmbedder valueEmbedder =
+ dataSourceFactory.createFilterValueEmbedder(ctx.getWarningCollector());
try {
switch (dataSourceFactory.getDataSourceType()) {
case RECORDS:
IRecordReaderFactory<?> recordReaderFactory = (IRecordReaderFactory<?>) dataSourceFactory;
IRecordReader<?> recordReader = recordReaderFactory.createRecordReader(ctx, partition);
+ recordReader.setValueEmbedder(valueEmbedder);
IRecordDataParserFactory<?> recordParserFactory = (IRecordDataParserFactory<?>) dataParserFactory;
IRecordDataParser<?> dataParser = recordParserFactory.createRecordParser(ctx);
// TODO(ali): revisit to think about passing data source name via setter or via createRecordParser
dataParser.configure(recordReader.getDataSourceName(), recordReader.getLineNumber());
+ dataParser.setValueEmbedder(valueEmbedder);
if (isFeed) {
boolean isChangeFeed = ExternalDataUtils.isChangeFeed(configuration);
boolean isRecordWithMeta = ExternalDataUtils.isRecordWithMeta(configuration);
@@ -95,10 +100,12 @@
case STREAM:
IInputStreamFactory streamFactory = (IInputStreamFactory) dataSourceFactory;
AsterixInputStream stream = streamFactory.createInputStream(ctx, partition);
+ stream.setValueEmbedder(valueEmbedder);
IStreamDataParserFactory streamParserFactory = (IStreamDataParserFactory) dataParserFactory;
// TODO(ali): revisit to think about passing data source name to parser
IStreamDataParser streamParser = streamParserFactory.createInputStreamParser(ctx, partition);
streamParser.setInputStream(stream);
+ streamParser.setValueEmbedder(valueEmbedder);
if (isFeed) {
return new FeedStreamDataFlowController(ctx, feedLogManager, streamParser, stream);
} else {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index bc2ce63..0bd274b 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -134,6 +134,8 @@
public static final String KEY_REQUESTED_FIELDS = "requested-fields";
public static final String KEY_EXTERNAL_SCAN_BUFFER_SIZE = "external-scan-buffer-size";
+ public static final String KEY_EMBED_FILTER_VALUES = "embed-filter-values";
+
/**
* Keys for adapter name
**/
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
index f8076ef..aa269fe 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
@@ -24,6 +24,7 @@
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_PATH;
import static org.apache.asterix.external.util.ExternalDataConstants.PREFIX_DEFAULT_DELIMITER;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -47,17 +48,15 @@
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.NoOpWarningCollector;
import org.apache.hyracks.api.exceptions.Warning;
import org.apache.hyracks.util.LogRedactionUtil;
-public class ExternalDataPrefix {
-
+public class ExternalDataPrefix implements Serializable {
+ private static final long serialVersionUID = -7612997190679310483L;
private final String original;
private String root;
private final boolean endsWithSlash;
private final List<String> segments;
- private final IWarningCollector warningCollector;
private final List<String> computedFieldNames = new ArrayList<>();
private final List<IAType> computedFieldTypes = new ArrayList<>();
@@ -75,16 +74,10 @@
}
public ExternalDataPrefix(Map<String, String> configuration) throws AlgebricksException {
- this(getDefinitionOrPath(configuration), null);
+ this(getDefinitionOrPath(configuration));
}
- public ExternalDataPrefix(Map<String, String> configuration, IWarningCollector warningCollector)
- throws AlgebricksException {
- this(getDefinitionOrPath(configuration), warningCollector);
- }
-
- public ExternalDataPrefix(String prefix, IWarningCollector warningCollector) throws AlgebricksException {
- this.warningCollector = warningCollector != null ? warningCollector : NoOpWarningCollector.INSTANCE;
+ public ExternalDataPrefix(String prefix) throws AlgebricksException {
this.original = prefix != null ? prefix : "";
this.endsWithSlash = this.original.endsWith("/");
@@ -237,6 +230,10 @@
}
}
+ public List<String> getValues(String key) {
+ return extractValues(extractPrefixSegments(key));
+ }
+
/**
* Evaluates whether the provided key satisfies the conditions of the evaluator or not
* TODO Check if {@link IExternalFilterEvaluator#isComputedFieldUsed(int)} is useful once we have regex extractor
@@ -245,7 +242,8 @@
* @param evaluator evaluator
* @return true if key satisfies the evaluator conditions, false otherwise
*/
- public boolean evaluate(String key, IExternalFilterEvaluator evaluator) throws HyracksDataException {
+ public boolean evaluate(String key, IExternalFilterEvaluator evaluator, IWarningCollector warningCollector)
+ throws HyracksDataException {
// TODO provide the List to avoid array creation
List<String> keySegments = extractPrefixSegments(key);
@@ -293,6 +291,11 @@
return evaluator.evaluate();
}
+ public static boolean containsComputedFields(Map<String, String> configuration) {
+ String path = getDefinitionOrPath(configuration);
+ return path != null && path.contains("{");
+ }
+
/**
* extracts the computed fields values from the object's key
*
@@ -334,7 +337,8 @@
return configuration.getOrDefault(DEFINITION_FIELD_NAME, configuration.get(KEY_PATH));
}
- public static class PrefixSegment {
+ public static class PrefixSegment implements Serializable {
+ private static final long serialVersionUID = 8788939199985336347L;
private String expression;
private final List<String> computedFieldNames = new ArrayList<>();
private final List<IAType> computedFieldTypes = new ArrayList<>();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 2605fe7..599d2d1 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -989,16 +989,17 @@
/**
* Tests the provided key against all the provided predicates/evaluators and return true if they all pass.
*
- * @param key key
- * @param predicate predicate
- * @param matchers matchers
+ * @param key key
+ * @param predicate predicate
+ * @param matchers matchers
* @param externalDataPrefix external data prefix
- * @param evaluator evaluator
- *
+ * @param evaluator evaluator
* @return true if key passes all tests, false otherwise
*/
public static boolean evaluate(String key, BiPredicate<List<Matcher>, String> predicate, List<Matcher> matchers,
- ExternalDataPrefix externalDataPrefix, IExternalFilterEvaluator evaluator) throws HyracksDataException {
- return !key.endsWith("/") && predicate.test(matchers, key) && externalDataPrefix.evaluate(key, evaluator);
+ ExternalDataPrefix externalDataPrefix, IExternalFilterEvaluator evaluator,
+ IWarningCollector warningCollector) throws HyracksDataException {
+ return !key.endsWith("/") && predicate.test(matchers, key)
+ && externalDataPrefix.evaluate(key, evaluator, warningCollector);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
index 1436e55..966b536 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
@@ -372,15 +372,15 @@
String prefix = getPrefix(configuration);
try {
- filesOnly =
- listS3Objects(s3Client, container, prefix, includeExcludeMatcher, externalDataPrefix, evaluator);
+ filesOnly = listS3Objects(s3Client, container, prefix, includeExcludeMatcher, externalDataPrefix, evaluator,
+ warningCollector);
} catch (S3Exception ex) {
// New API is not implemented, try falling back to old API
try {
// For error code, see https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
if (ex.awsErrorDetails().errorCode().equals(ERROR_METHOD_NOT_IMPLEMENTED)) {
filesOnly = oldApiListS3Objects(s3Client, container, prefix, includeExcludeMatcher,
- externalDataPrefix, evaluator);
+ externalDataPrefix, evaluator, warningCollector);
} else {
throw ex;
}
@@ -414,7 +414,8 @@
*/
private static List<S3Object> listS3Objects(S3Client s3Client, String container, String prefix,
AbstractExternalInputStreamFactory.IncludeExcludeMatcher includeExcludeMatcher,
- ExternalDataPrefix externalDataPrefix, IExternalFilterEvaluator evaluator) throws HyracksDataException {
+ ExternalDataPrefix externalDataPrefix, IExternalFilterEvaluator evaluator,
+ IWarningCollector warningCollector) throws HyracksDataException {
String newMarker = null;
List<S3Object> filesOnly = new ArrayList<>();
@@ -432,7 +433,8 @@
// Collect the paths to files only
collectAndFilterFiles(listObjectsResponse.contents(), includeExcludeMatcher.getPredicate(),
- includeExcludeMatcher.getMatchersList(), filesOnly, externalDataPrefix, evaluator);
+ includeExcludeMatcher.getMatchersList(), filesOnly, externalDataPrefix, evaluator,
+ warningCollector);
// Mark the flag as done if done, otherwise, get the marker of the previous response for the next request
if (!listObjectsResponse.isTruncated()) {
@@ -455,7 +457,8 @@
*/
private static List<S3Object> oldApiListS3Objects(S3Client s3Client, String container, String prefix,
AbstractExternalInputStreamFactory.IncludeExcludeMatcher includeExcludeMatcher,
- ExternalDataPrefix externalDataPrefix, IExternalFilterEvaluator evaluator) throws HyracksDataException {
+ ExternalDataPrefix externalDataPrefix, IExternalFilterEvaluator evaluator,
+ IWarningCollector warningCollector) throws HyracksDataException {
String newMarker = null;
List<S3Object> filesOnly = new ArrayList<>();
@@ -473,7 +476,8 @@
// Collect the paths to files only
collectAndFilterFiles(listObjectsResponse.contents(), includeExcludeMatcher.getPredicate(),
- includeExcludeMatcher.getMatchersList(), filesOnly, externalDataPrefix, evaluator);
+ includeExcludeMatcher.getMatchersList(), filesOnly, externalDataPrefix, evaluator,
+ warningCollector);
// Mark the flag as done if done, otherwise, get the marker of the previous response for the next request
if (!listObjectsResponse.isTruncated()) {
@@ -489,18 +493,19 @@
/**
* Collects only files that pass all tests
*
- * @param s3Objects s3 objects
- * @param predicate predicate
- * @param matchers matchers
- * @param filesOnly filtered files
+ * @param s3Objects s3 objects
+ * @param predicate predicate
+ * @param matchers matchers
+ * @param filesOnly filtered files
* @param externalDataPrefix external data prefix
- * @param evaluator evaluator
+ * @param evaluator evaluator
*/
private static void collectAndFilterFiles(List<S3Object> s3Objects, BiPredicate<List<Matcher>, String> predicate,
List<Matcher> matchers, List<S3Object> filesOnly, ExternalDataPrefix externalDataPrefix,
- IExternalFilterEvaluator evaluator) throws HyracksDataException {
+ IExternalFilterEvaluator evaluator, IWarningCollector warningCollector) throws HyracksDataException {
for (S3Object object : s3Objects) {
- if (ExternalDataUtils.evaluate(object.key(), predicate, matchers, externalDataPrefix, evaluator)) {
+ if (ExternalDataUtils.evaluate(object.key(), predicate, matchers, externalDataPrefix, evaluator,
+ warningCollector)) {
filesOnly.add(object);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java
index aced253..4062fc9 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java
@@ -425,7 +425,8 @@
// Collect the paths to files only
collectAndFilterBlobFiles(blobItems, includeExcludeMatcher.getPredicate(),
- includeExcludeMatcher.getMatchersList(), filesOnly, externalDataPrefix, evaluator);
+ includeExcludeMatcher.getMatchersList(), filesOnly, externalDataPrefix, evaluator,
+ warningCollector);
// Warn if no files are returned
if (filesOnly.isEmpty() && warningCollector.shouldWarn()) {
@@ -449,9 +450,11 @@
*/
private static void collectAndFilterBlobFiles(Iterable<BlobItem> items,
BiPredicate<List<Matcher>, String> predicate, List<Matcher> matchers, List<BlobItem> filesOnly,
- ExternalDataPrefix externalDataPrefix, IExternalFilterEvaluator evaluator) throws HyracksDataException {
+ ExternalDataPrefix externalDataPrefix, IExternalFilterEvaluator evaluator,
+ IWarningCollector warningCollector) throws HyracksDataException {
for (BlobItem item : items) {
- if (ExternalDataUtils.evaluate(item.getName(), predicate, matchers, externalDataPrefix, evaluator)) {
+ if (ExternalDataUtils.evaluate(item.getName(), predicate, matchers, externalDataPrefix, evaluator,
+ warningCollector)) {
filesOnly.add(item);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
index 0029e6a..e5ff733 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
@@ -174,7 +174,7 @@
// Collect the paths to files only
collectAndFilterFiles(items, includeExcludeMatcher.getPredicate(), includeExcludeMatcher.getMatchersList(),
- filesOnly, externalDataPrefix, evaluator);
+ filesOnly, externalDataPrefix, evaluator, warningCollector);
// Warn if no files are returned
if (filesOnly.isEmpty() && warningCollector.shouldWarn()) {
@@ -192,9 +192,10 @@
*/
private static void collectAndFilterFiles(Page<Blob> items, BiPredicate<List<Matcher>, String> predicate,
List<Matcher> matchers, List<Blob> filesOnly, ExternalDataPrefix externalDataPrefix,
- IExternalFilterEvaluator evaluator) throws HyracksDataException {
+ IExternalFilterEvaluator evaluator, IWarningCollector warningCollector) throws HyracksDataException {
for (Blob item : items.iterateAll()) {
- if (ExternalDataUtils.evaluate(item.getName(), predicate, matchers, externalDataPrefix, evaluator)) {
+ if (ExternalDataUtils.evaluate(item.getName(), predicate, matchers, externalDataPrefix, evaluator,
+ warningCollector)) {
filesOnly.add(item);
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
index 222ce81..5fa2af4 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
@@ -26,9 +26,9 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
-import org.apache.asterix.common.external.NoOpExternalFilterEvaluatorFactory;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.external.api.ITypedAdapterFactory;
+import org.apache.asterix.external.input.filter.NoOpExternalFilterEvaluatorFactory;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.om.types.ARecordType;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
index e9944ee..78421f2 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
@@ -32,13 +32,13 @@
import org.apache.asterix.common.exceptions.MetadataException;
import org.apache.asterix.common.external.IDataSourceAdapter;
import org.apache.asterix.common.external.IDataSourceAdapter.AdapterType;
-import org.apache.asterix.common.external.NoOpExternalFilterEvaluatorFactory;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.external.adapter.factory.ExternalAdapterFactory;
import org.apache.asterix.external.api.ITypedAdapterFactory;
import org.apache.asterix.external.feed.api.IFeed;
import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.input.filter.NoOpExternalFilterEvaluatorFactory;
import org.apache.asterix.external.provider.AdapterFactoryProvider;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
index b0d503c..87bc496 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
@@ -37,9 +37,9 @@
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
-import org.apache.asterix.common.external.NoOpExternalFilterEvaluatorFactory;
import org.apache.asterix.common.transactions.TxnId;
import org.apache.asterix.external.indexing.ExternalFile;
+import org.apache.asterix.external.input.filter.NoOpExternalFilterEvaluatorFactory;
import org.apache.asterix.external.util.ExternalDataPrefix;
import org.apache.asterix.metadata.dataset.DatasetFormatInfo;
import org.apache.asterix.metadata.declared.MetadataProvider;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/filter/ExternalFilterBuilder.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/filter/ExternalFilterBuilder.java
index 4165e32..4370cbf 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/filter/ExternalFilterBuilder.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/filter/ExternalFilterBuilder.java
@@ -18,12 +18,22 @@
*/
package org.apache.asterix.metadata.utils.filter;
+import static org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil.EMPTY_TYPE;
+import static org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil.getMergedPathRecordType;
+import static org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil.renameType;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
-import org.apache.asterix.common.external.NoOpExternalFilterEvaluatorFactory;
import org.apache.asterix.external.input.filter.ExternalFilterEvaluatorFactory;
import org.apache.asterix.external.input.filter.ExternalFilterValueEvaluatorFactory;
+import org.apache.asterix.external.input.filter.NoOpExternalFilterEvaluatorFactory;
import org.apache.asterix.external.util.ExternalDataPrefix;
import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil;
import org.apache.asterix.runtime.projection.ExternalDatasetProjectionFiltrationInfo;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -33,23 +43,38 @@
public class ExternalFilterBuilder extends AbstractFilterBuilder {
private final ExternalDataPrefix prefix;
+ private final boolean embedFilterValues;
public ExternalFilterBuilder(ExternalDatasetProjectionFiltrationInfo projectionFiltrationInfo,
JobGenContext context, IVariableTypeEnvironment typeEnv, ExternalDataPrefix prefix) {
super(projectionFiltrationInfo.getFilterPaths(), projectionFiltrationInfo.getFilterExpression(), context,
typeEnv);
this.prefix = prefix;
+ this.embedFilterValues = projectionFiltrationInfo.isEmbedFilterValues();
}
public IExternalFilterEvaluatorFactory build() throws AlgebricksException {
- if (filterExpression == null || filterPaths.isEmpty()) {
+ IScalarEvaluatorFactory evalFactory = null;
+ if (filterExpression != null && !filterPaths.isEmpty()) {
+ evalFactory = createEvaluator(filterExpression);
+ }
+
+ if (evalFactory == null && !embedFilterValues) {
return NoOpExternalFilterEvaluatorFactory.INSTANCE;
}
- IScalarEvaluatorFactory evalFactory = createEvaluator(filterExpression);
- if (evalFactory == null) {
- return NoOpExternalFilterEvaluatorFactory.INSTANCE;
+
+ List<String> fieldPaths = prefix.getComputedFieldNames();
+ int numberOfComputedFields = fieldPaths.size();
+ List<ProjectionFiltrationTypeUtil.RenamedType> leafs = new ArrayList<>(numberOfComputedFields);
+ ARecordType previousType = EMPTY_TYPE;
+ for (int i = 0; i < numberOfComputedFields; i++) {
+ ProjectionFiltrationTypeUtil.RenamedType renamedType = renameType(BuiltinType.ANY, i);
+ leafs.add(renamedType);
+ List<String> path = Arrays.asList(fieldPaths.get(i).split("\\."));
+ previousType = getMergedPathRecordType(previousType, path, renamedType);
}
- return new ExternalFilterEvaluatorFactory(prefix.getComputedFieldNames().size(), evalFactory);
+
+ return new ExternalFilterEvaluatorFactory(evalFactory, previousType, leafs, prefix, embedFilterValues);
}
@Override
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/utils/ProjectionFiltrationTypeUtil.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/utils/ProjectionFiltrationTypeUtil.java
index 41c990b..94ff865 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/utils/ProjectionFiltrationTypeUtil.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/utils/ProjectionFiltrationTypeUtil.java
@@ -20,13 +20,18 @@
import java.util.List;
+import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.om.typecomputer.impl.RecordMergeTypeComputer;
import org.apache.asterix.om.types.AOrderedListType;
import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.IATypeVisitor;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
public class ProjectionFiltrationTypeUtil {
//Default open record type when requesting the entire fields
public static final ARecordType ALL_FIELDS_TYPE = createType("");
@@ -69,6 +74,15 @@
return getRecordType(path, "root", 0, BuiltinType.ANY);
}
+ public static ARecordType getMergedPathRecordType(ARecordType previousType, List<String> path, IAType leafType)
+ throws AlgebricksException {
+ ARecordType type = getRecordType(path, "root", 0, leafType);
+ if (previousType == EMPTY_TYPE) {
+ return type;
+ }
+ return (ARecordType) RecordMergeTypeComputer.merge(previousType, type);
+ }
+
/**
* Get the expected type for an array index
*
@@ -102,6 +116,14 @@
return new ARecordType("root", result.getFieldNames(), result.getFieldTypes(), true);
}
+ public static IAType renameType(IAType type, String name) {
+ return new RenamedType(type, name);
+ }
+
+ public static RenamedType renameType(IAType type, int index) {
+ return new RenamedType(type, String.valueOf(index), index);
+ }
+
private static ARecordType getPathRecordType(List<String> path, IAType type) {
return getRecordType(path, "root", 0, type);
}
@@ -143,4 +165,78 @@
return new ARecordType(typeName, new String[] {}, new IAType[] {}, true);
}
+ public static class RenamedType implements IAType {
+ private static final long serialVersionUID = 992690669300951839L;
+ private final IAType originalType;
+ private final String name;
+ private final int index;
+
+ RenamedType(IAType originalType, String name) {
+ this(originalType, name, -1);
+ }
+
+ RenamedType(IAType originalType, String name, int index) {
+ this.originalType = originalType;
+ this.name = name;
+ this.index = index;
+ }
+
+ @Override
+ public IAType getType() {
+ return originalType.getType();
+ }
+
+ @Override
+ public boolean deepEqual(IAObject obj) {
+ return originalType.deepEqual(obj);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof RenamedType) {
+ return originalType.equals(((RenamedType) obj).originalType);
+ }
+ return originalType.equals(obj);
+ }
+
+ @Override
+ public int hash() {
+ return originalType.hash();
+ }
+
+ @Override
+ public ATypeTag getTypeTag() {
+ return originalType.getTypeTag();
+ }
+
+ @Override
+ public String getDisplayName() {
+ return originalType.getDisplayName();
+ }
+
+ @Override
+ public String getTypeName() {
+ return name;
+ }
+
+ @Override
+ public <R, T> R accept(IATypeVisitor<R, T> visitor, T arg) {
+ return visitor.visitFlat(this, arg);
+ }
+
+ @Override
+ public ObjectNode toJSON() {
+ return originalType.toJSON();
+ }
+
+ @Override
+ public String toString() {
+ return originalType.toString();
+ }
+
+ public int getIndex() {
+ return index;
+ }
+ }
+
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/ColumnDatasetProjectionFiltrationInfo.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/ColumnDatasetProjectionFiltrationInfo.java
index 4227c31..9e9011f 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/ColumnDatasetProjectionFiltrationInfo.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/ColumnDatasetProjectionFiltrationInfo.java
@@ -38,13 +38,13 @@
public ColumnDatasetProjectionFiltrationInfo(ARecordType recordRequestedType, ARecordType metaProjectedType,
Map<String, FunctionCallInformation> sourceInformationMap, Map<ILogicalExpression, ARecordType> filterPaths,
ILogicalExpression filterExpression, ILogicalExpression rangeFilterExpression) {
- super(recordRequestedType, sourceInformationMap, filterPaths, filterExpression);
+ super(recordRequestedType, sourceInformationMap, filterPaths, filterExpression, false);
this.metaProjectedType = metaProjectedType;
this.rangeFilterExpression = rangeFilterExpression;
}
private ColumnDatasetProjectionFiltrationInfo(ColumnDatasetProjectionFiltrationInfo other) {
- super(other.projectedType, other.functionCallInfoMap, other.filterPaths, other.filterExpression);
+ super(other.projectedType, other.functionCallInfoMap, other.filterPaths, other.filterExpression, false);
metaProjectedType = other.metaProjectedType;
rangeFilterExpression = other.rangeFilterExpression;
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/ExternalDatasetProjectionFiltrationInfo.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/ExternalDatasetProjectionFiltrationInfo.java
index a1d22ac..3f66e73 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/ExternalDatasetProjectionFiltrationInfo.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/projection/ExternalDatasetProjectionFiltrationInfo.java
@@ -43,14 +43,16 @@
protected final ILogicalExpression filterExpression;
protected final Map<ILogicalExpression, ARecordType> filterPaths;
protected final Map<String, FunctionCallInformation> functionCallInfoMap;
+ private final boolean embedFilterValues;
public ExternalDatasetProjectionFiltrationInfo(ARecordType projectedType,
Map<String, FunctionCallInformation> sourceInformationMap, Map<ILogicalExpression, ARecordType> filterPaths,
- ILogicalExpression filterExpression) {
+ ILogicalExpression filterExpression, boolean embedFilterValues) {
this.projectedType = projectedType;
this.functionCallInfoMap = sourceInformationMap;
this.filterExpression = filterExpression;
this.filterPaths = filterPaths;
+ this.embedFilterValues = embedFilterValues;
}
private ExternalDatasetProjectionFiltrationInfo(ExternalDatasetProjectionFiltrationInfo other) {
@@ -65,6 +67,7 @@
filterExpression = other.filterExpression;
filterPaths = new HashMap<>(other.filterPaths);
+ embedFilterValues = other.embedFilterValues;
}
@Override
@@ -88,6 +91,10 @@
return filterPaths;
}
+ public boolean isEmbedFilterValues() {
+ return embedFilterValues;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -117,6 +124,11 @@
writer.append(" prefix-filter on: ");
writer.append(filterExpression.toString());
}
+
+ if (embedFilterValues) {
+ writer.append(" embed-filter-value: ");
+ writer.append(String.valueOf(true));
+ }
}
@Override
@@ -133,6 +145,10 @@
if (filterExpression != null) {
generator.writeStringField("prefix-filter-on", filterExpression.toString());
}
+
+ if (embedFilterValues) {
+ generator.writeBooleanField("embed-filter-value", true);
+ }
}
protected String getOnelinerSchema(ARecordType type, StringBuilder builder) {