[ASTERIXDB-3323][COMP] Array index bulkload not indexing all values
- user model changes: no
- storage format changes: no
- interface changes: no
Change-Id: Ifc936ae867e90fff023f89ef2d21006e2e4f895d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17989
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.001.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.001.update.sqlpp
deleted file mode 100644
index 1822abe..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.001.update.sqlpp
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-USE test;
-UPSERT INTO ds1 {"id": 1, "a": ["hello", 1]};
-UPSERT INTO ds1 {"id": 2, "b": ["hello", "hello"]};
-UPSERT INTO ds1 {"id": 3, "b": [11, 12]};
-UPSERT INTO ds1 {"id": 4, "c": [10, 10]};
-
-UPSERT INTO ds1 {"id": 5, "d": [{"x":1}, {"x":"A"}]};
-UPSERT INTO ds1 {"id": 6, "d": [{"x":1}, {"x":"A"}]};
-
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.002.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.002.ddl.sqlpp
deleted file mode 100644
index fbe85d3..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.002.ddl.sqlpp
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-USE test;
-
-CREATE INDEX i2 ON ds1(UNNEST b:int) EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.003.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.003.ddl.sqlpp
deleted file mode 100644
index fbe85d3..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.003.ddl.sqlpp
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-USE test;
-
-CREATE INDEX i2 ON ds1(UNNEST b:int) EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.004.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.004.ddl.sqlpp
deleted file mode 100644
index 70119b5..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.004.ddl.sqlpp
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-USE test;
-
-CREATE INDEX i2 ON ds1(UNNEST d select x:string) EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.005.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.005.ddl.sqlpp
deleted file mode 100644
index ee3de29..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.005.ddl.sqlpp
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-USE test;
-
-CREATE INDEX i4 ON ds1(UNNEST c:int) EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.999.ddl.sqlpp
deleted file mode 100644
index 86a1b59..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.999.ddl.sqlpp
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-DROP DATAVERSE test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.008.ddl.sqlpp
similarity index 67%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.000.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.008.ddl.sqlpp
index efe6d6b..7850fde 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.008.ddl.sqlpp
@@ -21,8 +21,10 @@
* Expected Res : Said tuples are not indexed
*/
-CREATE DATAVERSE test;
USE test;
-CREATE TYPE dt1 as {id:int};
-CREATE DATASET ds1(dt1) primary key id;
+CREATE INDEX i6 ON ds1(UNNEST a : string) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i7 ON ds1(UNNEST b SELECT x : int) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i8 ON ds1(UNNEST b.x SELECT p: int) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i9 ON ds1(UNNEST b.c UNNEST d.e UNNEST t SELECT x.y : string, q.w: string, u: int) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i10 ON ds1(p: int, (UNNEST b.c UNNEST d.e UNNEST t SELECT x.y : string, q.w: string), z.m: double) EXCLUDE UNKNOWN KEY;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.009.query.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.009.query.sqlpp
index dcf23cf..a33bb23 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.009.query.sqlpp
@@ -20,6 +20,6 @@
USE test;
SET `import-private-functions` `true`;
-FROM DUMP_INDEX("test", "ds1", "i4") AS v
+FROM DUMP_INDEX("test", "ds1", "i6") AS v
SELECT VALUE v
ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.010.query.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.010.query.sqlpp
index dcf23cf..4134554 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.010.query.sqlpp
@@ -20,6 +20,6 @@
USE test;
SET `import-private-functions` `true`;
-FROM DUMP_INDEX("test", "ds1", "i4") AS v
+FROM DUMP_INDEX("test", "ds1", "i7") AS v
SELECT VALUE v
ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.011.query.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.011.query.sqlpp
index dcf23cf..75b56d1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.011.query.sqlpp
@@ -20,6 +20,6 @@
USE test;
SET `import-private-functions` `true`;
-FROM DUMP_INDEX("test", "ds1", "i4") AS v
+FROM DUMP_INDEX("test", "ds1", "i8") AS v
SELECT VALUE v
ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.012.query.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.012.query.sqlpp
index dcf23cf..5364ebb 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.012.query.sqlpp
@@ -20,6 +20,6 @@
USE test;
SET `import-private-functions` `true`;
-FROM DUMP_INDEX("test", "ds1", "i4") AS v
+FROM DUMP_INDEX("test", "ds1", "i9") AS v
SELECT VALUE v
ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.013.query.sqlpp
similarity index 94%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.013.query.sqlpp
index dcf23cf..44f59c8 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.013.query.sqlpp
@@ -20,6 +20,6 @@
USE test;
SET `import-private-functions` `true`;
-FROM DUMP_INDEX("test", "ds1", "i4") AS v
+FROM DUMP_INDEX("test", "ds1", "i10") AS v
SELECT VALUE v
ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.009.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.009.adm
new file mode 100644
index 0000000..a464380
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.009.adm
@@ -0,0 +1,10 @@
+{ "values": [ "hello", 1 ] }
+{ "values": [ "hello", 5 ] }
+{ "values": [ "hello", 6 ] }
+{ "values": [ "hello", 8 ] }
+{ "values": [ "hello", 10 ] }
+{ "values": [ "hello", 11 ] }
+{ "values": [ "hello", 12 ] }
+{ "values": [ "hello", 13 ] }
+{ "values": [ "hello", 14 ] }
+{ "values": [ "hello", 15 ] }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.010.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.010.adm
new file mode 100644
index 0000000..3fdbd81
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.010.adm
@@ -0,0 +1,2 @@
+{ "values": [ 1, 5 ] }
+{ "values": [ 10, 7 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.011.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.011.adm
new file mode 100644
index 0000000..6045d06
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.011.adm
@@ -0,0 +1 @@
+{ "values": [ 1, 10 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.012.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.012.adm
new file mode 100644
index 0000000..4c44dba
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.012.adm
@@ -0,0 +1,2 @@
+{ "values": [ "aab", "10", 90, 11 ] }
+{ "values": [ "aab", "10", 93, 14 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.013.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.013.adm
new file mode 100644
index 0000000..a38bbc6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.013.adm
@@ -0,0 +1 @@
+{ "values": [ 100, "aab", "10", 100.1, 14 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 9a3abae..72578fa 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -8356,8 +8356,6 @@
<test-case FilePath="array-index/bulk-loading/on-index-creation/open">
<compilation-unit name="composite-atomic">
<output-dir compare="Text">composite-atomic</output-dir>
- <expected-error>ASX0001: Field type string cannot be promoted to type bigint</expected-error>
- <source-location>false</source-location>
</compilation-unit>
</test-case>
</test-group>
@@ -8662,17 +8660,6 @@
</compilation-unit>
</test-case>
</test-group>
- <test-group name="array-index/index-bad-fields-bulkload">
- <test-case FilePath="array-index">
- <compilation-unit name="index-bad-fields-bulkload">
- <output-dir compare="Text">index-bad-fields-bulkload</output-dir>
- <expected-error>ASX0018: Cannot convert the string type to the bigint type</expected-error>
- <expected-error>ASX0018: Cannot convert the string type to the bigint type</expected-error>
- <expected-error>ASX0001: Field type bigint cannot be promoted to type string</expected-error>
- <source-location>false</source-location>
- </compilation-unit>
- </test-case>
- </test-group>
</test-group>
<test-group name="nestrecords">
<test-case FilePath="nestrecords">
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
index 7332dd0..498630a 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
@@ -35,6 +35,7 @@
import org.apache.asterix.om.types.AbstractCollectionType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.asterix.om.utils.RecordUtil;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -254,7 +255,8 @@
IAType workingType = baseRecordType;
for (int i = 0; i < fieldNamesPerArray.size(); i++) {
- ARecordType startingStepRecordType = (isTrackingType) ? (ARecordType) workingType : null;
+ ARecordType startingStepRecordType =
+ (isTrackingType) ? (ARecordType) workingType : RecordUtil.FULLY_OPEN_RECORD_TYPE;
if (isTrackingType) {
if (!workingType.getTypeTag().equals(ATypeTag.OBJECT)) {
throw new AsterixException(ErrorCode.COMPILATION_ERROR, "Mismatched record type to depth-"
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
index af0fd82..de0ca5a 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
@@ -158,7 +158,7 @@
int flattenedListPos = 0;
for (Index.ArrayIndexElement e : arrayIndexDetails.getElementList()) {
for (int i = 0; i < e.getProjectList().size(); i++) {
- addSKEvalFactories(isOverridingKeyFieldTypes ? enforcedItemType : itemType, flattenedListPos, false, e);
+ addSKEvalFactories(itemType, flattenedListPos, false, e);
Pair<IAType, Boolean> keyTypePair = ArrayIndexUtil.getNonNullableOpenFieldType(e.getTypeList().get(i),
e.getUnnestList(), e.getProjectList().get(i), itemType);
IAType keyType = keyTypePair.first;
@@ -268,12 +268,6 @@
spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
sourceOp = targetOp;
- if (arrayIndexDetails.isOverridingKeyFieldTypes() && !enforcedItemType.equals(itemType)) {
- // If we have an enforced type, insert a "cast" after the primary index scan.
- targetOp = createCastOp(spec, dataset.getDatasetType(), true);
- spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
- sourceOp = targetOp;
- }
// We do not index meta fields. Project away meta fields if they exist.
if (dataset.hasMetaPart()) {
@@ -306,7 +300,15 @@
if (anySecondaryKeyIsNullable || arrayIndexDetails.isOverridingKeyFieldTypes()) {
// If any of the secondary fields are nullable, then we need to filter out the nulls.
- targetOp = createFilterAnyUnknownSelectOp(spec, numTotalSecondaryKeys, secondaryRecDesc);
+ List<IAType> secondaryKeyTypes = new ArrayList<>();
+ if (arrayIndexDetails.isOverridingKeyFieldTypes() && !enforcedItemType.equals(itemType)) {
+ for (Index.ArrayIndexElement arrayIndexElement : arrayIndexDetails.getElementList()) {
+ List<IAType> typeList = arrayIndexElement.getTypeList();
+ secondaryKeyTypes.addAll(typeList);
+ }
+ }
+ targetOp = createCastFilterAnyUnknownSelectOp(spec, numTotalSecondaryKeys, secondaryRecDesc,
+ secondaryKeyTypes);
spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
sourceOp = targetOp;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index 694b153..ab64b18 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -478,9 +478,22 @@
return createFilterSelectOp(spec, numSecondaryKeyFields, secondaryRecDesc, AndDescriptor::new);
}
+ public AlgebricksMetaOperatorDescriptor createCastFilterAnyUnknownSelectOp(JobSpecification spec,
+ int numSecondaryKeyFields, RecordDescriptor secondaryRecDesc, List<IAType> castFieldTypes)
+ throws AlgebricksException {
+ return createFilterSelectOp(spec, numSecondaryKeyFields, secondaryRecDesc, AndDescriptor::new, castFieldTypes);
+ }
+
private AlgebricksMetaOperatorDescriptor createFilterSelectOp(JobSpecification spec, int numSecondaryKeyFields,
RecordDescriptor secondaryRecDesc, Supplier<AbstractFunctionDescriptor> predicatesCombinerFuncSupplier)
throws AlgebricksException {
+ return createFilterSelectOp(spec, numSecondaryKeyFields, secondaryRecDesc, predicatesCombinerFuncSupplier,
+ Collections.emptyList());
+ }
+
+ private AlgebricksMetaOperatorDescriptor createFilterSelectOp(JobSpecification spec, int numSecondaryKeyFields,
+ RecordDescriptor secondaryRecDesc, Supplier<AbstractFunctionDescriptor> predicatesCombinerFuncSupplier,
+ List<IAType> castFieldTypes) throws AlgebricksException {
IScalarEvaluatorFactory[] predicateArgsEvalFactories = new IScalarEvaluatorFactory[numSecondaryKeyFields];
NotDescriptor notDesc = new NotDescriptor();
notDesc.setSourceLocation(sourceLoc);
@@ -489,8 +502,14 @@
for (int i = 0; i < numSecondaryKeyFields; i++) {
// Access column i, and apply 'is not null'.
ColumnAccessEvalFactory columnAccessEvalFactory = new ColumnAccessEvalFactory(i);
+ IScalarEvaluatorFactory evalFactory = columnAccessEvalFactory;
+ if (castFieldTypes != null && !castFieldTypes.isEmpty()) {
+ IScalarEvaluatorFactory[] castArg = new IScalarEvaluatorFactory[] { columnAccessEvalFactory };
+ evalFactory = createCastFunction(castFieldTypes.get(i), BuiltinType.ANY, index.isEnforced(), sourceLoc)
+ .createEvaluatorFactory(castArg);
+ }
IScalarEvaluatorFactory isUnknownEvalFactory =
- isUnknownDesc.createEvaluatorFactory(new IScalarEvaluatorFactory[] { columnAccessEvalFactory });
+ isUnknownDesc.createEvaluatorFactory(new IScalarEvaluatorFactory[] { evalFactory });
IScalarEvaluatorFactory notEvalFactory =
notDesc.createEvaluatorFactory(new IScalarEvaluatorFactory[] { isUnknownEvalFactory });
predicateArgsEvalFactories[i] = notEvalFactory;