[ASTERIXDB-2959][IDX] Fix initial composite atomic-array index ingestion on creation
- user model changes: no
- storage format changes: no
- interface changes: no
Composite atomic-array indexes were not ingesting properly, due to the
wrong source column being UNNESTed in the loading job. This small change
addresses this.
Change-Id: I86af26a7649e0631a0555e1f98fb5fea8e63528e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13163
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dmitry.lychagin@couchbase.com>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.1.ddl.sqlpp
new file mode 100644
index 0000000..6312915
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.1.ddl.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE TestDataverse IF EXISTS;
+CREATE DATAVERSE TestDataverse;
+USE TestDataverse;
+
+CREATE TYPE GenericType AS { _id: bigint };
+CREATE DATASET TestDataset (GenericType)
+PRIMARY KEY _id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.2.update.sqlpp
new file mode 100644
index 0000000..e901b37
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.2.update.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+USE TestDataverse;
+
+INSERT INTO TestDataset [
+ { "_id": 100, "f": 1, "a": {"b": [{"c": {"d": 1}}]} },
+ { "_id": 101, "f": "a", "a": null },
+ { "_id": 102, "f": "a", "a": {"b": null} },
+ { "_id": 103, "f": "a", "a": {"b": [{"c": null}]} },
+ { "_id": 104, "f": "a", "a": {"b": [{"c": {"d": null}}]} }
+];
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.3.ddl.sqlpp
new file mode 100644
index 0000000..8bcf2c5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.3.ddl.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+USE TestDataverse;
+
+CREATE INDEX TestIdx
+ON TestDataset (
+ f : bigint,
+ UNNEST a.b
+ SELECT c.d : bigint
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.4.query.sqlpp
new file mode 100644
index 0000000..a5de0da
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.4.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+SET `compiler.arrayindex` "true";
+USE TestDataverse;
+
+FROM TestDataset D
+UNNEST D.a.b AB
+WHERE D.f = 1 AND AB.c.d = 1
+SELECT D._id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.1.adm
new file mode 100644
index 0000000..32c3a65
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.1.adm
@@ -0,0 +1 @@
+{ "_id": 100 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 6879938..4220f99 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -8217,8 +8217,13 @@
</compilation-unit>
</test-case>
<test-case FilePath="array-index/bulk-loading/on-index-creation/closed">
- <compilation-unit name="with-3-level-record-path">
- <output-dir compare="Text">with-3-level-record-path</output-dir>
+ <compilation-unit name="with-3-level-record-path">
+ <output-dir compare="Text">with-3-level-record-path</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/open">
+ <compilation-unit name="composite-atomic">
+ <output-dir compare="Text">composite-atomic</output-dir>
</compilation-unit>
</test-case>
</test-group>
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
index ab181cb..7282c7d 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
@@ -245,6 +245,7 @@
ArrayPath arrayPath = new ArrayPath(flattenedFieldName, unnestFlags).invoke();
List<List<String>> fieldNamesPerArray = arrayPath.fieldNamesPerArray;
List<Boolean> unnestFlagsPerArray = arrayPath.unnestFlagsPerArray;
+ boolean requiresOnlyOneUnnest = unnestFlags.stream().filter(f -> f).count() == 1;
// If we are given no base record type, then we do not need to keep track of the record type. We are solely
// using this walk for its flags.
@@ -290,7 +291,6 @@
}
if (i == fieldNamesPerArray.size() - 1) {
- boolean requiresOnlyOneUnnest = fieldNamesPerArray.size() == 1;
boolean isNonArrayStep = !unnestFlagsPerArray.get(i);
commandExecutor.executeActionOnFinalArrayStep(startingStepRecordType, fieldNamesPerArray.get(i),
isNonArrayStep, requiresOnlyOneUnnest);