[ASTERIXDB-3166][FUN] Error with get_object_fields with open list type
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
get_object_fields function causes null pointer exception when used
on a dataset containing an open list type.
Change-Id: Ie413d286738f571df81e2f450a3fb0b132dc5521
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17488
Reviewed-by: Peeyush Gupta <peeyush.gupta@couchbase.com>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
index 77fc7c5..1756378 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
@@ -82,6 +82,11 @@
<output-dir compare="Text">missing-fields</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="objects/get-object-fields">
+ <compilation-unit name="open-list-type">
+ <output-dir compare="Text">open-list-type</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="objects/get-object-field-value">
<compilation-unit name="documentation-example">
<output-dir compare="Text">documentation-example</output-dir>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.1.ddl.sqlpp
new file mode 100644
index 0000000..402cc11
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.1.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing get-object-fields where some schema (complex) fields are missing or null
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE t1 AS {
+ id: string
+};
+
+
+CREATE DATASET ds1(t1) PRIMARY KEY id;
+CREATE DATASET ds2(t1) PRIMARY KEY id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.2.update.sqlpp
new file mode 100644
index 0000000..d5e98b4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.2.update.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+INSERT INTO ds1([
+{"id": "xyz",
+"some_record": {"x1": 9, "nested_list": ["y1", "y2"], "nested_rec": {"f1": 7}},
+"some_list2": [[1,2],[3,4]],
+"some_list3": [{"a": 5}, {"b": 6}]
+}]);
+
+INSERT INTO ds2([
+{"id": "xyz",
+"some_record": {"x1": 9, "nested_list": {{"y1", "y2"}}, "nested_rec": {"f1": 7}},
+"some_list2": {{[1,2],{{3,4}}}},
+"some_list3": {{{"a": 5}, {"b": 6}}}
+}]);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.3.query.sqlpp
new file mode 100644
index 0000000..d45f083
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.3.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT VALUE get_object_fields(d) FROM ds1 d;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.4.query.sqlpp
new file mode 100644
index 0000000..5556e29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.4.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT VALUE get_object_fields(d) FROM ds2 d;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.3.adm
new file mode 100644
index 0000000..e506c07
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.3.adm
@@ -0,0 +1 @@
+[ { "field-name": "id", "field-type": "string", "is-open": false }, { "field-name": "some_record", "field-type": "object", "is-open": true, "nested": [ { "field-name": "x1", "field-type": "bigint", "is-open": true }, { "field-name": "nested_rec", "field-type": "object", "is-open": true, "nested": [ { "field-name": "f1", "field-type": "bigint", "is-open": true } ] }, { "field-name": "nested_list", "field-type": "array", "is-open": true, "list": [ { "field-type": "string" }, { "field-type": "string" } ] } ] }, { "field-name": "some_list3", "field-type": "array", "is-open": true, "list": [ { "field-type": "object", "nested": [ { "field-name": "a", "field-type": "bigint", "is-open": true } ] }, { "field-type": "object", "nested": [ { "field-name": "b", "field-type": "bigint", "is-open": true } ] } ] }, { "field-name": "some_list2", "field-type": "array", "is-open": true, "list": [ { "field-type": "array", "list": [ { "field-type": "bigint" }, { "field-type": "bigint" } ] }, { "field-type": "array", "list": [ { "field-type": "bigint" }, { "field-type": "bigint" } ] } ] } ]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.4.adm
new file mode 100644
index 0000000..b16e195
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/open-list-type/open-list-type.4.adm
@@ -0,0 +1 @@
+[ { "field-name": "id", "field-type": "string", "is-open": false }, { "field-name": "some_record", "field-type": "object", "is-open": true, "nested": [ { "field-name": "x1", "field-type": "bigint", "is-open": true }, { "field-name": "nested_rec", "field-type": "object", "is-open": true, "nested": [ { "field-name": "f1", "field-type": "bigint", "is-open": true } ] }, { "field-name": "nested_list", "field-type": "multiset", "is-open": true, "list": [ { "field-type": "string" }, { "field-type": "string" } ] } ] }, { "field-name": "some_list3", "field-type": "multiset", "is-open": true, "list": [ { "field-type": "object", "nested": [ { "field-name": "a", "field-type": "bigint", "is-open": true } ] }, { "field-type": "object", "nested": [ { "field-name": "b", "field-type": "bigint", "is-open": true } ] } ] }, { "field-name": "some_list2", "field-type": "multiset", "is-open": true, "list": [ { "field-type": "array", "list": [ { "field-type": "bigint" }, { "field-type": "bigint" } ] }, { "field-type": "multiset", "list": [ { "field-type": "bigint" }, { "field-type": "bigint" } ] } ] } ]
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/AListPointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/AListPointable.java
index 1e3ba12..20af4dc 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/AListPointable.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/AListPointable.java
@@ -190,7 +190,7 @@
if (isFixedType(inputType)) {
return getFixedLength(inputType);
} else {
- int nextOffset = (getItemCount() > index + 1) ? getItemOffset(inputType, index + 1) : getLength();
+ int nextOffset = (getItemCount() > index + 1) ? getItemOffset(inputType, index + 1) : getLength() - 1;
return nextOffset - getItemOffset(inputType, index);
}
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java
index 25dd73e..d7723f0 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java
@@ -37,6 +37,7 @@
import org.apache.asterix.om.types.AOrderedListType;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnorderedListType;
import org.apache.asterix.om.types.AbstractCollectionType;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.EnumDeserializer;
@@ -78,6 +79,9 @@
SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABOOLEAN);
private final static ARecordType openType = DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
+ private final static AOrderedListType openOrderedListType = DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE;
+ private final static AUnorderedListType openUnOrderedListType =
+ DefaultOpenFieldType.NESTED_OPEN_AUNORDERED_LIST_TYPE;
public void processRecord(ARecordPointable recordAccessor, ARecordType recType, DataOutput out, int level)
throws IOException {
@@ -91,7 +95,7 @@
OrderedListBuilder orderedListBuilder = getOrderedListBuilder();
orderedListBuilder.reset(listType);
IARecordBuilder fieldRecordBuilder = getRecordBuilder();
- fieldRecordBuilder.reset(null);
+ fieldRecordBuilder.reset(openType);
int schemeFieldCount = recordAccessor.getSchemeFieldCount(recType);
for (int i = 0; i < schemeFieldCount; ++i) {
@@ -119,7 +123,7 @@
tmpValue.reset();
recordAccessor.getClosedFieldValue(recType, i, tmpValue.getDataOutput());
if (tag == ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
- addNestedField(tmpValue, fieldType, fieldRecordBuilder, level + 1);
+ addRecordField(tmpValue, fieldType, fieldRecordBuilder, level + 1);
} else {
addListField(tmpValue, fieldType, fieldRecordBuilder, level + 1);
}
@@ -151,14 +155,15 @@
// write nested or list types
if (tag == ATypeTag.SERIALIZED_RECORD_TYPE_TAG || tag == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
|| tag == ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
- IAType fieldType = null;
ArrayBackedValueStorage tmpValue = getTempBuffer();
tmpValue.reset();
recordAccessor.getOpenFieldValue(recType, i, tmpValue.getDataOutput());
if (tag == ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
- addNestedField(tmpValue, fieldType, fieldRecordBuilder, level + 1);
+ addRecordField(tmpValue, openType, fieldRecordBuilder, level + 1);
+ } else if (tag == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG) {
+ addListField(tmpValue, openOrderedListType, fieldRecordBuilder, level + 1);
} else {
- addListField(tmpValue, fieldType, fieldRecordBuilder, level + 1);
+ addListField(tmpValue, openUnOrderedListType, fieldRecordBuilder, level + 1);
}
}
@@ -226,7 +231,7 @@
fieldRecordBuilder.addField(fieldAbvs, valueAbvs);
}
- public void addNestedField(IValueReference recordArg, IAType fieldType, IARecordBuilder fieldRecordBuilder,
+ public void addRecordField(IValueReference recordArg, IAType fieldType, IARecordBuilder fieldRecordBuilder,
int level) throws IOException {
ArrayBackedValueStorage fieldAbvs = getTempBuffer();
ArrayBackedValueStorage valueAbvs = getTempBuffer();
@@ -236,31 +241,26 @@
stringSerde.serialize(nestedName, fieldAbvs.getDataOutput());
// Value
valueAbvs.reset();
- ARecordType newType;
- if (fieldType == null) {
- newType = openType;
- } else {
- newType = (ARecordType) fieldType;
- }
+ ARecordType newType = (ARecordType) fieldType;
ARecordPointable recordP = getRecordPointable();
recordP.set(recordArg);
processRecord(recordP, newType, valueAbvs.getDataOutput(), level);
fieldRecordBuilder.addField(fieldAbvs, valueAbvs);
}
- public void processListValue(IValueReference listArg, IAType fieldType, DataOutput out, int level)
+ public void processListValue(IValueReference listArg, IAType listType, DataOutput out, int level)
throws IOException {
ArrayBackedValueStorage itemValue = getTempBuffer();
IARecordBuilder listRecordBuilder = getRecordBuilder();
- AListPointable list = getListPointable(fieldType.getTypeTag());
+ AListPointable list = getListPointable(listType.getTypeTag());
list.set(listArg);
OrderedListBuilder innerListBuilder = getOrderedListBuilder();
- innerListBuilder.reset(listType);
+ innerListBuilder.reset(RecordFieldsUtil.listType);
- listRecordBuilder.reset(null);
- AbstractCollectionType act = (AbstractCollectionType) fieldType;
+ listRecordBuilder.reset(openType);
+ AbstractCollectionType act = (AbstractCollectionType) listType;
int itemCount = list.getItemCount();
for (int l = 0; l < itemCount; l++) {
itemValue.reset();
@@ -269,11 +269,24 @@
byte tagId = list.getItemTag(act, l);
addFieldType(tagId, listRecordBuilder);
- if (tagId == ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
+ if (tagId == ATypeTag.SERIALIZED_RECORD_TYPE_TAG || tagId == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
+ || tagId == ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
ArrayBackedValueStorage tmpAbvs = getTempBuffer();
tmpAbvs.reset();
list.getItemValue(act, l, tmpAbvs.getDataOutput());
- addNestedField(tmpAbvs, act.getItemType(), listRecordBuilder, level + 1);
+ IAType itemType;
+
+ if (tagId == ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
+ itemType = act.getItemType().getTypeTag() != ATypeTag.ANY ? act.getItemType() : openType;
+ addRecordField(tmpAbvs, itemType, listRecordBuilder, level + 1);
+ } else if (tagId == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG) {
+ itemType = act.getItemType().getTypeTag() != ATypeTag.ANY ? act.getItemType() : openOrderedListType;
+ addListField(tmpAbvs, itemType, listRecordBuilder, level + 1);
+ } else {
+ itemType =
+ act.getItemType().getTypeTag() != ATypeTag.ANY ? act.getItemType() : openUnOrderedListType;
+ addListField(tmpAbvs, itemType, listRecordBuilder, level + 1);
+ }
}
listRecordBuilder.write(itemValue.getDataOutput(), true);