[ASTERIXDB-3299][COMP] Out of memory during query compilation
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
In case of deeply nested object or list is used in an upsert
statement or query, the compilation either fails with out of
memory error or takes a long time to complete. The issue is
with the recursive calls to getSerializerDeserializer in
ARecordSerializerDeserializer, AOrderedListSerializerDeserializer
and AUnorderdListSerializerDeserializer.
Change-Id: Iea45c4f77b30b6ca4de7f8355be3a135c6c87460
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17925
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Peeyush Gupta <peeyush.gupta@couchbase.com>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Tested-by: Peeyush Gupta <peeyush.gupta@couchbase.com>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.1.ddl.sqlpp
new file mode 100644
index 0000000..ebbe733
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.1.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create dataset ds primary key (_id:string);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.2.update.sqlpp
new file mode 100644
index 0000000..b4a58a2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.2.update.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+upsert into ds {"_id": "123", "data": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[{"xaasdasdad": 1}]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]};
+
+upsert into ds {"_id": "234", "level_24": {"level_23": {"level_22": {"level_21": {"level_20": {"level_19": {"level_18": {"level_17": {"level_16": {"level_15": {"level_14": {"level_13": {"level_12": {"level_11": {"level_10": {"level_9": {"level_8": {"level_7": {"level_6": {"level_5": {"level_4": {"level_3": {"level_2": {"level_1": {"level_0": {"leaf": [1, 2]}}}}}}}}}}}}}}}}}}}}}}}}}};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.3.query.sqlpp
new file mode 100644
index 0000000..fc3925c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.3.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select value s from [{"_id": "2431dfegh2345", "data": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[{"xaasdasdad": 1}]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]}] s;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.4.query.sqlpp
new file mode 100644
index 0000000..2410b17
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.4.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select value s from [{"_id": "2431dfegh2345", "level_24": {"level_23": {"level_22": {"level_21": {"level_20": {"level_19": {"level_18": {"level_17": {"level_16": {"level_15": {"level_14": {"level_13": {"level_12": {"level_11": {"level_10": {"level_9": {"level_8": {"level_7": {"level_6": {"level_5": {"level_4": {"level_3": {"level_2": {"level_1": {"level_0": {"leaf": [1, 2]}}}}}}}}}}}}}}}}}}}}}}}}}}] s;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.5.query.sqlpp
new file mode 100644
index 0000000..51985c0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.5.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select value ds from ds order by _id desc
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.6.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.6.ddl.sqlpp
new file mode 100644
index 0000000..dc10acd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.6.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.3.adm
new file mode 100644
index 0000000..c452c1f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.3.adm
@@ -0,0 +1 @@
+{ "_id": "2431dfegh2345", "data": [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ { "xaasdasdad": 1 } ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.4.adm
new file mode 100644
index 0000000..7693bb2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.4.adm
@@ -0,0 +1 @@
+{ "_id": "2431dfegh2345", "level_24": { "level_23": { "level_22": { "level_21": { "level_20": { "level_19": { "level_18": { "level_17": { "level_16": { "level_15": { "level_14": { "level_13": { "level_12": { "level_11": { "level_10": { "level_9": { "level_8": { "level_7": { "level_6": { "level_5": { "level_4": { "level_3": { "level_2": { "level_1": { "level_0": { "leaf": [ 1, 2 ] } } } } } } } } } } } } } } } } } } } } } } } } } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.5.adm
new file mode 100644
index 0000000..30f2a76
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3299/query-ASTERIXDB-3299.5.adm
@@ -0,0 +1,2 @@
+{ "_id": "234", "level_24": { "level_23": { "level_22": { "level_21": { "level_20": { "level_19": { "level_18": { "level_17": { "level_16": { "level_15": { "level_14": { "level_13": { "level_12": { "level_11": { "level_10": { "level_9": { "level_8": { "level_7": { "level_6": { "level_5": { "level_4": { "level_3": { "level_2": { "level_1": { "level_0": { "leaf": [ 1, 2 ] } } } } } } } } } } } } } } } } } } } } } } } } } }
+{ "_id": "123", "data": [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ { "xaasdasdad": 1 } ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
index b82481d..66286b0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
@@ -7350,6 +7350,11 @@
<output-dir compare="Text">serialized_size_fun</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-3299">
+ <output-dir compare="Text">query-ASTERIXDB-3299</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="multipart-dataverse">
<test-case FilePath="multipart-dataverse">
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AOrderedListSerializerDeserializer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AOrderedListSerializerDeserializer.java
index e443e13..5dffd60 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AOrderedListSerializerDeserializer.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AOrderedListSerializerDeserializer.java
@@ -59,10 +59,16 @@
public AOrderedListSerializerDeserializer(AOrderedListType orderedlistType) {
this.orderedlistType = orderedlistType;
this.itemType = orderedlistType.getItemType();
- serializer = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
- deserializer = itemType.getTypeTag() == ATypeTag.ANY
- ? SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType)
- : SerializerDeserializerProvider.INSTANCE.getNonTaggedSerializerDeserializer(itemType);
+ if (itemType.getTypeTag() == ATypeTag.ANY) {
+ serializer = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
+ deserializer = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
+ } else {
+ ISerializerDeserializer nonTaggedSerializerDeserializer =
+ SerializerDeserializerProvider.INSTANCE.getNonTaggedSerializerDeserializer(itemType);
+ serializer = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType,
+ nonTaggedSerializerDeserializer);
+ deserializer = nonTaggedSerializerDeserializer;
+ }
}
@Override
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java
index 3f8102e..45b9f7d 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java
@@ -73,8 +73,11 @@
for (int i = 0; i < numberOfSchemaFields; i++) {
IAType t = recordType.getFieldTypes()[i];
IAType t2 = (t.getTypeTag() == ATypeTag.UNION) ? ((AUnionType) t).getActualType() : t;
- serializers[i] = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(t2);
- deserializers[i] = SerializerDeserializerProvider.INSTANCE.getNonTaggedSerializerDeserializer(t2);
+ ISerializerDeserializer nonTaggedSerelaizerDeserializer =
+ SerializerDeserializerProvider.INSTANCE.getNonTaggedSerializerDeserializer(t2);
+ serializers[i] = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(t2,
+ nonTaggedSerelaizerDeserializer);
+ deserializers[i] = nonTaggedSerelaizerDeserializer;
}
} else {
this.recordType = null;
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AUnorderedListSerializerDeserializer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AUnorderedListSerializerDeserializer.java
index b5165d2..31c1424 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AUnorderedListSerializerDeserializer.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AUnorderedListSerializerDeserializer.java
@@ -60,10 +60,16 @@
public AUnorderedListSerializerDeserializer(AUnorderedListType unorderedlistType) {
this.unorderedlistType = unorderedlistType;
this.itemType = unorderedlistType.getItemType();
- serializer = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
- deserializer = itemType.getTypeTag() == ATypeTag.ANY
- ? SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType)
- : SerializerDeserializerProvider.INSTANCE.getNonTaggedSerializerDeserializer(itemType);
+ if (itemType.getTypeTag() == ATypeTag.ANY) {
+ serializer = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
+ deserializer = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
+ } else {
+ ISerializerDeserializer nonTaggedSerializerDeserializer =
+ SerializerDeserializerProvider.INSTANCE.getNonTaggedSerializerDeserializer(itemType);
+ serializer = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType,
+ nonTaggedSerializerDeserializer);
+ deserializer = nonTaggedSerializerDeserializer;
+ }
}
@Override
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/SerializerDeserializerProvider.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/SerializerDeserializerProvider.java
index 521e528..f86f067 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/SerializerDeserializerProvider.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/SerializerDeserializerProvider.java
@@ -102,6 +102,24 @@
}
@SuppressWarnings("rawtypes")
+ @Override
+ public ISerializerDeserializer getSerializerDeserializer(Object typeInfo,
+ ISerializerDeserializer nonTaggedSerializerDeserializer) {
+ IAType type = (IAType) typeInfo;
+ if (type == null) {
+ return null;
+ }
+ switch (type.getTypeTag()) {
+ case ANY:
+ case UNION:
+ // we could do smth better for nullable fields
+ return AObjectSerializerDeserializer.INSTANCE;
+ default:
+ return addTag(nonTaggedSerializerDeserializer);
+ }
+ }
+
+ @SuppressWarnings("rawtypes")
public ISerializerDeserializer getNonTaggedSerializerDeserializer(IAType type) {
switch (type.getTypeTag()) {
case CIRCLE:
diff --git a/hyracks-fullstack/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/ISerializerDeserializerProvider.java b/hyracks-fullstack/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/ISerializerDeserializerProvider.java
index c54cdb2..9450ffd 100644
--- a/hyracks-fullstack/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/ISerializerDeserializerProvider.java
+++ b/hyracks-fullstack/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/ISerializerDeserializerProvider.java
@@ -23,5 +23,10 @@
public interface ISerializerDeserializerProvider {
@SuppressWarnings("unchecked")
- public ISerializerDeserializer getSerializerDeserializer(Object type) throws AlgebricksException;
+ ISerializerDeserializer getSerializerDeserializer(Object type) throws AlgebricksException;
+
+ @SuppressWarnings("unchecked")
+ ISerializerDeserializer getSerializerDeserializer(Object type,
+ ISerializerDeserializer nonTaggedSerializerDeserializer) throws AlgebricksException;
+
}