[ASTERIXDB-3276][RT] Fix infinite loop in columnar filter
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Columnar filter can stuck in infinite loop when a
non-existing value is part of the filter expression.
Change-Id: Ib32c9a1fe2814da4091c96e504d694b67d0d0d56
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17849
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/filter/003/003.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/filter/003/003.006.query.sqlpp
new file mode 100644
index 0000000..8d6fa80
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/filter/003/003.006.query.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+SET `compiler.column.filter` "true";
+
+SELECT VALUE d
+FROM ColumnDataset d
+-- notAField would yield a missing; however (in this case) the missing value is part of the add operation.
+-- Thus, the value should evaluate to missing.
+-- Previously, this would stuck in infinite loop (See ASTERIXDB-3276).
+WHERE d.notAField + 1 > 4
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/filter/003/003.007.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/filter/003/003.007.query.sqlpp
new file mode 100644
index 0000000..a6af494
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/filter/003/003.007.query.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+SET `compiler.column.filter` "true";
+
+SELECT VALUE item
+FROM ColumnDataset d, d.array item
+-- notAField would yield a missing; however (in this case) the missing value is part of the add operation.
+-- Thus, the value should evaluate to missing.
+-- Previously, this would stuck in infinite loop (See ASTERIXDB-3276).
+WHERE item.notAField + 1 > 4
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/filter/003/003.006.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/filter/003/003.006.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/filter/003/003.006.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/filter/003/003.007.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/filter/003/003.007.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/filter/003/003.007.adm
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnIterableFilterEvaluatorFactory.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnIterableFilterEvaluatorFactory.java
index 1264c24..742511b 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnIterableFilterEvaluatorFactory.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnIterableFilterEvaluatorFactory.java
@@ -40,8 +40,8 @@
@Override
public IColumnIterableFilterEvaluator create(FilterAccessorProvider filterAccessorProvider,
IEvaluatorContext context) throws HyracksDataException {
- IScalarEvaluator evaluator = evaluatorFactory.createScalarEvaluator(context);
List<IColumnValuesReader> readers = filterAccessorProvider.getFilterColumnReaders();
+ IScalarEvaluator evaluator = evaluatorFactory.createScalarEvaluator(context);
if (readers.stream().anyMatch(IColumnValuesReader::isRepeated)) {
return new ColumnarRepeatedIterableFilterEvaluator(evaluator, readers);
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java
index de4df8a..efff918 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java
@@ -58,9 +58,7 @@
delimiters.clear();
AbstractSchemaNode node = path.accept(this, null);
ATypeTag typeTag = node.getTypeTag();
- if (typeTag == ATypeTag.MISSING) {
- return Collections.emptyList();
- } else if (typeTag == ATypeTag.UNION) {
+ if (typeTag == ATypeTag.UNION) {
UnionSchemaNode unionNode = (UnionSchemaNode) node;
Collection<AbstractSchemaNode> children = unionNode.getChildren().values();
List<IColumnValuesReader> unionReaders = new ArrayList<>();
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java
index 71497f5..ba5ffb7 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java
@@ -24,6 +24,7 @@
import org.apache.asterix.column.assembler.value.MissingValueGetter;
import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
import org.apache.asterix.column.bytes.stream.in.ByteBufferInputStream;
+import org.apache.asterix.column.bytes.stream.in.DummyBytesInputStream;
import org.apache.asterix.column.bytes.stream.in.MultiByteBufferInputStream;
import org.apache.asterix.column.filter.FilterAccessorProvider;
import org.apache.asterix.column.filter.IColumnFilterEvaluator;
@@ -68,7 +69,10 @@
int numberOfPrimaryKeys = columnMetadata.getNumberOfPrimaryKeys();
filteredColumnStreams = new AbstractBytesInputStream[columnMetadata.getNumberOfFilteredColumns()];
for (int i = 0; i < filteredColumnStreams.length; i++) {
- if (filterColumnReaders.get(i).getColumnIndex() >= numberOfPrimaryKeys) {
+ int columnIndex = filterColumnReaders.get(i).getColumnIndex();
+ if (columnIndex < 0) {
+ filteredColumnStreams[i] = DummyBytesInputStream.INSTANCE;
+ } else if (columnIndex >= numberOfPrimaryKeys) {
filteredColumnStreams[i] = new MultiByteBufferInputStream();
} else {
filteredColumnStreams[i] = new ByteBufferInputStream();
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java
index 798c15f..a2d6b54 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java
@@ -24,6 +24,7 @@
import org.apache.asterix.column.assembler.value.MissingValueGetter;
import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
import org.apache.asterix.column.bytes.stream.in.ByteBufferInputStream;
+import org.apache.asterix.column.bytes.stream.in.DummyBytesInputStream;
import org.apache.asterix.column.bytes.stream.in.MultiByteBufferInputStream;
import org.apache.asterix.column.filter.FilterAccessorProvider;
import org.apache.asterix.column.filter.IColumnFilterEvaluator;
@@ -71,7 +72,10 @@
int numberOfPrimaryKeys = columnMetadata.getNumberOfPrimaryKeys();
filteredColumnStreams = new AbstractBytesInputStream[columnMetadata.getNumberOfFilteredColumns()];
for (int i = 0; i < filteredColumnStreams.length; i++) {
- if (filterColumnReaders.get(i).getColumnIndex() >= numberOfPrimaryKeys) {
+ int columnIndex = filterColumnReaders.get(i).getColumnIndex();
+ if (columnIndex < 0) {
+ filteredColumnStreams[i] = DummyBytesInputStream.INSTANCE;
+ } else if (columnIndex >= numberOfPrimaryKeys) {
filteredColumnStreams[i] = new MultiByteBufferInputStream();
} else {
filteredColumnStreams[i] = new ByteBufferInputStream();
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java
index 021b9f5..8ed6df6 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java
@@ -57,18 +57,19 @@
AbstractColumnValuesReader(AbstractValueReader valueReader, int columnIndex, int maxLevel, boolean primaryKey) {
this.valueReader = valueReader;
this.columnIndex = columnIndex;
- this.maxLevel = maxLevel;
+ this.maxLevel = valueReader.getTypeTag() == ATypeTag.MISSING ? Integer.MAX_VALUE : maxLevel;
definitionLevels = new ParquetRunLengthBitPackingHybridDecoder(ColumnValuesUtil.getBitWidth(maxLevel));
valuesStream = primaryKey ? new ByteBufferInputStream() : new MultiByteBufferInputStream();
this.primaryKey = primaryKey;
}
final void nextLevel() throws HyracksDataException {
+ valueIndex++;
+
if (allMissing) {
return;
}
- valueIndex++;
int actualLevel = definitionLevels.readInt();
//Check whether the level is for a null value
nullLevel = ColumnValuesUtil.isNull(nullBitMask, actualLevel);
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/NoOpValueReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/NoOpValueReader.java
index 1982c54..0d59499 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/NoOpValueReader.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/value/NoOpValueReader.java
@@ -24,7 +24,7 @@
import org.apache.asterix.om.types.ATypeTag;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public class NoOpValueReader extends AbstractValueReader {
+public final class NoOpValueReader extends AbstractValueReader {
public static final AbstractValueReader INSTANCE = new NoOpValueReader();
private NoOpValueReader() {
@@ -42,7 +42,7 @@
@Override
public ATypeTag getTypeTag() {
- throw new UnsupportedOperationException(getClass().getName());
+ return ATypeTag.MISSING;
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java
index 0ac8c75..283522a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java
@@ -68,7 +68,9 @@
filterBufferProviders = new IColumnBufferProvider[numberOfFilteredColumns];
for (int i = 0; i < numberOfFilteredColumns; i++) {
int columnIndex = info.getFilteredColumnIndex(i);
- if (columnIndex >= numberOfPrimaryKeys) {
+ if (columnIndex < 0) {
+ filterBufferProviders[i] = DummyColumnBufferProvider.INSTANCE;
+ } else if (columnIndex >= numberOfPrimaryKeys) {
filterBufferProviders[i] = new ColumnMultiBufferProvider(columnIndex, multiPageOp);
} else {
filterBufferProviders[i] = new ColumnSingleBufferProvider(columnIndex);