[ASTERIXDB-3586][STO] Sync tupleIndex while skipping tuples

- user model changes: no
- storage format changes: no
- interface changes: yes

Details:
When `compiler.column.filter` is enabled, the assembler’s `valueIndex`
(denoted as `tupleIndex` currently) may become out of sync with the
filter’s `tupleIndex`.
This misalignment between the filter and assembler
can lead to incorrect query results. This change ensures proper
synchronization of `tupleIndex` while skipping tuples to maintain
correctness.

Ext-ref: MB-66000
Change-Id: I260612851c4dabfb9e74f2902f421720d9f88657
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19555
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Peeyush Gupta <peeyush.gupta@couchbase.com>
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AbstractPrimitiveValueAssembler.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AbstractPrimitiveValueAssembler.java
index 3c5d726..eb2ddf8 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AbstractPrimitiveValueAssembler.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/AbstractPrimitiveValueAssembler.java
@@ -81,6 +81,10 @@
         reader.skip(count);
     }
 
+    public IColumnValuesReader getReader() {
+        return reader;
+    }
+
     /**
      * Reset the assembler
      *
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/FalseColumnFilterEvaluator.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/FalseColumnFilterEvaluator.java
index 51f125b..5ea7fc3 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/FalseColumnFilterEvaluator.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/FalseColumnFilterEvaluator.java
@@ -19,6 +19,7 @@
 package org.apache.asterix.column.filter;
 
 import org.apache.asterix.column.filter.iterable.IColumnIterableFilterEvaluator;
+import org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException;
 
 public class FalseColumnFilterEvaluator implements IColumnIterableFilterEvaluator {
     public static final IColumnIterableFilterEvaluator INSTANCE = new FalseColumnFilterEvaluator();
@@ -27,7 +28,7 @@
     }
 
     @Override
-    public void reset() {
+    public void reset(int tupleCount) {
 
     }
 
@@ -50,4 +51,9 @@
     public void setAt(int index) {
         // NoOp
     }
+
+    @Override
+    public void appendInformation(ColumnarValueException e) {
+        // NoOp
+    }
 }
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/TrueColumnFilterEvaluator.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/TrueColumnFilterEvaluator.java
index 0841e01..125d79c 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/TrueColumnFilterEvaluator.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/TrueColumnFilterEvaluator.java
@@ -20,6 +20,7 @@
 
 import org.apache.asterix.column.filter.iterable.IColumnIterableFilterEvaluator;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException;
 
 /**
  * This evaluator is also used to indicate a NoOp filter
@@ -36,7 +37,7 @@
     }
 
     @Override
-    public void reset() {
+    public void reset(int tupleCount) {
         // NoOp
     }
 
@@ -54,4 +55,9 @@
     public void setAt(int index) {
         // NoOp
     }
+
+    @Override
+    public void appendInformation(ColumnarValueException e) {
+        // NoOp
+    }
 }
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/IColumnIterableFilterEvaluator.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/IColumnIterableFilterEvaluator.java
index 1d2838e..582f9ca 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/IColumnIterableFilterEvaluator.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/IColumnIterableFilterEvaluator.java
@@ -20,13 +20,16 @@
 
 import org.apache.asterix.column.filter.IColumnFilterEvaluator;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException;
 
 public interface IColumnIterableFilterEvaluator extends IColumnFilterEvaluator {
-    void reset();
+    void reset(int tupleCount);
 
     int getTupleIndex();
 
     int getValueIndex();
 
     void setAt(int index) throws HyracksDataException;
+
+    void appendInformation(ColumnarValueException e);
 }
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/AbstractIterableFilterEvaluator.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/AbstractIterableFilterEvaluator.java
index 8f0ad89..30c051f 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/AbstractIterableFilterEvaluator.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/AbstractIterableFilterEvaluator.java
@@ -26,6 +26,10 @@
 import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException;
+
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 
 abstract class AbstractIterableFilterEvaluator implements IColumnIterableFilterEvaluator {
     protected final IScalarEvaluator evaluator;
@@ -34,19 +38,21 @@
     private final VoidPointable booleanResult;
     protected int tupleIndex;
     protected int valueIndex;
+    private int tupleCount;
 
     AbstractIterableFilterEvaluator(IScalarEvaluator evaluator, List<IColumnValuesReader> readers) {
         this.evaluator = evaluator;
         this.primaryKeyReader = readers.get(0);
         this.readers = readers;
         this.booleanResult = new VoidPointable();
-        reset();
+        reset(-1);
     }
 
     @Override
-    public final void reset() {
+    public final void reset(int tupleCount) {
         tupleIndex = -1;
         valueIndex = -1;
+        this.tupleCount = tupleCount;
     }
 
     @Override
@@ -68,6 +74,9 @@
         // 0(skip) --> 1(skip) --> 2nd tuple
         // so the gap between index and tupleIndex is count.
         // and after increasing by (count - 1), evaluate() for the Xth tuple.
+        if (index > tupleCount) {
+            index = tupleCount;
+        }
         int count = index - this.tupleIndex;
         if (count > 0) {
             tupleIndex += count - 1;
@@ -101,7 +110,7 @@
         int nonAntiMatterCount = 0;
         for (int i = 0; i < count; i++) {
             primaryKeyReader.next();
-            nonAntiMatterCount += primaryKeyReader.isValue() ? 0 : 1;
+            nonAntiMatterCount += primaryKeyReader.isValue() ? 1 : 0;
         }
         for (int i = 1; nonAntiMatterCount > 0 && i < readers.size(); i++) {
             readers.get(i).skip(nonAntiMatterCount);
@@ -114,4 +123,17 @@
         return BinaryBooleanInspector.getBooleanValue(booleanResult.getByteArray(), booleanResult.getStartOffset(),
                 booleanResult.getLength());
     }
+
+    @Override
+    public void appendInformation(ColumnarValueException e) {
+        ObjectNode filterIteratorNode = e.createNode(getClass().getSimpleName());
+        filterIteratorNode.put("filterTupleIndex", tupleIndex);
+        filterIteratorNode.put("filterValueIndex", valueIndex);
+        ArrayNode pkNodes = filterIteratorNode.putArray("filterPrimaryKeyReaders");
+        primaryKeyReader.appendReaderInformation(pkNodes.addObject());
+        ArrayNode valueNodes = filterIteratorNode.putArray("filterValueReaders");
+        for (int i = 1; i < readers.size(); i++) {
+            readers.get(i).appendReaderInformation(valueNodes.addObject());
+        }
+    }
 }
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnIterableFilterEvaluator.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnIterableFilterEvaluator.java
index 094b2bf..ad0e8a9 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnIterableFilterEvaluator.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnIterableFilterEvaluator.java
@@ -33,14 +33,13 @@
     @Override
     public boolean evaluate() throws HyracksDataException {
         boolean result = false;
+        // If next() returns false, it means there are no more matching tuples left.
+        // Inside the next() function, tupleIndex and valueIndex are incremented.
+        // This causes tupleIndex to eventually reach tupleCount,
+        // indicating that all tuples have been processed.
         while (!result && next()) {
             result = inspect();
         }
-        if (!result) {
-            // Last tuple does not satisfy the condition
-            tupleIndex++;
-            valueIndex++;
-        }
         return result;
     }
 }
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnarRepeatedIterableFilterEvaluator.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnarRepeatedIterableFilterEvaluator.java
index 9a1f07f..05e1dae 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnarRepeatedIterableFilterEvaluator.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/filter/iterable/evaluator/ColumnarRepeatedIterableFilterEvaluator.java
@@ -46,11 +46,6 @@
             // TODO we need a way to 'rewind' y for each x
             result = evaluateRepeated();
         }
-        if (!result) {
-            // Last tuple does not satisfy the condition
-            tupleIndex++;
-            valueIndex++;
-        }
         return result;
     }
 
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/ColumnAssembler.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/ColumnAssembler.java
index 82a9b63..e6ff021 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/ColumnAssembler.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/ColumnAssembler.java
@@ -31,6 +31,7 @@
 import org.apache.hyracks.data.std.api.IValueReference;
 import org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException;
 
+import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 
 public final class ColumnAssembler {
@@ -132,6 +133,15 @@
     }
 
     public void setAt(int index) throws HyracksDataException {
+        int skipCount = index - tupleIndex;
+        if (skipCount < 0) {
+            // should never happen as tupleIndex progress forward.
+            ColumnarValueException e = new ColumnarValueException();
+            e.getNode().put("message",
+                    "skipCount is negative, currentTupleIndex: " + tupleIndex + ", finalTupleIndex: " + index);
+            appendInformation(e);
+            throw e;
+        }
         skip(index - tupleIndex);
     }
 
@@ -140,6 +150,10 @@
         assemblerNode.put("tupleIndex", tupleIndex);
         assemblerNode.put("numberOfTuples", numberOfTuples);
         assemblerNode.put("numberOfSkips", numberOfSkips);
+        ArrayNode assemblerReaders = assemblerNode.putArray("assemblerReaders");
+        for (int i = 0; i < assemblers.length; i++) {
+            assemblers[i].getReader().appendReaderInformation(assemblerReaders.addObject());
+        }
         state.appendStateInfo(e);
     }
 }
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/MergeColumnTupleReference.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/MergeColumnTupleReference.java
index 8c434d7..6e13113 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/MergeColumnTupleReference.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/MergeColumnTupleReference.java
@@ -101,7 +101,16 @@
     }
 
     @Override
-    public void skip(int count) throws HyracksDataException {
+    public void initSkip(int tupleIndex, int count) throws HyracksDataException {
+        skip(count);
+    }
+
+    @Override
+    public void skipCurrentTuple() throws HyracksDataException {
+        skip(1);
+    }
+
+    private void skip(int count) throws HyracksDataException {
         skipCount += count;
     }
 
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java
index 64ac0a2..8113288 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnTupleReference.java
@@ -104,7 +104,7 @@
         //Check if we should read all column pages
         boolean readColumns = rangeFilterEvaluator.evaluate();
         assembler.reset(readColumns ? numberOfTuples : 0);
-        columnFilterEvaluator.reset();
+        columnFilterEvaluator.reset(numberOfTuples);
         previousIndex = -1;
         return readColumns;
     }
@@ -131,8 +131,29 @@
     }
 
     @Override
-    public void skip(int count) throws HyracksDataException {
-        columnFilterEvaluator.setAt(assembler.skip(count));
+    public void initSkip(int tupleIndex, int count) throws HyracksDataException {
+        columnFilterEvaluator.setAt(tupleIndex);
+        skip(count);
+    }
+
+    @Override
+    public void skipCurrentTuple() throws HyracksDataException {
+        if (isFilterApplied()) {
+            // since cursorIndex is markedAsDeleted
+            // move filter evaluator to the next valid tuple
+            columnFilterEvaluator.setAt(tupleIndex + 1);
+            assembler.setAt(columnFilterEvaluator.getValueIndex());
+        } else {
+            skip(1);
+        }
+    }
+
+    private boolean isFilterApplied() {
+        return columnFilterEvaluator.getTupleIndex() != -1;
+    }
+
+    private void skip(int count) throws HyracksDataException {
+        assembler.skip(count);
     }
 
     public IValueReference getAssembledValue() throws HyracksDataException {
@@ -158,19 +179,22 @@
     }
 
     private IValueReference getFilteredAssembledValue() throws HyracksDataException {
-        int index = columnFilterEvaluator.getTupleIndex();
-        // index == -1 if the normalized filter indicated that a mega leaf node
-        // is filtered
-        if (index == tupleIndex) {
-            antimatterGap = 0;
-            // setAt in the assembler expect the value index (i.e., tupleCount - antiMatterCount)
-            assembler.setAt(columnFilterEvaluator.getValueIndex());
-            // set the next tuple index that satisfies the filter
-            columnFilterEvaluator.evaluate();
-            return assembler.nextValue();
-        } else {
-            antimatterGap++;
+        try {
+            int index = columnFilterEvaluator.getTupleIndex();
+            // index == -1 if the normalized filter indicated that a mega leaf node
+            // is filtered
+            if (index == tupleIndex) {
+                // setAt in the assembler expect the value index (i.e., tupleCount - antiMatterCount)
+                assembler.setAt(columnFilterEvaluator.getValueIndex());
+                // set the next tuple index that satisfies the filter
+                columnFilterEvaluator.evaluate();
+                return assembler.nextValue();
+            }
+            return MissingValueGetter.MISSING;
+        } catch (ColumnarValueException e) {
+            columnFilterEvaluator.appendInformation(e);
+            e.getNode().put("cursorTupleIndex", tupleIndex);
+            throw e;
         }
-        return MissingValueGetter.MISSING;
     }
 }
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java
index 4202f47..2b8da6d 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/tuple/QueryColumnWithMetaTupleReference.java
@@ -109,7 +109,7 @@
         boolean readColumns = rangeFilterEvaluator.evaluate();
         assembler.reset(readColumns ? numberOfTuples : 0);
         metaAssembler.reset(readColumns ? numberOfTuples : 0);
-        columnFilterEvaluator.reset();
+        columnFilterEvaluator.reset(numberOfTuples);
         previousIndex = -1;
         return readColumns;
     }
@@ -141,9 +141,31 @@
     }
 
     @Override
-    public void skip(int count) throws HyracksDataException {
+    public void initSkip(int tupleIndex, int count) throws HyracksDataException {
+        columnFilterEvaluator.setAt(tupleIndex);
+        skip(count);
+    }
+
+    @Override
+    public void skipCurrentTuple() throws HyracksDataException {
+        if (isFilterApplied()) {
+            // since cursorIndex is markedAsDeleted
+            // move filter evaluator to the next valid tuple
+            columnFilterEvaluator.setAt(tupleIndex + 1);
+            assembler.setAt(columnFilterEvaluator.getValueIndex());
+            metaAssembler.setAt(columnFilterEvaluator.getValueIndex());
+        } else {
+            skip(1);
+        }
+    }
+
+    private boolean isFilterApplied() {
+        return columnFilterEvaluator.getTupleIndex() != -1;
+    }
+
+    private void skip(int count) throws HyracksDataException {
         metaAssembler.skip(count);
-        columnFilterEvaluator.setAt(assembler.skip(count));
+        assembler.skip(count);
     }
 
     public IValueReference getAssembledValue() throws HyracksDataException {
@@ -181,22 +203,25 @@
     }
 
     private IValueReference getFilteredAssembledValue() throws HyracksDataException {
-        int index = columnFilterEvaluator.getTupleIndex();
+        try {
+            int index = columnFilterEvaluator.getTupleIndex();
 
-        // index == -1 if the normalized filter indicated that a mega leaf node is filtered
-        if (index == tupleIndex) {
-            // setAt in the assembler expect the value index (i.e., tupleCount - antiMatterCount)
-            antimatterGap = 0;
-            int valueIndex = columnFilterEvaluator.getValueIndex();
-            assembler.setAt(valueIndex);
-            metaAssembler.setAt(valueIndex);
-            // set the next tuple index that satisfies the filter
-            columnFilterEvaluator.evaluate();
-            return assembler.nextValue();
-        } else {
-            antimatterGap++;
+            // index == -1 if the normalized filter indicated that a mega leaf node is filtered
+            if (index == tupleIndex) {
+                // setAt in the assembler expect the value index (i.e., tupleCount - antiMatterCount)
+                int valueIndex = columnFilterEvaluator.getValueIndex();
+                assembler.setAt(valueIndex);
+                metaAssembler.setAt(valueIndex);
+                // set the next tuple index that satisfies the filter
+                columnFilterEvaluator.evaluate();
+                return assembler.nextValue();
+            }
+
+            return MissingValueGetter.MISSING;
+        } catch (ColumnarValueException e) {
+            columnFilterEvaluator.appendInformation(e);
+            e.getNode().put("cursorTupleIndex", tupleIndex);
+            throw e;
         }
-
-        return MissingValueGetter.MISSING;
     }
 }
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java
index 576b884..62d45a4 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java
@@ -31,6 +31,7 @@
 import org.apache.asterix.om.types.ATypeTag;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.parquet.bytes.BytesUtils;
@@ -93,10 +94,12 @@
             numberOfEncounteredMissing += isMissing() ? 1 : 0;
             numberOfEncounteredNull += isNull() ? 1 : 0;
         } catch (Exception e) {
-            ObjectNode infoNode = OBJECT_MAPPER.createObjectNode();
-            appendReaderInformation(infoNode);
-            LOGGER.error("error reading nextLevel, collected info: {}", infoNode);
-            throw HyracksDataException.create(e);
+            ColumnarValueException ex = new ColumnarValueException();
+            ObjectNode readerNode = ex.createNode(getClass().getSimpleName());
+            appendReaderInformation(readerNode);
+            LOGGER.error("error reading nextLevel, collected info: {}", ex.getNode());
+            ex.addSuppressed(e);
+            throw ex;
         }
     }
 
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/PrimitiveColumnValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/PrimitiveColumnValuesReader.java
index 7b02c70..07569e6 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/PrimitiveColumnValuesReader.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/PrimitiveColumnValuesReader.java
@@ -117,8 +117,8 @@
 
     @Override
     public int reset(int startIndex, int skipCount) throws HyracksDataException {
-        ((IColumnKeyValueReader) valueReader).reset(startIndex, skipCount);
         // first item
+        ((IColumnKeyValueReader) valueReader).reset(startIndex, skipCount);
         nextLevel();
         int numberOfAntiMatters = level < maxLevel ? 1 : 0;
         for (int i = 0; i < skipCount; i++) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnTupleIterator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnTupleIterator.java
index 4b9cc99..7576f4a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnTupleIterator.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnTupleIterator.java
@@ -79,11 +79,21 @@
     boolean isConsumed();
 
     /**
-     * Skip a number of tuples
+     * Skip a number of tuples on page initialization
      *
-     * @param count the number of tuples that needed to be skipped
+     * @param tupleIndex
+     * @param count      the number of tuples that needed to be skipped
      */
-    void skip(int count) throws HyracksDataException;
+    void initSkip(int tupleIndex, int count) throws HyracksDataException;
+
+    /**
+     * Skips the current tuple.
+     * This is used when a tuple is marked as deleted by the cursor, prompting
+     * the filter and column to also skip it.
+     *
+     * @throws HyracksDataException if an error occurs while skipping the tuple.
+     */
+    void skipCurrentTuple() throws HyracksDataException;
 
     /**
      * Move to the next tuple
@@ -108,11 +118,4 @@
     void unpinColumnsPages() throws HyracksDataException;
 
     void close();
-
-    /**
-     * @return the gap between the lastAssembledTuple index and the current antimatter tuple index
-     */
-    int getAntimatterGap();
-
-    void resetAntimatterGap();
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/error/ColumnarValueException.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/error/ColumnarValueException.java
index 79c7864..d3de601 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/error/ColumnarValueException.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/error/ColumnarValueException.java
@@ -39,6 +39,10 @@
         node = OBJECT_MAPPER.createObjectNode();
     }
 
+    public ObjectNode getNode() {
+        return node;
+    }
+
     public ObjectNode createNode(String fieldName) {
         return node.putObject(fieldName);
     }
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeRangeSearchCursor.java
index cee77edd..c3fc53d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeRangeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTreeRangeSearchCursor.java
@@ -86,9 +86,7 @@
         IColumnTupleIterator columnTuple = (IColumnTupleIterator) e.getTuple();
         if (!columnTuple.isAntimatter()) {
             // Skip non-key columns
-            int antiMatterGap = columnTuple.getAntimatterGap();
-            columnTuple.skip(antiMatterGap + 1);
-            columnTuple.resetAntimatterGap();
+            columnTuple.skipCurrentTuple();
         }
     }
 
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java
index 1937ffc..f079f51 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/tuples/AbstractColumnTupleReference.java
@@ -44,9 +44,8 @@
     private final IColumnBufferProvider[] filterBufferProviders;
     private final IColumnBufferProvider[] buffersProviders;
     private final int numberOfPrimaryKeys;
-    private int endIndex;
+    protected int endIndex;
     protected int tupleIndex;
-    protected int antimatterGap;
 
     // For logging
     private final LongSet pinnedPages;
@@ -103,7 +102,6 @@
     @Override
     public final void newPage() throws HyracksDataException {
         tupleIndex = 0;
-        antimatterGap = 0;
         ByteBuffer pageZero = frame.getBuffer();
         pageZero.clear();
         pageZero.position(HEADER_SIZE);
@@ -121,7 +119,6 @@
     @Override
     public final void reset(int startIndex, int endIndex) throws HyracksDataException {
         tupleIndex = startIndex;
-        antimatterGap = 0;
         this.endIndex = endIndex;
         ByteBuffer pageZero = frame.getBuffer();
         int numberOfTuples = frame.getTupleCount();
@@ -156,7 +153,7 @@
              * skipCount from calling setPrimaryKeysAt(startIndex, startIndex) is a negative value. For that reason,
              * non-key column should not skip any value.
              */
-            skip(Math.max(skipCount, 0));
+            initSkip(tupleIndex, Math.max(skipCount, 0));
         } else {
             skipMegaLeafNode();
             numOfSkippedMegaLeafNodes++;
@@ -194,7 +191,7 @@
          * For values, we need to do 6 skips, as next will be called later by the assembler
          * -- setting the position at 12 as well.
          */
-        skip(skipCount);
+        initSkip(tupleIndex, skipCount);
     }
 
     protected abstract int setPrimaryKeysAt(int index, int skipCount) throws HyracksDataException;
@@ -296,14 +293,4 @@
     public final void resetByTupleIndex(ITreeIndexFrame frame, int tupleIndex) {
         throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG);
     }
-
-    @Override
-    public int getAntimatterGap() {
-        return antimatterGap;
-    }
-
-    @Override
-    public void resetAntimatterGap() {
-        antimatterGap = 0;
-    }
 }