[ASTERIXDB-3601][STO] Properly unpinning segment files
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Properly unpinning the segment files, in case
of failure. Otherwise it might cause resource
leak.
Ext-ref: MB-66306
Change-Id: I1dec880871008fc6b9be6662df9fa034992858c9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19990
Reviewed-by: Peeyush Gupta <peeyush.gupta@couchbase.com>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Ritik Raj <ritik.raj@couchbase.com>
diff --git a/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf b/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf
index d5837ad..23071d6 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf
@@ -53,4 +53,4 @@
compiler.ordered.fields=false
messaging.frame.size=4096
messaging.frame.count=512
-storage.buffercache.pagesize=32KB
+storage.buffercache.pagesize=32KB
\ No newline at end of file
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentPageZeroBufferBytesOutputStream.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentPageZeroBufferBytesOutputStream.java
index c156e49..4ae0153 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentPageZeroBufferBytesOutputStream.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/out/MultiPersistentPageZeroBufferBytesOutputStream.java
@@ -42,7 +42,7 @@
@Override
protected void preReset() {
- if (allocatedBytes > 0) {
+ if (allocatedBytes > 0 || !buffers.isEmpty()) {
//This should not be the case, with the pageZero segments.
//As the stream should be finished after the flush.
throw new IllegalStateException("PageZero segments should already be finished after flush");
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/PageZeroWriterFlavorSelector.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/PageZeroWriterFlavorSelector.java
index 06e441a..831e459 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/PageZeroWriterFlavorSelector.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/PageZeroWriterFlavorSelector.java
@@ -44,12 +44,12 @@
* This class implements an optimization strategy for sparse columns:
* - Default writer: Allocates space for all columns in the schema (suitable for dense data)
* - Sparse writer: Only allocates space for present columns (suitable for sparse data)
+ * - Multi-page writers: Variants of the above writers that support multi-page operations
*</p>
* The selector automatically chooses the most space-efficient option based on the actual
* space requirements of each approach.
*/
public class PageZeroWriterFlavorSelector implements IColumnPageZeroWriterFlavorSelector {
- // Flag indicating which writer type is currently selected (DEFAULT_WRITER_FLAG=default, SPARSE_WRITER_FLAG=sparse)
protected byte writerFlag = IColumnPageZeroWriter.ColumnPageZeroWriterType.ADAPTIVE.getWriterFlag();
// Cache of writer instances to avoid repeated object creation
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/DefaultColumnPageZeroReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/DefaultColumnPageZeroReader.java
index 1692e70..a34d8c1 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/DefaultColumnPageZeroReader.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/readers/DefaultColumnPageZeroReader.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.column.zero.readers;
+import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.FLAG_OFFSET;
import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.LEFT_MOST_KEY_OFFSET;
import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.MEGA_LEAF_NODE_LENGTH;
import static org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.AbstractColumnBTreeLeafFrame.NEXT_LEAF_OFFSET;
@@ -31,9 +32,16 @@
import org.apache.asterix.column.zero.writers.DefaultColumnPageZeroWriter;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
import org.apache.hyracks.storage.am.lsm.btree.column.cloud.IntPairUtil;
+import org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException;
import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroReader;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
public class DefaultColumnPageZeroReader implements IColumnPageZeroReader {
+ protected static Logger LOGGER = LogManager.getLogger();
+
protected ByteBuffer pageZeroBuf;
protected BitSet pageZeroSegmentsPages;
protected int numberOfPresentColumns;
@@ -182,4 +190,14 @@
pageZeroSegmentsPages.set(0);
return pageZeroSegmentsPages;
}
+
+ @Override
+ public void printPageZeroReaderInfo() {
+ ColumnarValueException ex = new ColumnarValueException();
+ ObjectNode readerNode = ex.createNode(getClass().getSimpleName());
+ readerNode.put("headerSize", headerSize);
+ readerNode.put("numberOfPresentColumns", numberOfPresentColumns);
+ readerNode.put("flag", pageZeroBuf.get(FLAG_OFFSET));
+ LOGGER.debug("SingleColumnPageZeroReader Info: {}", readerNode.toPrettyString());
+ }
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/AbstractColumnMultiPageZeroReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/AbstractColumnMultiPageZeroReader.java
index 3b05fc3..b0fc721 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/AbstractColumnMultiPageZeroReader.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/AbstractColumnMultiPageZeroReader.java
@@ -20,8 +20,11 @@
import org.apache.asterix.column.bytes.stream.in.MultiPageZeroByteBuffersReader;
import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.IColumnPageZeroReader;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public abstract class AbstractColumnMultiPageZeroReader implements IColumnPageZeroReader {
+ protected static final Logger LOGGER = LogManager.getLogger();
protected MultiPageZeroByteBuffersReader segmentBuffers;
AbstractColumnMultiPageZeroReader() {
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java
index db800ae..5a0b180 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/DefaultColumnMultiPageZeroReader.java
@@ -40,6 +40,9 @@
import org.apache.hyracks.data.std.primitive.VoidPointable;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
import org.apache.hyracks.storage.am.lsm.btree.column.cloud.IntPairUtil;
+import org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
public class DefaultColumnMultiPageZeroReader extends AbstractColumnMultiPageZeroReader {
public static final int headerSize = DefaultColumnMultiPageZeroWriter.EXTENDED_HEADER_SIZE;
@@ -273,4 +276,15 @@
public int getHeaderSize() {
return EXTENDED_HEADER_SIZE;
}
+
+ @Override
+ public void printPageZeroReaderInfo() {
+ ColumnarValueException ex = new ColumnarValueException();
+ ObjectNode readerNode = ex.createNode(getClass().getSimpleName());
+ readerNode.put("headerSize", headerSize);
+ readerNode.put("maxColumnsInZerothSegment", zerothSegmentMaxColumns);
+ readerNode.put("maxNumberOfColumnsInAPage", maxNumberOfColumnsInAPage);
+ readerNode.put("numberOfPageZeroSegments", numberOfPageZeroSegments);
+ LOGGER.debug("DefaultColumnMultiPageZeroReader Info: {}", readerNode.toPrettyString());
+ }
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroReader.java
index d555bdc..0a0a817 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroReader.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/zero/writers/multipage/SparseColumnMultiPageZeroReader.java
@@ -40,6 +40,9 @@
import org.apache.hyracks.data.std.primitive.VoidPointable;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnBufferProvider;
import org.apache.hyracks.storage.am.lsm.btree.column.cloud.IntPairUtil;
+import org.apache.hyracks.storage.am.lsm.btree.column.error.ColumnarValueException;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
@@ -358,4 +361,16 @@
}
return pageZeroSegmentsPages;
}
+
+ @Override
+ public void printPageZeroReaderInfo() {
+ ColumnarValueException ex = new ColumnarValueException();
+ ObjectNode readerNode = ex.createNode(getClass().getSimpleName());
+ readerNode.put("headerSize", headerSize);
+ readerNode.put("maxColumnIndexInZerothSegment", maxColumnIndexInZerothSegment);
+ readerNode.put("numberOfColumnInZerothSegment", numberOfColumnInZerothSegment);
+ readerNode.put("maxNumberOfColumnsInAPage", maxNumberOfColumnsInAPage);
+ readerNode.put("numberOfPageZeroSegments", numberOfPageZeroSegments);
+ LOGGER.debug("SparseColumnMultiPageZeroReader Info: {}", readerNode.toPrettyString());
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweeper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweeper.java
index 1f0c7f4..ba4376f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweeper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweeper.java
@@ -165,6 +165,7 @@
int freedSpace = 0;
context.open(fileId);
try {
+ Throwable failure = null;
while (nextPageId >= 0) {
if (context.stopSweeping()) {
// Exit as the index is being dropped
@@ -183,17 +184,30 @@
ranges.reset(leafFrame, plan);
freedSpace += punchHoles(context, leafFrame);
}
+ } catch (Throwable th) {
+ failure = th;
} finally {
if (columnsLocked) {
page0.sweepUnlock();
}
// unpin segment pages
for (ICachedPage cachedPage : segmentPagesTempHolder) {
- context.unpin(cachedPage, bcOpCtx);
+ try {
+ context.unpin(cachedPage, bcOpCtx);
+ } catch (Throwable e) {
+ if (failure == null) {
+ failure = e;
+ } else {
+ failure.addSuppressed(e);
+ }
+ }
}
context.unpin(page0, bcOpCtx);
}
}
+ if (failure != null) {
+ throw HyracksDataException.create(failure);
+ }
} finally {
context.close();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java
index 4a78400..c3f8228 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeReadLeafFrame.java
@@ -188,4 +188,8 @@
public BitSet markRequiredPageZeroSegments(BitSet projectedColumns, int pageZeroId, boolean markAll) {
return columnPageZeroReader.markRequiredPageSegments(projectedColumns, pageZeroId, markAll);
}
+
+ public void printPageZeroReaderInfo() {
+ columnPageZeroReader.printPageZeroReaderInfo();
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroReader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroReader.java
index db4f778..d9cf11b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroReader.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/IColumnPageZeroReader.java
@@ -75,4 +75,6 @@
void resetStream(IColumnBufferProvider pageZeroSegmentBufferProvider) throws HyracksDataException;
BitSet markRequiredPageSegments(BitSet projectedColumns, int pageZeroId, boolean markAll);
+
+ void printPageZeroReaderInfo();
}