[ASTERIXDB-3202][STO] Clean dangling LAF
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Clean LAF when its corresponding on-disk
component is missing.
Change-Id: I19d2177f8e77c47ae256855a0d599fa19a2625bd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17584
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
index 08eba0d..98b82db 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
@@ -25,6 +25,7 @@
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
import org.apache.hyracks.api.exceptions.ErrorCode;
@@ -77,16 +78,19 @@
@Override
public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException {
List<LSMComponentFileReferences> validFiles = new ArrayList<>();
- ArrayList<IndexComponentFileReference> allBTreeFiles = new ArrayList<>();
- ArrayList<IndexComponentFileReference> allBloomFilterFiles = new ArrayList<>();
+ List<IndexComponentFileReference> allBTreeFiles = new ArrayList<>();
+ List<IndexComponentFileReference> allBloomFilterFiles = new ArrayList<>();
// List of valid BTree files.
cleanupAndGetValidFilesInternal(BTREE_FILTER, btreeFactory, allBTreeFiles, btreeFactory.getBufferCache());
- HashSet<String> btreeFilesSet = new HashSet<>();
+ Set<String> btreeFilesSet = new HashSet<>();
for (IndexComponentFileReference cmpFileName : allBTreeFiles) {
int index = cmpFileName.getFileName().lastIndexOf(DELIMITER);
btreeFilesSet.add(cmpFileName.getFileName().substring(0, index));
}
+ // Clean up LAFs if any
+ cleanLookAsideFiles(btreeFilesSet, btreeFactory.getBufferCache());
+
if (hasBloomFilter) {
validateFiles(btreeFilesSet, allBloomFilterFiles, BLOOM_FILTER_FILTER, null, btreeFactory.getBufferCache());
// Sanity check.
@@ -156,11 +160,11 @@
// Sort valid files in reverse lexicographical order, such that newer
// files come first.
- Collections.sort(validComparableBTreeFiles, recencyCmp);
+ validComparableBTreeFiles.sort(recencyCmp);
Iterator<IndexComponentFileReference> btreeFileIter = validComparableBTreeFiles.iterator();
Iterator<IndexComponentFileReference> bloomFilterFileIter = null;
if (hasBloomFilter) {
- Collections.sort(validComparableBloomFilterFiles, recencyCmp);
+ validComparableBloomFilterFiles.sort(recencyCmp);
bloomFilterFileIter = validComparableBloomFilterFiles.iterator();
}
IndexComponentFileReference cmpBTreeFileName = null;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
index 610232f..6194854 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
@@ -23,7 +23,6 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
-import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -82,6 +81,7 @@
public static final FilenameFilter COMPONENT_FILES_FILTER = (dir, name) -> !name.startsWith(".");
protected static FilenameFilter BLOOM_FILTER_FILTER =
(dir, name) -> !name.startsWith(".") && name.endsWith(BLOOM_FILTER_SUFFIX);
+ protected static FilenameFilter LAF_FILTER = (dir, name) -> !name.startsWith(".") && name.endsWith(LAF_SUFFIX);
protected static final Comparator<String> cmp = new FileNameComparator();
protected final IIOManager ioManager;
@@ -137,7 +137,7 @@
}
protected void cleanupAndGetValidFilesInternal(FilenameFilter filter,
- TreeIndexFactory<? extends ITreeIndex> treeFactory, ArrayList<IndexComponentFileReference> allFiles,
+ TreeIndexFactory<? extends ITreeIndex> treeFactory, List<IndexComponentFileReference> allFiles,
IBufferCache bufferCache) throws HyracksDataException {
Set<FileReference> files = ioManager.list(baseDir, filter);
for (FileReference filePath : files) {
@@ -155,7 +155,7 @@
}
}
- protected void validateFiles(HashSet<String> groundTruth, ArrayList<IndexComponentFileReference> validFiles,
+ protected void validateFiles(Set<String> groundTruth, List<IndexComponentFileReference> validFiles,
FilenameFilter filter, TreeIndexFactory<? extends ITreeIndex> treeFactory, IBufferCache bufferCache)
throws HyracksDataException {
ArrayList<IndexComponentFileReference> tmpAllInvListsFiles = new ArrayList<>();
@@ -307,6 +307,21 @@
return baseDir.getChild(name);
}
+ protected void cleanLookAsideFiles(Set<String> groundTruth, IBufferCache bufferCache) throws HyracksDataException {
+ ICompressorDecompressor compDecomp = compressorDecompressorFactory.createInstance();
+ if (compDecomp == NoOpCompressorDecompressor.INSTANCE) {
+ return;
+ }
+
+ List<IndexComponentFileReference> allLookAsideFiles = new ArrayList<>();
+ cleanupAndGetValidFilesInternal(LAF_FILTER, null, allLookAsideFiles, null);
+ for (IndexComponentFileReference laf : allLookAsideFiles) {
+ if (!groundTruth.contains(laf.getSequence())) {
+ delete(bufferCache, laf.getFileRef());
+ }
+ }
+ }
+
private boolean isCompressible(String fileName) {
return !fileName.endsWith(BLOOM_FILTER_SUFFIX) && !fileName.endsWith(DELETE_TREE_SUFFIX);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
index 28b0139..1477535 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
@@ -57,7 +57,8 @@
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeComponentLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeComponentLifecycleTest.java
index 7f8fd8a..8608cac 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeComponentLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeComponentLifecycleTest.java
@@ -82,7 +82,8 @@
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
scheduler, ioCallbackFactory, harness.getPageWriteCallbackFactory(),
- harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
}
private OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys)
@@ -425,7 +426,7 @@
private class User extends SingleThreadEventProcessor<Request> {
- private Semaphore step = new Semaphore(0);
+ private final Semaphore step = new Semaphore(0);
public User(String username) {
super(username);
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
index 857a20c..fb25ee7 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
@@ -57,7 +57,8 @@
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java
index a6ba178..c607cae 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java
@@ -22,6 +22,7 @@
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.IoUtil;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
@@ -30,6 +31,7 @@
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.common.compression.file.CompressedFileReference;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -56,7 +58,8 @@
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, 1,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
ctx.getIndex().create();
ctx.getIndex().activate();
@@ -80,16 +83,22 @@
ctx.getIndex().deactivate();
// Delete the btree file and keep the bloom filter file from the disk component
- LSMBTreeWithBloomFilterDiskComponent ilsmDiskComponent =
+ LSMBTreeWithBloomFilterDiskComponent diskComponent =
(LSMBTreeWithBloomFilterDiskComponent) btree.getDiskComponents().get(0);
- ilsmDiskComponent.getIndex().getFileReference().delete();
+ CompressedFileReference fileReference = (CompressedFileReference) diskComponent.getIndex().getFileReference();
- File bloomFilterFile = ilsmDiskComponent.getBloomFilter().getFileReference().getFile().getAbsoluteFile();
- Assert.assertEquals("Check bloom filter file exists", true, bloomFilterFile.exists());
+ // Only delete the index file
+ IoUtil.delete(fileReference.getFile());
+ Assert.assertTrue("Check LAF exists", fileReference.getLAFFileReference().getFile().exists());
+
+ File bloomFilterFile = diskComponent.getBloomFilter().getFileReference().getFile().getAbsoluteFile();
+ Assert.assertTrue("Check bloom filter file exists", bloomFilterFile.exists());
// Activating the index again should delete the orphaned bloom filter file as well as the disk component
ctx.getIndex().activate();
- Assert.assertEquals("Check bloom filter file deleted", false, bloomFilterFile.exists());
+ Assert.assertFalse("Check bloom filter file deleted", bloomFilterFile.exists());
Assert.assertEquals("Check disk components", 0, btree.getDiskComponents().size());
+ // After index activation, the dangling LAF should be removed
+ Assert.assertFalse("Check LAF deleted", fileReference.getLAFFileReference().getFile().exists());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFilterMergeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFilterMergeTest.java
index 7778308..bf76e20 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFilterMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFilterMergeTest.java
@@ -56,7 +56,8 @@
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), filtered, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), filtered, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
index 69760bf..dace731 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
@@ -57,7 +57,8 @@
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
index 431abb8..d131ccb 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
@@ -43,10 +43,7 @@
@Override
protected boolean persistentStateExists() throws Exception {
// make sure all of the directories exist
- if (!harness.getFileReference().getFile().exists()) {
- return false;
- }
- return true;
+ return harness.getFileReference().getFile().exists();
}
@Override
@@ -61,7 +58,8 @@
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, fieldSerdes.length,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
index = testCtx.getIndex();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeFailTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeFailTest.java
index bd5d97d..bfa9ba6 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeFailTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeFailTest.java
@@ -105,7 +105,8 @@
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
scheduler, harness.getIOOperationCallbackFactory(), harness.getPageWriteCallbackFactory(),
- harness.getMetadataPageManagerFactory(), filtered, true, false);
+ harness.getMetadataPageManagerFactory(), filtered, true, false,
+ harness.getCompressorDecompressorFactory());
}
private class TestIoScheduler implements ILSMIOOperationScheduler {
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
index c9a2a99..b21a8b9 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
@@ -56,7 +56,8 @@
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), filtered, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), filtered, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
index 07cd08f..d131b1f 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
@@ -58,7 +58,8 @@
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java
index 4e59ab0..cdf1500 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java
@@ -44,6 +44,7 @@
import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback;
import org.apache.hyracks.storage.common.buffercache.IRateLimiter;
import org.apache.hyracks.storage.common.buffercache.SleepRateLimiter;
+import org.apache.hyracks.storage.common.compression.NoOpCompressorDecompressorFactory;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -53,14 +54,15 @@
private final OrderedIndexTestUtils orderedIndexTestUtils;
- private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
+ // Uses NoOp compressor/decompressor to mitigate assertions against number of pages
+ private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness(NoOpCompressorDecompressorFactory.INSTANCE);
private final int PAGES_PER_FORCE = 16;
private int pageCounter = 0;
private LSMIndexPageWriteCallback lastCallback = null;
private final IRateLimiter testLimiter = new IRateLimiter() {
- IRateLimiter limiter = SleepRateLimiter.create(100 * 1000);
+ final IRateLimiter limiter = SleepRateLimiter.create(100 * 1000);
@Override
public void setRate(double ratePerSecond) {
@@ -112,7 +114,8 @@
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(), pageWriteCallbackFactory,
- harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeScanDiskComponentsTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeScanDiskComponentsTest.java
index c9c87bd..4f2b2f7 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeScanDiskComponentsTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeScanDiskComponentsTest.java
@@ -80,7 +80,8 @@
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java
index 8d82405..0125318 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java
@@ -1,20 +1,20 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.apache.hyracks.storage.am.lsm.btree;
@@ -124,7 +124,7 @@
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, !hasOnlyKeys,
- hasOnlyKeys);
+ hasOnlyKeys, harness.getCompressorDecompressorFactory());
}
interface IndexModification {
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
index ceb8c11..de2dfa5 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
@@ -57,7 +57,8 @@
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
index 928f60c..82f6f4b 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
@@ -22,6 +22,7 @@
import java.util.Collection;
import java.util.List;
+import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
@@ -42,7 +43,6 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMPageWriteCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.compression.NoOpCompressorDecompressorFactory;
import org.apache.hyracks.util.trace.ITraceCategoryRegistry;
import org.apache.hyracks.util.trace.ITracer;
import org.apache.hyracks.util.trace.TraceCategoryRegistry;
@@ -82,7 +82,8 @@
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
IMetadataPageManagerFactory metadataPageManagerFactory, boolean filtered, boolean needKeyDupCheck,
- boolean updateAware) throws HyracksDataException {
+ boolean updateAware, ICompressorDecompressorFactory compressorDecompressorFactory)
+ throws HyracksDataException {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
int[] bloomFilterKeyFields = new int[numKeyFields];
@@ -103,7 +104,7 @@
cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, needKeyDupCheck, filterTypeTraits,
filterCmp, btreefields, filterfields, true, metadataPageManagerFactory, updateAware, ITracer.NONE,
- NoOpCompressorDecompressorFactory.INSTANCE, true, null, null);
+ compressorDecompressorFactory, true, null, null);
} else {
lsmTree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache, typeTraits,
cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
@@ -111,7 +112,7 @@
true, metadataPageManagerFactory,
updateAware, new Tracer(LSMBTreeTestContext.class.getSimpleName(),
ITraceCategoryRegistry.CATEGORIES_ALL, new TraceCategoryRegistry()),
- NoOpCompressorDecompressorFactory.INSTANCE, true, null, null);
+ compressorDecompressorFactory, true, null, null);
}
LSMBTreeTestContext testCtx = new LSMBTreeTestContext(fieldSerdes, lsmTree, filtered);
return testCtx;
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
index d64ea30..4d2b086 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
@@ -27,6 +27,7 @@
import java.util.List;
import java.util.Random;
+import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
@@ -50,6 +51,7 @@
import org.apache.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.compression.SnappyCompressorDecompressorFactory;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.apache.hyracks.test.support.TestUtils;
import org.apache.logging.log4j.LogManager;
@@ -71,6 +73,7 @@
protected final int hyracksFrameSize;
protected final double bloomFilterFalsePositiveRate;
protected final int numMutableComponents;
+ private final ICompressorDecompressorFactory compressorDecompressorFactory;
protected IOManager ioManager;
protected int ioDeviceId;
@@ -91,6 +94,10 @@
protected FileReference file;
public LSMBTreeTestHarness() {
+ this(new SnappyCompressorDecompressorFactory());
+ }
+
+ public LSMBTreeTestHarness(ICompressorDecompressorFactory compressorDecompressorFactory) {
this.diskPageSize = AccessMethodTestsConfig.LSM_BTREE_DISK_PAGE_SIZE;
this.diskNumPages = AccessMethodTestsConfig.LSM_BTREE_DISK_NUM_PAGES;
this.diskMaxOpenFiles = AccessMethodTestsConfig.LSM_BTREE_DISK_MAX_OPEN_FILES;
@@ -105,6 +112,7 @@
this.metadataPageManagerFactory = AppendOnlyLinkedMetadataPageManagerFactory.INSTANCE;
this.ioOpCallbackFactory = new CountingIoOperationCallbackFactory();
this.pageWriteCallbackFactory = NoOpPageWriteCallbackFactory.INSTANCE;
+ this.compressorDecompressorFactory = compressorDecompressorFactory;
}
public void setUp() throws HyracksDataException {
@@ -224,4 +232,8 @@
public IMetadataPageManagerFactory getMetadataPageManagerFactory() {
return metadataPageManagerFactory;
}
+
+ public ICompressorDecompressorFactory getCompressorDecompressorFactory() {
+ return compressorDecompressorFactory;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
index c255ee5..9c65703 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
@@ -20,7 +20,7 @@
package org.apache.hyracks.storage.am.lsm.common.component;
import java.io.FilenameFilter;
-import java.util.ArrayList;
+import java.util.List;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
@@ -42,7 +42,7 @@
@Override
protected void cleanupAndGetValidFilesInternal(FilenameFilter filter,
- TreeIndexFactory<? extends ITreeIndex> treeFactory, ArrayList<IndexComponentFileReference> allFiles,
+ TreeIndexFactory<? extends ITreeIndex> treeFactory, List<IndexComponentFileReference> allFiles,
IBufferCache bufferCache) {
String[] files = baseDir.getFile().list(filter);
for (String fileName : files) {