Implemented registration of LSM components via atomic renaming of a temp component. Modified LSMFileNameManager to use IOManager, such that new LSM components are written to IODevices in a round rob in fashion. Added tests using multiple IO devices.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_lsm_tree@1151 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/FileReference.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/FileReference.java
index b1a5008..e6f3481 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/FileReference.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/FileReference.java
@@ -21,19 +21,26 @@
private static final long serialVersionUID = 1L;
private final File file;
+ private final IODeviceHandle dev;
public FileReference(IODeviceHandle dev, String devRelPath) {
file = new File(dev.getPath(), devRelPath);
+ this.dev = dev;
}
public FileReference(File file) {
this.file = file;
+ this.dev = null;
}
public File getFile() {
return file;
}
+ public IODeviceHandle getDevideHandle() {
+ return dev;
+ }
+
@Override
public String toString() {
return file.getAbsolutePath();
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexMultiThreadTest.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexMultiThreadTest.java
index f7beb65..a2d85df 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexMultiThreadTest.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexMultiThreadTest.java
@@ -25,6 +25,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
@@ -48,7 +49,7 @@
protected ArrayList<TestWorkloadConf> workloadConfs = getTestWorkloadConf();
- protected abstract void setUp() throws HyracksDataException;
+ protected abstract void setUp() throws HyracksException;
protected abstract void tearDown() throws HyracksDataException;
@@ -70,7 +71,7 @@
return opProbs;
}
- protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, int numThreads, TestWorkloadConf conf, String dataMsg) throws HyracksDataException, InterruptedException, TreeIndexException {
+ protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, int numThreads, TestWorkloadConf conf, String dataMsg) throws InterruptedException, TreeIndexException, HyracksException {
setUp();
if (LOGGER.isLoggable(Level.INFO)) {
@@ -100,7 +101,7 @@
}
@Test
- public void oneIntKeyAndValue() throws InterruptedException, HyracksDataException, TreeIndexException {
+ public void oneIntKeyAndValue() throws InterruptedException, TreeIndexException, HyracksException {
ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
int numKeys = 1;
String dataMsg = "One Int Key And Value";
@@ -112,7 +113,7 @@
}
@Test
- public void oneStringKeyAndValue() throws InterruptedException, HyracksDataException, TreeIndexException {
+ public void oneStringKeyAndValue() throws InterruptedException, TreeIndexException, HyracksException {
ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE };
int numKeys = 1;
String dataMsg = "One String Key And Value";
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
index f0adb0e..99ebab7 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
@@ -56,7 +56,7 @@
} catch (HyracksDataException e) {
// Revert state of buffer cache since file failed to open.
if (!fileIsMapped) {
- bufferCache.deleteFile(fileId);
+ bufferCache.deleteFile(fileId, false);
}
throw e;
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorNodePushable.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorNodePushable.java
index ec44982..5c96af8 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorNodePushable.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorNodePushable.java
@@ -85,7 +85,7 @@
}
// remove name to id mapping
- bufferCache.deleteFile(indexFileId);
+ bufferCache.deleteFile(indexFileId, false);
}
// TODO: for the time being we don't throw,
// with proper exception handling (no hanging job problem) we should
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index ace9bf3..ba32461 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -60,14 +60,14 @@
private final BTree memBTree;
private final InMemoryFreePageManager memFreePageManager;
- // On-disk components.
+ // On-disk components.
private final ILSMFileManager fileManager;
// For creating BTree's used in flush and merge.
private final BTreeFactory diskBTreeFactory;
// For creating BTree's used in bulk load. Different from diskBTreeFactory
// because it should have a different tuple writer in it's leaf frames.
private final BTreeFactory bulkLoadBTreeFactory;
- private final IBufferCache diskBufferCache;
+ private final IBufferCache diskBufferCache;
private final IFileMapProvider diskFileMapProvider;
// List of BTree instances. Using Object for better sharing via ILSMTree + LSMHarness.
private LinkedList<Object> diskBTrees = new LinkedList<Object>();
@@ -99,19 +99,12 @@
@Override
public void create(int indexFileId) throws HyracksDataException {
memBTree.create(indexFileId);
+ fileManager.createDirs();
}
/**
- * Opens LSMBTree, assuming a consistent state of the disk-resident
- * components. In particular, registers all files in in base dir of
- * fileNameManager as on-disk BTrees.
- *
- * Example pathological scenario to explain "consistent state assumption":
- * Suppose a merge finished, but before the original files were deleted the
- * system crashes. We are left in a state where we have the original BTrees
- * in addition to the merged one. We assume that prior to calling this
- * method a separate recovery process has ensured the consistent of the
- * disk-resident components.
+ * Opens LSMBTree, cleaning up invalid files from base dir, and registering
+ * all valid files as on-disk BTrees.
*
* @param indexFileId
* Dummy file id used for in-memory BTree.
@@ -121,7 +114,7 @@
public void open(int indexFileId) throws HyracksDataException {
memBTree.open(indexFileId);
List<String> validFileNames = fileManager.cleanupAndGetValidFiles();
- for (String fileName : validFileNames) {
+ for (String fileName : validFileNames) {
BTree btree = createDiskBTree(diskBTreeFactory, fileName, false);
diskBTrees.add(btree);
}
@@ -249,7 +242,7 @@
ITreeIndexAccessor memBTreeAccessor = memBTree.createAccessor();
ITreeIndexCursor scanCursor = memBTreeAccessor.createSearchCursor();
memBTreeAccessor.search(scanCursor, nullPred);
- BTree diskBTree = createFlushTargetBTree();
+ BTree diskBTree = createTempBTree(diskBTreeFactory);
// Bulk load the tuples from the in-memory BTree into the new disk BTree.
IIndexBulkLoadContext bulkLoadCtx = diskBTree.beginBulkLoad(1.0f);
try {
@@ -261,6 +254,9 @@
scanCursor.close();
}
diskBTree.endBulkLoad(bulkLoadCtx);
+
+ String finalFileName = fileManager.getFlushFileName();
+ rename(diskBTree, finalFileName);
return diskBTree;
}
@@ -275,33 +271,48 @@
memBTree.create(memBTree.getFileId());
}
- private BTree bulkLoadTargetBTree() throws HyracksDataException {
- // Note that by using a flush target file name, we state that the new
- // bulk loaded tree is "newer" than any other merged tree.
- String fileName = fileManager.getFlushFileName();
- return createDiskBTree(bulkLoadBTreeFactory, fileName, true);
- }
-
- private BTree createFlushTargetBTree() throws HyracksDataException {
- String fileName = fileManager.getFlushFileName();
- return createDiskBTree(diskBTreeFactory, fileName, true);
- }
-
- private BTree createMergeTargetBTree(List<Object> mergingDiskBTrees) throws HyracksDataException {
+ private String getMergeTargetFileName(List<Object> mergingDiskBTrees) throws HyracksDataException {
BTree firstBTree = (BTree) mergingDiskBTrees.get(0);
BTree lastBTree = (BTree) mergingDiskBTrees.get(mergingDiskBTrees.size() - 1);
FileReference firstFile = diskFileMapProvider.lookupFileName(firstBTree.getFileId());
FileReference lastFile = diskFileMapProvider.lookupFileName(lastBTree.getFileId());
String fileName = fileManager.getMergeFileName(firstFile.getFile().getName(), lastFile.getFile().getName());
- return createDiskBTree(diskBTreeFactory, fileName, true);
+ return fileName;
+ }
+
+ private void rename(BTree srcTmpBTree, String dest) throws HyracksDataException {
+ int tmpFileId = srcTmpBTree.getFileId();
+ FileReference srcFileRef = diskFileMapProvider.lookupFileName(tmpFileId);
+ diskBufferCache.closeFile(tmpFileId);
+ diskBufferCache.deleteFile(tmpFileId, true);
+ FileReference destFileRef = fileManager.rename(srcFileRef, dest);
+ // File will be deleted during cleanup of merge().
+ diskBufferCache.createFile(destFileRef);
+ int newFileId = diskFileMapProvider.lookupFileId(destFileRef);
+ diskBufferCache.openFile(newFileId);
+ srcTmpBTree.open(newFileId);
+ }
+
+ private BTree createTempBTree(BTreeFactory factory) throws HyracksDataException {
+ FileReference file = fileManager.createTempFile();
+ // File will be deleted during rename().
+ diskBufferCache.createFile(file);
+ int diskBTreeFileId = diskFileMapProvider.lookupFileId(file);
+ // File will be closed during rename().
+ diskBufferCache.openFile(diskBTreeFileId);
+ // Create new BTree instance.
+ BTree diskBTree = factory.createBTreeInstance(diskBTreeFileId);
+ diskBTree.create(diskBTreeFileId);
+ // BTree will be closed during cleanup of merge().
+ diskBTree.open(diskBTreeFileId);
+ return diskBTree;
}
private BTree createDiskBTree(BTreeFactory factory, String fileName, boolean createBTree) throws HyracksDataException {
- // Register the new BTree file.
- FileReference file = new FileReference(new File(fileName));
+ FileReference fileRef = new FileReference(new File(fileName));
// File will be deleted during cleanup of merge().
- diskBufferCache.createFile(file);
- int diskBTreeFileId = diskFileMapProvider.lookupFileId(file);
+ diskBufferCache.createFile(fileRef);
+ int diskBTreeFileId = diskFileMapProvider.lookupFileId(fileRef);
// File will be closed during cleanup of merge().
diskBufferCache.openFile(diskBTreeFileId);
// Create new BTree instance.
@@ -363,7 +374,8 @@
}
// Bulk load the tuples from all on-disk BTrees into the new BTree.
- BTree mergedBTree = createMergeTargetBTree(mergedComponents);
+ //BTree mergedBTree = createMergeTargetBTree(mergedComponents);
+ BTree mergedBTree = createTempBTree(diskBTreeFactory);
IIndexBulkLoadContext bulkLoadCtx = mergedBTree.beginBulkLoad(1.0f);
try {
while (cursor.hasNext()) {
@@ -375,6 +387,9 @@
cursor.close();
}
mergedBTree.endBulkLoad(bulkLoadCtx);
+
+ String finalFileName = getMergeTargetFileName(mergingDiskBTrees);
+ rename(mergedBTree, finalFileName);
return mergedBTree;
}
@@ -428,7 +443,7 @@
@Override
public IIndexBulkLoadContext beginBulkLoad(float fillFactor) throws TreeIndexException, HyracksDataException {
- BTree diskBTree = bulkLoadTargetBTree();
+ BTree diskBTree = createTempBTree(bulkLoadBTreeFactory);
LSMTreeBulkLoadContext bulkLoadCtx = new LSMTreeBulkLoadContext(diskBTree);
bulkLoadCtx.beginBulkLoad(fillFactor);
return bulkLoadCtx;
@@ -443,7 +458,9 @@
@Override
public void endBulkLoad(IIndexBulkLoadContext ictx) throws HyracksDataException {
LSMTreeBulkLoadContext bulkLoadCtx = (LSMTreeBulkLoadContext) ictx;
- bulkLoadCtx.getBTree().endBulkLoad(bulkLoadCtx.getBulkLoadCtx());
+ bulkLoadCtx.getBTree().endBulkLoad(bulkLoadCtx.getBulkLoadCtx());
+ String finalFileName = fileManager.getFlushFileName();
+ rename(bulkLoadCtx.getBTree(), finalFileName);
lsmHarness.addBulkLoadedComponent(bulkLoadCtx.getBTree());
}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
index a440f04..1758a19 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
@@ -17,6 +17,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
@@ -36,7 +37,7 @@
public class LSMBTreeUtils {
public static LSMBTree createLSMTree(InMemoryBufferCache memBufferCache,
- InMemoryFreePageManager memFreePageManager, String onDiskDir, IBufferCache diskBufferCache,
+ InMemoryFreePageManager memFreePageManager, IOManager ioManager, String onDiskDir, IBufferCache diskBufferCache,
IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories) {
LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
cmpFactories.length, false);
@@ -55,7 +56,7 @@
typeTraits.length, interiorFrameFactory, copyTupleLeafFrameFactory);
BTreeFactory bulkLoadBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, cmpFactories,
typeTraits.length, interiorFrameFactory, insertLeafFrameFactory);
- ILSMFileManager fileNameManager = new LSMTreeFileManager(onDiskDir);
+ ILSMFileManager fileNameManager = new LSMTreeFileManager(ioManager, onDiskDir);
LSMBTree lsmTree = new LSMBTree(memBufferCache, memFreePageManager, interiorFrameFactory,
insertLeafFrameFactory, deleteLeafFrameFactory, fileNameManager, diskBTreeFactory,
bulkLoadBTreeFactory, diskFileMapProvider, typeTraits.length, cmpFactories);
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFileManager.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFileManager.java
index 752d2bd..982749d 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFileManager.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFileManager.java
@@ -19,6 +19,8 @@
import java.util.List;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.control.nc.io.IOManager;
/**
* Provides file names for LSM on-disk components. Also cleans up invalid files.
@@ -31,15 +33,24 @@
*
*/
public interface ILSMFileManager {
- public String getFlushFileName();
+ public void createDirs();
+
+ public String getFlushFileName();
public String getMergeFileName(String firstFileName, String lastFileName) throws HyracksDataException;
public String getBaseDir();
+ public FileReference createTempFile() throws HyracksDataException;
+
+ // Atomically renames src file ref to dest on same IODevice as src, and returns file ref of dest.
+ public FileReference rename(FileReference src, String dest) throws HyracksDataException;
+
// Deletes invalid files, and returns list of valid files from baseDir.
// The returned valid files are correctly sorted (based on the recency of data).
public List<String> cleanupAndGetValidFiles() throws HyracksDataException;
public Comparator<String> getFileNameComparator();
+
+ public IOManager getIOManager();
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryBufferCache.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryBufferCache.java
index 8fcb4a0..1e2da77 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryBufferCache.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryBufferCache.java
@@ -107,7 +107,7 @@
}
@Override
- public void deleteFile(int fileId) throws HyracksDataException {
+ public void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException {
// Do nothing.
}
@@ -120,7 +120,7 @@
public void close() {
// Do nothing.
}
-
+
public class CachedPage implements ICachedPageInternal {
private final int cpid;
private final ByteBuffer buffer;
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java
index e44e6e5..ab88b45 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java
@@ -17,6 +17,9 @@
import java.io.File;
import java.io.FilenameFilter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
import java.text.Format;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@@ -26,22 +29,57 @@
import java.util.List;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.io.IODeviceHandle;
+import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFileManager;
public class LSMTreeFileManager implements ILSMFileManager {
private static final String SPLIT_STRING = "_";
+ private static final String TEMP_FILE_PREFIX = "lsm_tree";
+ // Currently uses all IODevices registered in ioManager in a round-robin fashion.
+ private final IOManager ioManager;
+ // baseDir should reflect dataset name, and partition name.
private final String baseDir;
- private final Format formatter = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS");
+ private final Format formatter = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS");
private final Comparator<String> cmp = new FileNameComparator();
- private final Comparator<String[]> intervalCmp = new IntervalComparator();
+ private final Comparator<ComparableFileName> recencyCmp = new RecencyComparator();
- public LSMTreeFileManager(String baseDir) {
+ public LSMTreeFileManager(IOManager ioManager, String baseDir) {
if (!baseDir.endsWith(System.getProperty("file.separator"))) {
baseDir += System.getProperty("file.separator");
}
+ this.ioManager = ioManager;
this.baseDir = baseDir;
+ createDirs();
+ }
+
+ @Override
+ public void createDirs() {
+ for(IODeviceHandle dev : ioManager.getIODevices()) {
+ File f = new File(dev.getPath(), baseDir);
+ f.mkdirs();
+ }
+ }
+
+ @Override
+ public FileReference createTempFile() throws HyracksDataException {
+ // Cycles through the IODevices in round-robin fashion.
+ return ioManager.createWorkspaceFile(TEMP_FILE_PREFIX);
+ }
+
+ // Atomically renames src fileref to dest on same IODevice as src, and returns file ref of dest.
+ @Override
+ public FileReference rename(FileReference src, String dest) throws HyracksDataException {
+ FileReference destFile = new FileReference(src.getDevideHandle(), dest);
+ try {
+ Files.move(src.getFile().toPath(), destFile.getFile().toPath(), StandardCopyOption.ATOMIC_MOVE);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ return destFile;
}
@Override
@@ -83,17 +121,6 @@
}
}
- private class IntervalComparator implements Comparator<String[]> {
- @Override
- public int compare(String[] a, String[] b) {
- int startCmp = a[0].compareTo(b[0]);
- if (startCmp != 0) {
- return startCmp;
- }
- return b[1].compareTo(a[1]);
- }
- }
-
@Override
public String getBaseDir() {
return baseDir;
@@ -101,42 +128,46 @@
@Override
public List<String> cleanupAndGetValidFiles() throws HyracksDataException {
- List<String> validFiles = new ArrayList<String>();
- File dir = new File(baseDir);
- FilenameFilter filter = new FilenameFilter() {
- public boolean accept(File dir, String name) {
- return !name.startsWith(".");
+ List<String> validFiles = new ArrayList<String>();
+ ArrayList<ComparableFileName> allFiles = new ArrayList<ComparableFileName>();
+ // Gather files from all IODeviceHandles.
+ for(IODeviceHandle dev : ioManager.getIODevices()) {
+ File dir = new File(dev.getPath(), baseDir);
+ FilenameFilter filter = new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return !name.startsWith(".");
+ }
+ };
+ String[] files = dir.list(filter);
+ for (String file : files) {
+ allFiles.add(new ComparableFileName(dir.getPath() + File.separator + file));
}
- };
- String[] files = dir.list(filter);
+ }
// Trivial cases.
- if (files == null) {
+ if (allFiles.isEmpty()) {
return validFiles;
}
- if (files.length == 1) {
- validFiles.add(files[0]);
+ if (allFiles.size() == 1) {
+ validFiles.add(allFiles.get(0).fullPath);
return validFiles;
}
- List<String[]> intervals = new ArrayList<String[]>();
- for (String fileName : files) {
- intervals.add(fileName.split(SPLIT_STRING));
- }
- // Sorts files from earliest to latest timestamp.
- Collections.sort(intervals, intervalCmp);
+ // Sorts files names from earliest to latest timestamp.
+ Collections.sort(allFiles);
- String[] lastInterval = intervals.get(0);
- validFiles.add(getFileNameFromInterval(intervals.get(0)));
- for (int i = 1; i < intervals.size(); i++) {
- String[] currentInterval = intervals.get(i);
+ List<ComparableFileName> validComparableFiles = new ArrayList<ComparableFileName>();
+ ComparableFileName last = allFiles.get(0);
+ validComparableFiles.add(last);
+ for (int i = 1; i < allFiles.size(); i++) {
+ ComparableFileName current = allFiles.get(i);
// Current start timestamp is greater than last stop timestamp.
- if (currentInterval[0].compareTo(lastInterval[1]) > 0) {
- validFiles.add(getFileNameFromInterval(currentInterval));
- lastInterval = currentInterval;
- } else if (currentInterval[0].compareTo(lastInterval[0]) >= 0
- && currentInterval[1].compareTo(lastInterval[1]) <= 0) {
+ if (current.interval[0].compareTo(last.interval[1]) > 0) {
+ validComparableFiles.add(current);
+ last = current;
+ } else if (current.interval[0].compareTo(last.interval[0]) >= 0
+ && current.interval[1].compareTo(last.interval[1]) <= 0) {
// Invalid files are completely contained in last interval.
- File invalidFile = new File(getFileNameFromInterval(currentInterval));
+ File invalidFile = new File(current.fullPath);
invalidFile.delete();
} else {
// This scenario should not be possible.
@@ -144,11 +175,47 @@
}
}
// Sort valid files in reverse lexicographical order, such that newer files come first.
- Collections.sort(validFiles, cmp);
+ Collections.sort(validComparableFiles, recencyCmp);
+ for (ComparableFileName cmpFileName : validComparableFiles) {
+ validFiles.add(cmpFileName.fullPath);
+ }
return validFiles;
}
- private String getFileNameFromInterval(String[] interval) {
- return baseDir + interval[0] + SPLIT_STRING + interval[1];
+ private class ComparableFileName implements Comparable<ComparableFileName> {
+ public final String fullPath;
+ // Timestamp interval.
+ public final String[] interval;
+
+ public ComparableFileName(String fullPath) {
+ this.fullPath = fullPath;
+ File f = new File(fullPath);
+ interval = f.getName().split(SPLIT_STRING);
+ }
+
+ @Override
+ public int compareTo(ComparableFileName b) {
+ int startCmp = interval[0].compareTo(b.interval[0]);
+ if (startCmp != 0) {
+ return startCmp;
+ }
+ return b.interval[1].compareTo(interval[1]);
+ }
+ }
+
+ private class RecencyComparator implements Comparator<ComparableFileName> {
+ @Override
+ public int compare(ComparableFileName a, ComparableFileName b) {
+ int cmp = -a.interval[0].compareTo(b.interval[0]);
+ if (cmp != 0) {
+ return cmp;
+ }
+ return -a.interval[1].compareTo(b.interval[1]);
+ }
+ }
+
+ @Override
+ public IOManager getIOManager() {
+ return ioManager;
}
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
index 4ec80e0..eed2d8b 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
@@ -17,6 +17,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
@@ -39,7 +40,7 @@
public class LSMRTreeUtils {
public static LSMRTree createLSMTree(InMemoryBufferCache memBufferCache,
- InMemoryFreePageManager memFreePageManager, String onDiskDir, IBufferCache diskBufferCache,
+ InMemoryFreePageManager memFreePageManager, IOManager ioManager, String onDiskDir, IBufferCache diskBufferCache,
IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories) {
@@ -63,7 +64,7 @@
BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, btreeCmpFactories,
typeTraits.length, btreeInteriorFrameFactory, btreeLeafFrameFactory);
- ILSMFileManager fileNameManager = new LSMTreeFileManager(onDiskDir);
+ ILSMFileManager fileNameManager = new LSMTreeFileManager(ioManager, onDiskDir);
LSMRTree lsmTree = new LSMRTree(memBufferCache, memFreePageManager, rtreeInteriorFrameFactory,
rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager,
diskRTreeFactory, diskBTreeFactory, diskFileMapProvider, typeTraits.length, rtreeCmpFactories,
diff --git a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/BufferCache.java b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/BufferCache.java
index 58d731c..f724d06 100644
--- a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/BufferCache.java
+++ b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/BufferCache.java
@@ -726,10 +726,15 @@
}
@Override
- public synchronized void deleteFile(int fileId) throws HyracksDataException {
+ public synchronized void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Deleting file: " + fileId + " in cache: " + this);
}
+ if (flushDirtyPages) {
+ synchronized (fileInfoMap) {
+ sweepAndFlush(fileId, flushDirtyPages);
+ }
+ }
synchronized (fileInfoMap) {
BufferedFileHandle fInfo = null;
try {
diff --git a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/DebugBufferCache.java b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/DebugBufferCache.java
index dc00df0..40afc95 100644
--- a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/DebugBufferCache.java
+++ b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/DebugBufferCache.java
@@ -65,8 +65,8 @@
}
@Override
- public void deleteFile(int fileId) throws HyracksDataException {
- bufferCache.deleteFile(fileId);
+ public void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException {
+ bufferCache.deleteFile(fileId, flushDirtyPages);
deleteFileCount.addAndGet(1);
}
diff --git a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/IBufferCache.java b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/IBufferCache.java
index e82c01b..ab0f03c 100644
--- a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/IBufferCache.java
+++ b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/IBufferCache.java
@@ -24,8 +24,8 @@
public void closeFile(int fileId) throws HyracksDataException;
- public void deleteFile(int fileId) throws HyracksDataException;
-
+ public void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException;
+
public ICachedPage tryPin(long dpid) throws HyracksDataException;
public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException;
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerComponentHolder.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerComponentHolder.java
index d006f42..ccbe862 100644
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerComponentHolder.java
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerComponentHolder.java
@@ -14,7 +14,15 @@
*/
package edu.uci.ics.hyracks.test.support;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executors;
+
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.io.IODeviceHandle;
+import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexRegistry;
import edu.uci.ics.hyracks.storage.common.buffercache.BufferCache;
@@ -31,7 +39,8 @@
private static IBufferCache bufferCache;
private static IFileMapProvider fileMapProvider;
private static IndexRegistry<IIndex> indexRegistry;
-
+ private static IOManager ioManager;
+
private static int pageSize;
private static int numPages;
private static int maxOpenFiles;
@@ -69,4 +78,13 @@
}
return indexRegistry;
}
+
+ public synchronized static IOManager getIOManager() throws HyracksException {
+ if (ioManager == null) {
+ List<IODeviceHandle> devices = new ArrayList<IODeviceHandle>();
+ devices.add(new IODeviceHandle(new File(System.getProperty("java.io.tmpdir")), "iodev_test_wa"));
+ ioManager = new IOManager(devices, Executors.newCachedThreadPool());
+ }
+ return ioManager;
+ }
}
\ No newline at end of file
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
index 396dddc..5dee899 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexBulkLoadTest;
import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexTestContext;
@@ -38,7 +39,7 @@
private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
@@ -51,7 +52,7 @@
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
numKeys, harness.getFileId());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
index 1bc7530..2c1190a 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexDeleteTest;
import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexTestContext;
@@ -38,7 +39,7 @@
private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
@@ -51,7 +52,7 @@
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
numKeys, harness.getFileId());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
index fb1b227..0e34a0f3 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
@@ -21,6 +21,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexExamplesTest;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
@@ -34,7 +35,7 @@
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] cmpFactories) throws TreeIndexException {
return LSMBTreeUtils.createLSMTree(harness.getMemBufferCache(),
- harness.getMemFreePageManager(), harness.getOnDiskDir(),
+ harness.getMemFreePageManager(), harness.getIOManager(), harness.getOnDiskDir(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
typeTraits, cmpFactories);
}
@@ -45,7 +46,7 @@
}
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
index b292b26..68560de 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexInsertTest;
import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexTestContext;
@@ -38,7 +39,7 @@
private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
@@ -51,7 +52,7 @@
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
numKeys, harness.getFileId());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
index fb10d4f..2c4e3dc 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexTestContext;
import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext;
@@ -37,7 +38,7 @@
private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
@@ -50,7 +51,7 @@
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
numKeys, harness.getFileId());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
index a93308f..f7c878f 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexBulkLoadTest;
import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexTestContext;
@@ -38,7 +39,7 @@
private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
@@ -51,7 +52,7 @@
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
numKeys, harness.getFileId());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
index 2269865..c325025 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexTestContext;
import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexUpdateTest;
@@ -38,7 +39,7 @@
private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
@@ -51,7 +52,7 @@
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
numKeys, harness.getFileId());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
index 1267510..049d6dc 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
@@ -20,6 +20,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexMultiThreadTest;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
@@ -36,7 +37,7 @@
private LSMBTreeTestWorkerFactory workerFactory = new LSMBTreeTestWorkerFactory();
@Override
- protected void setUp() throws HyracksDataException {
+ protected void setUp() throws HyracksException {
harness.setUp();
}
@@ -48,7 +49,7 @@
@Override
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories) throws TreeIndexException {
return LSMBTreeUtils.createLSMTree(harness.getMemBufferCache(),
- harness.getMemFreePageManager(), harness.getOnDiskDir(),
+ harness.getMemFreePageManager(), harness.getIOManager(), harness.getOnDiskDir(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
typeTraits, cmpFactories);
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
index 2cbee2f..275621f 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
@@ -21,7 +21,8 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
@@ -44,11 +45,11 @@
private static final int HYRACKS_FRAME_SIZE = 128;
protected IHyracksTaskContext ctx;
+ protected IOManager ioManager;
protected IBufferCache bufferCache;
protected int lsmtreeFileId;
protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
- protected final static String tmpDir = System.getProperty("java.io.tmpdir");
protected final static String sep = System.getProperty("file.separator");
protected final static String classDir = "/lsmtree/";
protected String onDiskDir;
@@ -59,23 +60,24 @@
private final int onDiskPageSize;
private final int onDiskNumPages;
- public LSMTreeRunner(int numBatches, int inMemPageSize, int inMemNumPages, int onDiskPageSize, int onDiskNumPages, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories) throws HyracksDataException, BTreeException {
+ public LSMTreeRunner(int numBatches, int inMemPageSize, int inMemNumPages, int onDiskPageSize, int onDiskNumPages, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories) throws BTreeException, HyracksException {
this.numBatches = numBatches;
this.onDiskPageSize = onDiskPageSize;
this.onDiskNumPages = onDiskNumPages;
- onDiskDir = tmpDir + classDir + sep + simpleDateFormat.format(new Date()) + sep;
+ onDiskDir = classDir + sep + simpleDateFormat.format(new Date()) + sep;
ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
TestStorageManagerComponentHolder.init(this.onDiskPageSize, this.onDiskNumPages, MAX_OPEN_FILES);
bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ ioManager = TestStorageManagerComponentHolder.getIOManager();
IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
InMemoryBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), inMemPageSize, inMemNumPages);
InMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(inMemNumPages, new LIFOMetaDataFrameFactory());
- lsmtree = LSMBTreeUtils.createLSMTree(memBufferCache, memFreePageManager, onDiskDir, bufferCache, fmp, typeTraits, cmpFactories);
+ lsmtree = LSMBTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ioManager, onDiskDir, bufferCache, fmp, typeTraits, cmpFactories);
}
@Override
public void init() throws Exception {
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
index 6daa36e..c8023b0 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
@@ -20,6 +20,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexTestContext;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
@@ -61,12 +62,12 @@
}
public static LSMBTreeTestContext create(InMemoryBufferCache memBufferCache,
- InMemoryFreePageManager memFreePageManager, String onDiskDir, IBufferCache diskBufferCache,
+ InMemoryFreePageManager memFreePageManager, IOManager ioManager, String onDiskDir, IBufferCache diskBufferCache,
IFileMapProvider diskFileMapProvider, ISerializerDeserializer[] fieldSerdes, int numKeyFields, int fileId)
throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
- LSMBTree lsmTree = LSMBTreeUtils.createLSMTree(memBufferCache, memFreePageManager, onDiskDir, diskBufferCache,
+ LSMBTree lsmTree = LSMBTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ioManager, onDiskDir, diskBufferCache,
diskFileMapProvider, typeTraits, cmpFactories);
lsmTree.create(fileId);
lsmTree.open(fileId);
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
index 531592d..ba9025f 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
@@ -23,6 +23,8 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
@@ -54,6 +56,7 @@
protected final int memNumPages;
protected final int hyracksFrameSize;
+ protected IOManager ioManager;
protected IBufferCache diskBufferCache;
protected IFileMapProvider diskFileMapProvider;
protected InMemoryBufferCache memBufferCache;
@@ -62,8 +65,7 @@
protected final Random rnd = new Random();
protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
- protected final static String tmpDir = System.getProperty("java.io.tmpdir");
- protected final static String sep = System.getProperty("file.separator");
+ protected final static String sep = System.getProperty("file.separator");
protected String onDiskDir;
public LSMBTreeTestHarness() {
@@ -86,14 +88,15 @@
this.hyracksFrameSize = hyracksFrameSize;
}
- public void setUp() throws HyracksDataException {
- onDiskDir = tmpDir + sep + "lsm_btree_" + simpleDateFormat.format(new Date()) + sep;
+ public void setUp() throws HyracksException {
+ onDiskDir = "lsm_btree_" + simpleDateFormat.format(new Date()) + sep;
ctx = TestUtils.create(getHyracksFrameSize());
TestStorageManagerComponentHolder.init(diskPageSize, diskNumPages, diskMaxOpenFiles);
diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
diskFileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), memPageSize, memNumPages);
memFreePageManager = new InMemoryFreePageManager(memNumPages, new LIFOMetaDataFrameFactory());
+ ioManager = TestStorageManagerComponentHolder.getIOManager();
rnd.setSeed(RANDOM_SEED);
}
@@ -132,6 +135,10 @@
return DUMMY_FILE_ID;
}
+ public IOManager getIOManager() {
+ return ioManager;
+ }
+
public IBufferCache getDiskBufferCache() {
return diskBufferCache;
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/LSMTreeFileManagerTest.java b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/LSMTreeFileManagerTest.java
index f2964c9..162cd93 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/LSMTreeFileManagerTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/LSMTreeFileManagerTest.java
@@ -22,56 +22,65 @@
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
+import java.util.concurrent.Executors;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.io.IODeviceHandle;
+import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeFileManager;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
public class LSMTreeFileManagerTest {
+ private static final int DEFAULT_PAGE_SIZE = 256;
+ private static final int DEFAULT_NUM_PAGES = 100;
+ private static final int DEFAULT_MAX_OPEN_FILES = 10;
protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
- protected final static String tmpDir = System.getProperty("java.io.tmpdir");
protected final static String sep = System.getProperty("file.separator");
- protected String baseDir;
-
+ protected IOManager ioManager;
+ protected String baseDir;
+
@Before
- public void setUp() throws HyracksDataException {
- baseDir = tmpDir + sep + "lsm_tree" + simpleDateFormat.format(new Date()) + sep;
+ public void setUp() throws HyracksException {
+ TestStorageManagerComponentHolder.init(DEFAULT_PAGE_SIZE, DEFAULT_NUM_PAGES, DEFAULT_MAX_OPEN_FILES);
+ ioManager = TestStorageManagerComponentHolder.getIOManager();
+ baseDir = "lsm_tree" + simpleDateFormat.format(new Date()) + sep;
File f = new File(baseDir);
f.mkdirs();
}
@After
public void tearDown() throws HyracksDataException {
- baseDir = tmpDir + sep + "lsm_tree" + simpleDateFormat.format(new Date()) + sep;
File f = new File(baseDir);
f.deleteOnExit();
}
public void sortOrderTest(boolean testFlushFileName) throws InterruptedException, HyracksDataException {
- ILSMFileManager fileNameManager = new LSMTreeFileManager(baseDir);
+ ILSMFileManager fileManager = new LSMTreeFileManager(ioManager, baseDir);
LinkedList<String> fileNames = new LinkedList<String>();
int numFileNames = 100;
long sleepTime = 5;
for (int i = 0; i < numFileNames; i++) {
- String flushFileName = fileNameManager.getFlushFileName();
+ String flushFileName = fileManager.getFlushFileName();
if (testFlushFileName) {
fileNames.addFirst(flushFileName);
}
Thread.sleep(sleepTime);
if (!testFlushFileName) {
- String secondFlushFileName = fileNameManager.getFlushFileName();
- String mergeFileName = getMergeFileName(fileNameManager, flushFileName, secondFlushFileName);
+ String secondFlushFileName = fileManager.getFlushFileName();
+ String mergeFileName = getMergeFileName(fileManager, flushFileName, secondFlushFileName);
fileNames.addFirst(mergeFileName);
Thread.sleep(sleepTime);
}
@@ -82,112 +91,169 @@
// Make sure the comparator sorts in the correct order (i.e., the
// reverse insertion order in this case).
- Comparator<String> cmp = fileNameManager.getFileNameComparator();
+ Comparator<String> cmp = fileManager.getFileNameComparator();
Collections.sort(sortedFileNames, cmp);
for (int i = 0; i < numFileNames; i++) {
assertEquals(fileNames.get(i), sortedFileNames.get(i));
}
}
- //@Test
+ @Test
public void flushAndMergeFilesSortOrderTest() throws InterruptedException, HyracksDataException {
sortOrderTest(true);
sortOrderTest(false);
}
- @Test
- public void cleanInvalidFilesTest() throws InterruptedException, IOException {
- ILSMFileManager fileNameManager = new LSMTreeFileManager(baseDir);
- List<String> flushFileNames = new ArrayList<String>();
- List<String> allFiles = new ArrayList<String>();
+ public void cleanInvalidFilesTest(IOManager ioManager) throws InterruptedException, IOException {
+ ILSMFileManager fileManager = new LSMTreeFileManager(ioManager, baseDir);
+ fileManager.createDirs();
+
+ List<FileReference> flushFiles = new ArrayList<FileReference>();
+ List<FileReference> allFiles = new ArrayList<FileReference>();
int numFileNames = 100;
long sleepTime = 5;
// Generate a bunch of flush files.
for (int i = 0; i < numFileNames; i++) {
- String flushFileName = fileNameManager.getFlushFileName();
- flushFileNames.add(flushFileName);
- Thread.sleep(sleepTime);
+ FileReference flushTempFile = fileManager.createTempFile();
+ String flushFileName = fileManager.getFlushFileName();
+ FileReference flushFile = fileManager.rename(flushTempFile, flushFileName);
+ flushFiles.add(flushFile);
+ Thread.sleep(sleepTime);
}
- allFiles.addAll(flushFileNames);
+ allFiles.addAll(flushFiles);
// Simulate merging some of the flush files.
// Merge range 0 to 4.
- String mergeFile1 = getMergeFileName(fileNameManager, flushFileNames.get(0), flushFileNames.get(4));
+ FileReference mergeFile1 = simulateMerge(fileManager, flushFiles.get(0), flushFiles.get(4));
allFiles.add(mergeFile1);
// Merge range 5 to 9.
- String mergeFile2 = getMergeFileName(fileNameManager, flushFileNames.get(5), flushFileNames.get(9));
+ FileReference mergeFile2 = simulateMerge(fileManager, flushFiles.get(5), flushFiles.get(9));
allFiles.add(mergeFile2);
// Merge range 10 to 19.
- String mergeFile3 = getMergeFileName(fileNameManager, flushFileNames.get(10), flushFileNames.get(19));
+ FileReference mergeFile3 = simulateMerge(fileManager, flushFiles.get(10), flushFiles.get(19));
allFiles.add(mergeFile3);
// Merge range 20 to 29.
- String mergeFile4 = getMergeFileName(fileNameManager, flushFileNames.get(20), flushFileNames.get(29));
+ FileReference mergeFile4 = simulateMerge(fileManager, flushFiles.get(20), flushFiles.get(29));
allFiles.add(mergeFile4);
// Merge range 50 to 79.
- String mergeFile5 = getMergeFileName(fileNameManager, flushFileNames.get(50), flushFileNames.get(79));
+ FileReference mergeFile5 = simulateMerge(fileManager, flushFiles.get(50), flushFiles.get(79));
allFiles.add(mergeFile5);
// Simulate merging of merge files.
- String mergeFile6 = getMergeFileName(fileNameManager, mergeFile1, mergeFile2);
+ FileReference mergeFile6 = simulateMerge(fileManager, mergeFile1, mergeFile2);
allFiles.add(mergeFile6);
- String mergeFile7 = getMergeFileName(fileNameManager, mergeFile3, mergeFile4);
+ FileReference mergeFile7 = simulateMerge(fileManager, mergeFile3, mergeFile4);
allFiles.add(mergeFile7);
- // Create all files.
- for (String fileName : allFiles) {
- File f = new File(fileName);
- f.createNewFile();
- f.deleteOnExit();
+ // Set delete on exit for all files.
+ for (FileReference fileRef : allFiles) {
+ fileRef.getFile().deleteOnExit();
}
// Populate expected valid flush files.
List<String> expectedValidFiles = new ArrayList<String>();
for (int i = 30; i < 50; i++) {
- expectedValidFiles.add(flushFileNames.get(i));
+ expectedValidFiles.add(flushFiles.get(i).getFile().getName());
}
for (int i = 80; i < 100; i++) {
- expectedValidFiles.add(flushFileNames.get(i));
+ expectedValidFiles.add(flushFiles.get(i).getFile().getName());
}
// Populate expected valid merge files.
- expectedValidFiles.add(mergeFile5);
- expectedValidFiles.add(mergeFile6);
- expectedValidFiles.add(mergeFile7);
+ expectedValidFiles.add(mergeFile5.getFile().getName());
+ expectedValidFiles.add(mergeFile6.getFile().getName());
+ expectedValidFiles.add(mergeFile7.getFile().getName());
// Sort expected files.
- Collections.sort(expectedValidFiles, fileNameManager.getFileNameComparator());
+ Collections.sort(expectedValidFiles, fileManager.getFileNameComparator());
- List<String> validFiles = fileNameManager.cleanupAndGetValidFiles();
+ List<String> validFiles = fileManager.cleanupAndGetValidFiles();
// Check actual files against expected files.
assertEquals(expectedValidFiles.size(), validFiles.size());
for (int i = 0; i < expectedValidFiles.size(); i++) {
- assertEquals(expectedValidFiles.get(i), validFiles.get(i));
+ File f = new File(validFiles.get(i));
+ assertEquals(expectedValidFiles.get(i), f.getName());
}
- // Make sure invalid files were removed from baseDir.
- File dir = new File(baseDir);
- FilenameFilter filter = new FilenameFilter() {
- public boolean accept(File dir, String name) {
- return !name.startsWith(".");
+ // Make sure invalid files were removed from all IODevices.
+ ArrayList<String> remainingFiles = new ArrayList<String>();
+ for(IODeviceHandle dev : ioManager.getIODevices()) {
+ File dir = new File(dev.getPath(), baseDir);
+ FilenameFilter filter = new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return !name.startsWith(".");
+ }
+ };
+ String[] files = dir.list(filter);
+ for (String file : files) {
+ File f = new File(file);
+ remainingFiles.add(f.getName());
}
- };
- String[] files = dir.list(filter);
- Arrays.sort(files, fileNameManager.getFileNameComparator());
- // Check actual files in directory against expected files.
- assertEquals(expectedValidFiles.size(), files.length);
- for (int i = 0; i < expectedValidFiles.size(); i++) {
- assertEquals(expectedValidFiles.get(i), baseDir + files[i]);
}
- // Cleanup.
- for (String fileName : files) {
- File f = new File(fileName);
- f.deleteOnExit();
+ Collections.sort(remainingFiles, fileManager.getFileNameComparator());
+ // Check actual files in directory against expected files.
+ assertEquals(expectedValidFiles.size(), remainingFiles.size());
+ for (int i = 0; i < expectedValidFiles.size(); i++) {
+ assertEquals(expectedValidFiles.get(i), remainingFiles.get(i));
}
}
+ @Test
+ public void singleIODeviceTest() throws InterruptedException, IOException {
+ IOManager singleDeviceIOManager = createIOManager(1);
+ cleanInvalidFilesTest(singleDeviceIOManager);
+ cleanDirs(singleDeviceIOManager);
+ }
+
+ @Test
+ public void twoIODevicesTest() throws InterruptedException, IOException {
+ IOManager twoDevicesIOManager = createIOManager(2);
+ cleanInvalidFilesTest(twoDevicesIOManager);
+ cleanDirs(twoDevicesIOManager);
+ }
+
+ @Test
+ public void fourIODevicesTest() throws InterruptedException, IOException {
+ IOManager fourDevicesIOManager = createIOManager(4);
+ cleanInvalidFilesTest(fourDevicesIOManager);
+ cleanDirs(fourDevicesIOManager);
+ }
+
+ private void cleanDirs(IOManager ioManager) {
+ for(IODeviceHandle dev : ioManager.getIODevices()) {
+ File dir = new File(dev.getPath(), baseDir);
+ FilenameFilter filter = new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return !name.startsWith(".");
+ }
+ };
+ String[] files = dir.list(filter);
+ for (String file : files) {
+ File f = new File(file);
+ f.delete();
+ }
+ }
+ }
+
+ private IOManager createIOManager(int numDevices) throws HyracksException {
+ List<IODeviceHandle> devices = new ArrayList<IODeviceHandle>();
+ for (int i = 0; i < numDevices; i++) {
+ String iodevPath = System.getProperty("java.io.tmpdir") + sep + "test_iodev" + i;
+ devices.add(new IODeviceHandle(new File(iodevPath), "wa"));
+ }
+ return new IOManager(devices, Executors.newCachedThreadPool());
+ }
+
+ private FileReference simulateMerge(ILSMFileManager fileManager, FileReference a, FileReference b) throws HyracksDataException {
+ FileReference tempMergeFile = fileManager.createTempFile();
+ String mergeFileName = fileManager.getMergeFileName(a.getFile().getName(), b.getFile().getName());
+ FileReference mergeFile = fileManager.rename(tempMergeFile, mergeFileName);
+ return mergeFile;
+ }
+
private String getMergeFileName(ILSMFileManager fileNameManager, String firstFile, String lastFile) throws HyracksDataException {
File f1 = new File(firstFile);
File f2 = new File(lastFile);
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/AbstractLSMRTreeTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/AbstractLSMRTreeTest.java
index f8bc3e4..0c80f13 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/AbstractLSMRTreeTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/AbstractLSMRTreeTest.java
@@ -26,6 +26,8 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
@@ -56,6 +58,7 @@
protected final int memNumPages;
protected final int hyracksFrameSize;
+ protected IOManager ioManager;
protected IBufferCache diskBufferCache;
protected IFileMapProvider diskFileMapProvider;
protected InMemoryBufferCache memBufferCache;
@@ -64,7 +67,6 @@
protected final Random rnd = new Random();
protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
- protected final static String tmpDir = System.getProperty("java.io.tmpdir");
protected final static String sep = System.getProperty("file.separator");
protected String onDiskDir;
@@ -88,14 +90,15 @@
}
@Before
- public void setUp() throws HyracksDataException {
- onDiskDir = tmpDir + sep + "lsm_rtree_" + simpleDateFormat.format(new Date()) + sep;
+ public void setUp() throws HyracksException {
+ onDiskDir = "lsm_rtree_" + simpleDateFormat.format(new Date()) + sep;
ctx = TestUtils.create(getHyracksFrameSize());
TestStorageManagerComponentHolder.init(diskPageSize, diskNumPages, diskMaxOpenFiles);
diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
diskFileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
memBufferCache = new LSMRTreeInMemoryBufferCache(new HeapBufferAllocator(), getMemPageSize(), getMemNumPages());
memFreePageManager = new LSMRTreeInMemoryFreePageManager(memNumPages, new LIFOMetaDataFrameFactory());
+ ioManager = TestStorageManagerComponentHolder.getIOManager();
rnd.setSeed(RANDOM_SEED);
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
index 96ff631..c372e66 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestContext;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestHarness;
@@ -38,7 +39,7 @@
private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
@@ -51,7 +52,7 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys) throws Exception {
return LSMRTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
valueProviderFactories, numKeys, harness.getFileId());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
index 4b44895..3fff66f 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestContext;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestHarness;
@@ -34,7 +35,7 @@
private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
@@ -47,7 +48,7 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys) throws Exception {
return LSMRTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
valueProviderFactories, numKeys, harness.getFileId());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
index 4bb5ab9..9860315 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
@@ -21,6 +21,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
@@ -36,7 +37,7 @@
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories)
throws TreeIndexException {
return LSMRTreeUtils.createLSMTree(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits,
+ harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits,
rtreeCmpFactories, btreeCmpFactories, valueProviderFactories);
}
@@ -46,7 +47,7 @@
}
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
index 887bde7..26c974e 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestContext;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestHarness;
@@ -34,7 +35,7 @@
private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
@@ -47,7 +48,7 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys) throws Exception {
return LSMRTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
valueProviderFactories, numKeys, harness.getFileId());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
index b91cdac..d244683 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestContext;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestHarness;
@@ -33,7 +34,7 @@
private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
@@ -46,7 +47,7 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys) throws Exception {
return LSMRTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
valueProviderFactories, numKeys, harness.getFileId());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMultiBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMultiBulkLoadTest.java
index c2ef5a4..5356d85 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMultiBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMultiBulkLoadTest.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestContext;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestHarness;
@@ -38,7 +39,7 @@
private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
@Before
- public void setUp() throws HyracksDataException {
+ public void setUp() throws HyracksException {
harness.setUp();
}
@@ -51,7 +52,7 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys) throws Exception {
return LSMRTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
valueProviderFactories, numKeys, harness.getFileId());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeSerachTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeSerachTest.java
index d004073..af1c78c 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeSerachTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeSerachTest.java
@@ -127,7 +127,7 @@
BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, btreeCmpFactories, fieldCount,
btreeInteriorFrameFactory, btreeLeafFrameFactory);
- ILSMFileManager fileNameManager = new LSMTreeFileManager(onDiskDir);
+ ILSMFileManager fileNameManager = new LSMTreeFileManager(ioManager, onDiskDir);
LSMRTree lsmRTree = new LSMRTree(memBufferCache, memFreePageManager, rtreeInteriorFrameFactory,
rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager,
diskRTreeFactory, diskBTreeFactory, diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories);
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeTest.java
index f4f72ea..5acf8ee 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeTest.java
@@ -121,7 +121,7 @@
BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, btreeCmpFactories, fieldCount,
btreeInteriorFrameFactory, btreeLeafFrameFactory);
- ILSMFileManager fileNameManager = new LSMTreeFileManager(onDiskDir);
+ ILSMFileManager fileNameManager = new LSMTreeFileManager(ioManager, onDiskDir);
LSMRTree lsmRTree = new LSMRTree(memBufferCache, memFreePageManager, rtreeInteriorFrameFactory,
rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager,
diskRTreeFactory, diskBTreeFactory, diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories);
@@ -267,7 +267,7 @@
BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, btreeCmpFactories, fieldCount,
btreeInteriorFrameFactory, btreeLeafFrameFactory);
- ILSMFileManager fileNameManager = new LSMTreeFileManager(onDiskDir);
+ ILSMFileManager fileNameManager = new LSMTreeFileManager(ioManager, onDiskDir);
LSMRTree lsmRTree = new LSMRTree(memBufferCache, memFreePageManager, rtreeInteriorFrameFactory,
rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager,
diskRTreeFactory, diskBTreeFactory, diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories);
@@ -464,7 +464,7 @@
BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, btreeCmpFactories, fieldCount,
btreeInteriorFrameFactory, btreeLeafFrameFactory);
- ILSMFileManager fileNameManager = new LSMTreeFileManager(onDiskDir);
+ ILSMFileManager fileNameManager = new LSMTreeFileManager(ioManager, onDiskDir);
LSMRTree lsmRTree = new LSMRTree(memBufferCache, memFreePageManager, rtreeInteriorFrameFactory,
rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager,
diskRTreeFactory, diskBTreeFactory, diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories);
@@ -615,7 +615,7 @@
BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, btreeCmpFactories, fieldCount,
btreeInteriorFrameFactory, btreeLeafFrameFactory);
- ILSMFileManager fileNameManager = new LSMTreeFileManager(onDiskDir);
+ ILSMFileManager fileNameManager = new LSMTreeFileManager(ioManager, onDiskDir);
LSMRTree lsmRTree = new LSMRTree(memBufferCache, memFreePageManager, rtreeInteriorFrameFactory,
rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager,
diskRTreeFactory, diskBTreeFactory, diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories);
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
index fc3405a..9552dbc 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
@@ -20,6 +20,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
@@ -61,7 +62,7 @@
}
public static LSMRTreeTestContext create(InMemoryBufferCache memBufferCache,
- InMemoryFreePageManager memFreePageManager, String onDiskDir, IBufferCache diskBufferCache,
+ InMemoryFreePageManager memFreePageManager, IOManager ioManager, String onDiskDir, IBufferCache diskBufferCache,
IFileMapProvider diskFileMapProvider, ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeyFields, int fileId) throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
@@ -69,7 +70,7 @@
.serdesToComparatorFactories(fieldSerdes, numKeyFields);
IBinaryComparatorFactory[] btreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes,
fieldSerdes.length);
- LSMRTree lsmTree = LSMRTreeUtils.createLSMTree(memBufferCache, memFreePageManager, onDiskDir, diskBufferCache,
+ LSMRTree lsmTree = LSMRTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ioManager, onDiskDir, diskBufferCache,
diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories);
lsmTree.create(fileId);
lsmTree.open(fileId);
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
index 57976fe..9844539 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
@@ -23,6 +23,8 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
@@ -54,6 +56,7 @@
protected final int memNumPages;
protected final int hyracksFrameSize;
+ protected IOManager ioManager;
protected IBufferCache diskBufferCache;
protected IFileMapProvider diskFileMapProvider;
protected LSMRTreeInMemoryBufferCache memBufferCache;
@@ -62,7 +65,6 @@
protected final Random rnd = new Random();
protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
- protected final static String tmpDir = System.getProperty("java.io.tmpdir");
protected final static String sep = System.getProperty("file.separator");
protected String onDiskDir;
@@ -85,14 +87,15 @@
this.hyracksFrameSize = hyracksFrameSize;
}
- public void setUp() throws HyracksDataException {
- onDiskDir = tmpDir + sep + "lsm_rtree_" + simpleDateFormat.format(new Date()) + sep;
+ public void setUp() throws HyracksException {
+ onDiskDir = "lsm_rtree_" + simpleDateFormat.format(new Date()) + sep;
ctx = TestUtils.create(getHyracksFrameSize());
TestStorageManagerComponentHolder.init(diskPageSize, diskNumPages, diskMaxOpenFiles);
diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
diskFileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
memBufferCache = new LSMRTreeInMemoryBufferCache(new HeapBufferAllocator(), memPageSize, memNumPages);
memFreePageManager = new LSMRTreeInMemoryFreePageManager(memNumPages, new LIFOMetaDataFrameFactory());
+ ioManager = TestStorageManagerComponentHolder.getIOManager();
rnd.setSeed(RANDOM_SEED);
}
@@ -132,6 +135,10 @@
return DUMMY_FILE_ID;
}
+ public IOManager getIOManager() {
+ return ioManager;
+ }
+
public IBufferCache getDiskBufferCache() {
return diskBufferCache;
}
diff --git a/hyracks-tests/hyracks-storage-common-test/src/test/java/edu/uci/ics/hyracks/storage/common/BufferCacheRegressionTests.java b/hyracks-tests/hyracks-storage-common-test/src/test/java/edu/uci/ics/hyracks/storage/common/BufferCacheRegressionTests.java
index 6c91cdd..fff9e31 100644
--- a/hyracks-tests/hyracks-storage-common-test/src/test/java/edu/uci/ics/hyracks/storage/common/BufferCacheRegressionTests.java
+++ b/hyracks-tests/hyracks-storage-common-test/src/test/java/edu/uci/ics/hyracks/storage/common/BufferCacheRegressionTests.java
@@ -76,7 +76,7 @@
}
bufferCache.closeFile(firstFileId);
if (deleteFile) {
- bufferCache.deleteFile(firstFileId);
+ bufferCache.deleteFile(firstFileId, false);
}
// Create a file with the same name.
@@ -121,7 +121,7 @@
testFileHandle.close();
bufferCache.closeFile(secondFileId);
if (deleteFile) {
- bufferCache.deleteFile(secondFileId);
+ bufferCache.deleteFile(secondFileId, false);
}
bufferCache.close();
}