Completed consistency checking for components of the LSM B-Tree and LSM R-Tree in open().
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_lsm_tree@1177 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
index 1060d28..f0a6a8f 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
@@ -37,9 +37,9 @@
public ITreeIndex createIndexInstance() throws HyracksDataException {
IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
ITreeIndexMetaDataFrameFactory metaDataFrameFactory = new LIFOMetaDataFrameFactory();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, indexFileId, 0,
- metaDataFrameFactory);
- return new BTree(bufferCache, treeOpDesc.getTreeIndexTypeTraits().length, treeOpDesc.getTreeIndexComparatorFactories(), freePageManager,
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaDataFrameFactory);
+ return new BTree(bufferCache, treeOpDesc.getTreeIndexTypeTraits().length,
+ treeOpDesc.getTreeIndexComparatorFactories(), freePageManager,
treeOpDesc.getTreeIndexInteriorFactory(), treeOpDesc.getTreeIndexLeafFactory());
}
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
index 0389034..8726030 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
@@ -86,6 +86,7 @@
ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
this.fileId = fileId;
+ freePageManager.open(fileId);
freePageManager.init(metaFrame, rootPage);
initRoot(leafFrame, true);
} finally {
@@ -94,13 +95,15 @@
}
@Override
- public void open(int fileId) {
+ public void open(int fileId) {
this.fileId = fileId;
+ freePageManager.open(fileId);
}
@Override
public void close() {
fileId = -1;
+ freePageManager.close();
}
private void diskOrderScan(ITreeIndexCursor icursor, BTreeOpContext ctx) throws HyracksDataException {
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java
index 556ce04..675eff1 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java
@@ -26,7 +26,7 @@
ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
BTree btree = new BTree(bufferCache, typeTraits.length, cmpFactories, freePageManager, interiorFrameFactory, leafFrameFactory);
return btree;
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IFreePageManager.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IFreePageManager.java
index 6db933d..98b9a7e 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IFreePageManager.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IFreePageManager.java
@@ -3,6 +3,10 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
public interface IFreePageManager {
+ public void open(int fileId);
+
+ public void close();
+
public int getFreePage(ITreeIndexMetaDataFrame metaFrame)
throws HyracksDataException;
@@ -27,5 +31,5 @@
public boolean isFreePage(ITreeIndexMetaDataFrame metaFrame);
- public int getFirstMetadataPage();
+ public int getFirstMetadataPage();
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DataGenThread.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DataGenThread.java
index 2f62231..e8d3d56 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DataGenThread.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DataGenThread.java
@@ -8,9 +8,10 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
/**
- * Quick & dirty data generator for performance testing.
+ * Quick & dirty data generator for multi-thread testing.
*
*/
+@SuppressWarnings("rawtypes")
public class DataGenThread extends Thread {
public final BlockingQueue<TupleBatch> tupleBatchQueue;
private final int maxNumBatches;
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/TupleBatch.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/TupleBatch.java
index c500c94..bfa523f 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/TupleBatch.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/TupleBatch.java
@@ -6,6 +6,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+@SuppressWarnings("rawtypes")
public class TupleBatch {
private final int size;
private final TupleGenerator[] tupleGens;
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManager.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManager.java
index e72de8b..036aa09 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManager.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManager.java
@@ -13,14 +13,13 @@
private static final byte META_PAGE_LEVEL_INDICATOR = -1;
private static final byte FREE_PAGE_LEVEL_INDICATOR = -2;
private final IBufferCache bufferCache;
- private final int fileId;
- private final int headPage;
+ private final int headPage;
+ private int fileId = -1;
private final ITreeIndexMetaDataFrameFactory metaDataFrameFactory;
- public LinkedListFreePageManager(IBufferCache bufferCache, int fileId,
+ public LinkedListFreePageManager(IBufferCache bufferCache,
int headPage, ITreeIndexMetaDataFrameFactory metaDataFrameFactory) {
this.bufferCache = bufferCache;
- this.fileId = fileId;
this.headPage = headPage;
this.metaDataFrameFactory = metaDataFrameFactory;
}
@@ -204,4 +203,14 @@
public int getFirstMetadataPage() {
return headPage;
}
+
+ @Override
+ public void open(int fileId) {
+ this.fileId = fileId;
+ }
+
+ @Override
+ public void close() {
+ fileId = -1;
+ }
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java
index bcf7e70..157b563 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java
@@ -29,8 +29,7 @@
this.bufferCache = bufferCache;
}
- public IFreePageManager createFreePageManager(int fileId) {
- return new LinkedListFreePageManager(bufferCache, fileId, 0, metaDataFrameFactory);
+ public IFreePageManager createFreePageManager() {
+ return new LinkedListFreePageManager(bufferCache, 0, metaDataFrameFactory);
}
-
}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/BTreeFactory.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/BTreeFactory.java
deleted file mode 100644
index 51c8ce3..0000000
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/BTreeFactory.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-
-public class BTreeFactory {
-
- private IBufferCache bufferCache;
- private int fieldCount;
- private IBinaryComparatorFactory[] cmpFactories;
- private ITreeIndexFrameFactory interiorFrameFactory;
- private ITreeIndexFrameFactory leafFrameFactory;
- private LinkedListFreePageManagerFactory freePageManagerFactory;
-
- public BTreeFactory(IBufferCache bufferCache, LinkedListFreePageManagerFactory freePageManagerFactory, IBinaryComparatorFactory[] cmpFactories,
- int fieldCount, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
- this.bufferCache = bufferCache;
- this.fieldCount = fieldCount;
- this.cmpFactories = cmpFactories;
- this.interiorFrameFactory = interiorFrameFactory;
- this.leafFrameFactory = leafFrameFactory;
- this.freePageManagerFactory = freePageManagerFactory;
- }
-
- public BTree createBTreeInstance(int fileId) {
- return new BTree(bufferCache, fieldCount, cmpFactories, freePageManagerFactory.createFreePageManager(fileId),
- interiorFrameFactory, leafFrameFactory);
- }
-
- public IBufferCache getBufferCache() {
- return bufferCache;
- }
-}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index f6849eb..725f680 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -47,6 +47,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMTree;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexComponentFinalizer;
@@ -81,6 +82,8 @@
private final ITreeIndexFrameFactory deleteLeafFrameFactory;
private final IBinaryComparatorFactory[] cmpFactories;
+ private boolean isOpen = false;
+
public LSMBTree(IBufferCache memBufferCache, InMemoryFreePageManager memFreePageManager,
ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory,
ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMFileManager fileNameManager, BTreeFactory diskBTreeFactory,
@@ -98,7 +101,7 @@
this.diskBTrees = new LinkedList<Object>();
this.fileManager = fileNameManager;
lsmHarness = new LSMHarness(this);
- componentFinalizer = new TreeIndexComponentFinalizer();
+ componentFinalizer = new TreeIndexComponentFinalizer(diskFileMapProvider);
}
@Override
@@ -117,26 +120,39 @@
*/
@Override
public void open(int indexFileId) throws HyracksDataException {
- memBTree.open(indexFileId);
- List<Object> validFileNames = fileManager.cleanupAndGetValidFiles();
- for (Object o : validFileNames) {
- String fileName = (String) o;
- FileReference fileRef = new FileReference(new File(fileName));
- BTree btree = createDiskBTree(diskBTreeFactory, fileRef, false);
- diskBTrees.add(btree);
+ synchronized(this) {
+ if (isOpen) {
+ return;
+ }
+ memBTree.open(indexFileId);
+ BTree dummyBTree = diskBTreeFactory.createIndexInstance();
+ List<Object> validFileNames = fileManager.cleanupAndGetValidFiles(dummyBTree, componentFinalizer);
+ for (Object o : validFileNames) {
+ String fileName = (String) o;
+ FileReference fileRef = new FileReference(new File(fileName));
+ BTree btree = createDiskBTree(diskBTreeFactory, fileRef, false);
+ diskBTrees.add(btree);
+ }
+ isOpen = true;
}
}
@Override
- public void close() throws HyracksDataException {
- for (Object o : diskBTrees) {
- BTree btree = (BTree) o;
- diskBufferCache.closeFile(btree.getFileId());
- diskBufferCache.deleteFile(btree.getFileId(), false);
- btree.close();
+ public void close() throws HyracksDataException {
+ synchronized(this) {
+ if (!isOpen) {
+ return;
+ }
+ for (Object o : diskBTrees) {
+ BTree btree = (BTree) o;
+ diskBufferCache.closeFile(btree.getFileId());
+ diskBufferCache.deleteFile(btree.getFileId(), false);
+ btree.close();
+ }
+ diskBTrees.clear();
+ memBTree.close();
+ isOpen = false;
}
- diskBTrees.clear();
- memBTree.close();
}
@Override
@@ -305,7 +321,7 @@
// File will be closed during cleanup of merge().
diskBufferCache.openFile(diskBTreeFileId);
// Create new BTree instance.
- BTree diskBTree = factory.createBTreeInstance(diskBTreeFileId);
+ BTree diskBTree = factory.createIndexInstance();
if (createBTree) {
diskBTree.create(diskBTreeFileId);
}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
index 1758a19..a27adf5 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
@@ -24,13 +24,13 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
-import edu.uci.ics.hyracks.storage.am.lsm.btree.impls.BTreeFactory;
import edu.uci.ics.hyracks.storage.am.lsm.btree.impls.LSMBTree;
import edu.uci.ics.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory;
import edu.uci.ics.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeFileManager;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@@ -56,7 +56,7 @@
typeTraits.length, interiorFrameFactory, copyTupleLeafFrameFactory);
BTreeFactory bulkLoadBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, cmpFactories,
typeTraits.length, interiorFrameFactory, insertLeafFrameFactory);
- ILSMFileManager fileNameManager = new LSMTreeFileManager(ioManager, onDiskDir);
+ ILSMFileManager fileNameManager = new LSMTreeFileManager(ioManager, diskFileMapProvider, onDiskDir);
LSMBTree lsmTree = new LSMBTree(memBufferCache, memFreePageManager, interiorFrameFactory,
insertLeafFrameFactory, deleteLeafFrameFactory, fileNameManager, diskBTreeFactory,
bulkLoadBTreeFactory, diskFileMapProvider, typeTraits.length, cmpFactories);
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFinalizer.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFinalizer.java
index 3423e75..f241fd6 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFinalizer.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFinalizer.java
@@ -15,20 +15,21 @@
package edu.uci.ics.hyracks.storage.am.lsm.common.api;
+import java.io.File;
+
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
public interface ILSMComponentFinalizer {
/**
- * @return Whether the given LSM component is considered valid. Used for guaranteeing
+ * @return Checks whether the given file is valid with respect to the given LSM component. Used for guaranteeing
* atomicity of LSM component writes.
*/
- public boolean isValid(Object lsmComponent) throws HyracksDataException;
+ public boolean isValid(File file, Object lsmComponent) throws HyracksDataException;
/**
* Marks the given LSM component as physically valid, synchronously forcing
- * the necessary information to disk. This call only return once the
- * physical consistency of the given component is guaranteed.
+ * the necessary information to disk.
*
*/
public void finalize(Object lsmComponent) throws HyracksDataException;
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFileManager.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFileManager.java
index 1184b76..9150b01 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFileManager.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFileManager.java
@@ -47,7 +47,7 @@
// Deletes invalid files, and returns list of valid files from baseDir.
// The returned valid files are correctly sorted (based on the recency of data).
- public List<Object> cleanupAndGetValidFiles() throws HyracksDataException;
+ public List<Object> cleanupAndGetValidFiles(Object lsmComponent, ILSMComponentFinalizer componentFinalizer) throws HyracksDataException;
public Comparator<String> getFileNameComparator();
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryFreePageManager.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryFreePageManager.java
index 846c274..dfd437b 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryFreePageManager.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryFreePageManager.java
@@ -99,4 +99,14 @@
// Method doesn't make sense for this free page manager.
return -1;
}
+
+ @Override
+ public void open(int fileId) {
+ // Method doesn't make sense for this free page manager.
+ }
+
+ @Override
+ public void close() {
+ // Method doesn't make sense for this free page manager.
+ }
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BTreeFactory.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
index fcc43e7..c5a6a1a 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
@@ -17,12 +17,11 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-public class BTreeFactory extends TreeFactory {
+public class BTreeFactory extends TreeFactory<BTree> {
public BTreeFactory(IBufferCache bufferCache, LinkedListFreePageManagerFactory freePageManagerFactory, IBinaryComparatorFactory[] cmpFactories,
int fieldCount, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
@@ -30,8 +29,8 @@
}
@Override
- public ITreeIndex createIndexInstance(int fileId) {
- return new BTree(bufferCache, fieldCount, cmpFactories, freePageManagerFactory.createFreePageManager(fileId),
+ public BTree createIndexInstance() {
+ return new BTree(bufferCache, fieldCount, cmpFactories, freePageManagerFactory.createFreePageManager(),
interiorFrameFactory, leafFrameFactory);
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java
index 2c179e7..bafdbf0 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java
@@ -29,7 +29,9 @@
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.control.nc.io.IOManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFinalizer;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFileManager;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class LSMTreeFileManager implements ILSMFileManager {
@@ -37,6 +39,7 @@
// Currently uses all IODevices registered in ioManager in a round-robin fashion.
protected final IOManager ioManager;
+ protected final IFileMapProvider fileMapProvider;
// baseDir should reflect dataset name, and partition name.
protected final String baseDir;
protected final Format formatter = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS");
@@ -45,10 +48,17 @@
// To implement round-robin assignment of files onto I/O devices.
private int ioDeviceIndex = 0;
- public LSMTreeFileManager(IOManager ioManager, String baseDir) {
+ private static FilenameFilter fileNameFilter = new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return !name.startsWith(".");
+ }
+ };
+
+ public LSMTreeFileManager(IOManager ioManager, IFileMapProvider fileMapProvider, String baseDir) {
if (!baseDir.endsWith(System.getProperty("file.separator"))) {
baseDir += System.getProperty("file.separator");
}
+ this.fileMapProvider = fileMapProvider;
this.ioManager = ioManager;
this.baseDir = baseDir;
createDirs();
@@ -117,22 +127,28 @@
return baseDir;
}
+ protected void getValidFiles(IODeviceHandle dev, FilenameFilter filter, Object lsmComponent,
+ ILSMComponentFinalizer componentFinalizer, ArrayList<ComparableFileName> allFiles)
+ throws HyracksDataException {
+ File dir = new File(dev.getPath(), baseDir);
+ String[] files = dir.list(filter);
+ for (String fileName : files) {
+ File file = new File(dir.getPath() + File.separator + fileName);
+ if (componentFinalizer.isValid(file, lsmComponent)) {
+ allFiles.add(new ComparableFileName(file.getAbsolutePath()));
+ } else {
+ file.delete();
+ }
+ }
+ }
+
@Override
- public List<Object> cleanupAndGetValidFiles() throws HyracksDataException {
+ public List<Object> cleanupAndGetValidFiles(Object lsmComponent, ILSMComponentFinalizer componentFinalizer) throws HyracksDataException {
List<Object> validFiles = new ArrayList<Object>();
ArrayList<ComparableFileName> allFiles = new ArrayList<ComparableFileName>();
// Gather files from all IODeviceHandles.
- for(IODeviceHandle dev : ioManager.getIODevices()) {
- File dir = new File(dev.getPath(), baseDir);
- FilenameFilter filter = new FilenameFilter() {
- public boolean accept(File dir, String name) {
- return !name.startsWith(".");
- }
- };
- String[] files = dir.list(filter);
- for (String file : files) {
- allFiles.add(new ComparableFileName(dir.getPath() + File.separator + file));
- }
+ for(IODeviceHandle dev : ioManager.getIODevices()) {
+ getValidFiles(dev, fileNameFilter, lsmComponent, componentFinalizer, allFiles);
}
// Trivial cases.
if (allFiles.isEmpty()) {
@@ -174,14 +190,16 @@
}
protected class ComparableFileName implements Comparable<ComparableFileName> {
- public final String fullPath;
+ public final String fullPath;
+ public final String fileName;
// Timestamp interval.
public final String[] interval;
public ComparableFileName(String fullPath) {
- this.fullPath = fullPath;
+ this.fullPath = fullPath;
File f = new File(fullPath);
- interval = f.getName().split(SPLIT_STRING);
+ this.fileName = f.getName();
+ interval = fileName.split(SPLIT_STRING);
}
@Override
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java
index 4be0afb..a6aa7f9 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java
@@ -21,7 +21,7 @@
import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-public abstract class TreeFactory {
+public abstract class TreeFactory <T extends ITreeIndex> {
protected IBufferCache bufferCache;
protected int fieldCount;
@@ -40,7 +40,7 @@
this.freePageManagerFactory = freePageManagerFactory;
}
- public abstract ITreeIndex createIndexInstance(int fileId);
+ public abstract T createIndexInstance();
public IBufferCache getBufferCache() {
return bufferCache;
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeIndexComponentFinalizer.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeIndexComponentFinalizer.java
index 77efb16..0f5c2ff 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeIndexComponentFinalizer.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeIndexComponentFinalizer.java
@@ -15,30 +15,51 @@
package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
+import java.io.File;
+
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFinalizer;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class TreeIndexComponentFinalizer implements ILSMComponentFinalizer {
+ protected final IFileMapProvider fileMapProvider;
+
+ public TreeIndexComponentFinalizer(IFileMapProvider fileMapProvider) {
+ this.fileMapProvider = fileMapProvider;
+ }
+
@Override
- public boolean isValid(Object lsmComponent) throws HyracksDataException {
+ public boolean isValid(File file, Object lsmComponent) throws HyracksDataException {
ITreeIndex treeIndex = (ITreeIndex) lsmComponent;
IBufferCache bufferCache = treeIndex.getBufferCache();
- int metadataPage = treeIndex.getFreePageManager().getFirstMetadataPage();
- ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory().createFrame();
- ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(treeIndex.getFileId(), metadataPage), false);
- page.acquireReadLatch();
+ FileReference fileRef = new FileReference(file);
+ bufferCache.createFile(fileRef);
+ int fileId = fileMapProvider.lookupFileId(fileRef);
+ bufferCache.openFile(fileId);
+ treeIndex.open(fileId);
try {
- metadataFrame.setPage(page);
- return metadataFrame.isValid();
+ int metadataPage = treeIndex.getFreePageManager().getFirstMetadataPage();
+ ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory().createFrame();
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(treeIndex.getFileId(), metadataPage), false);
+ page.acquireReadLatch();
+ try {
+ metadataFrame.setPage(page);
+ return metadataFrame.isValid();
+ } finally {
+ page.releaseReadLatch();
+ bufferCache.unpin(page);
+ }
} finally {
- page.releaseReadLatch();
- bufferCache.unpin(page);
+ treeIndex.close();
+ bufferCache.closeFile(fileId);
+ bufferCache.deleteFile(fileId, false);
}
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 9a831d8..0a364f2 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -132,7 +132,7 @@
this.btreeCmpFactories = btreeCmpFactories;
this.rtreeCmpFactories = rtreeCmpFactories;
this.lsmHarness = new LSMHarness(this);
- componentFinalizer = new LSMRTreeComponentFinalizer();
+ componentFinalizer = new LSMRTreeComponentFinalizer(diskFileMapProvider);
}
@Override
@@ -154,7 +154,10 @@
public void open(int indexFileId) throws HyracksDataException {
memComponent.getRTree().open(MEM_RTREE_FILE_ID);
memComponent.getBTree().open(MEM_BTREE_FILE_ID);
- List<Object> validFileNames = fileManager.cleanupAndGetValidFiles();
+ RTree dummyRTree = diskRTreeFactory.createIndexInstance();
+ BTree dummyBTree = diskBTreeFactory.createIndexInstance();
+ LSMRTreeComponent dummyComponent = new LSMRTreeComponent(dummyRTree, dummyBTree);
+ List<Object> validFileNames = fileManager.cleanupAndGetValidFiles(dummyComponent, componentFinalizer);
for (Object o : validFileNames) {
LSMRTreeFileNameComponent component = (LSMRTreeFileNameComponent) o;
FileReference rtreeFile = new FileReference(new File(component.getRTreeFileName()));
@@ -194,6 +197,7 @@
return component;
}
+ @SuppressWarnings("rawtypes")
protected ITreeIndex createDiskTree(TreeFactory diskTreeFactory, FileReference fileRef, boolean createTree)
throws HyracksDataException {
// File will be deleted during cleanup of merge().
@@ -202,7 +206,7 @@
// File will be closed during cleanup of merge().
diskBufferCache.openFile(diskTreeFileId);
// Create new tree instance.
- ITreeIndex diskTree = diskTreeFactory.createIndexInstance(diskTreeFileId);
+ ITreeIndex diskTree = diskTreeFactory.createIndexInstance();
if (createTree) {
diskTree.create(diskTreeFileId);
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeComponentFinalizer.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeComponentFinalizer.java
index 1a9d9b0..b3fbce7 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeComponentFinalizer.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeComponentFinalizer.java
@@ -16,25 +16,20 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFinalizer;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexComponentFinalizer;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTree.LSMRTreeComponent;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-public class LSMRTreeComponentFinalizer implements ILSMComponentFinalizer {
+public class LSMRTreeComponentFinalizer extends TreeIndexComponentFinalizer {
- TreeIndexComponentFinalizer treeIndexFinalizer = new TreeIndexComponentFinalizer();
-
- @Override
- public boolean isValid(Object lsmComponent) throws HyracksDataException {
- LSMRTreeComponent component = (LSMRTreeComponent) lsmComponent;
- return treeIndexFinalizer.isValid(component.getRTree())
- && treeIndexFinalizer.isValid(component.getBTree());
- }
+ public LSMRTreeComponentFinalizer(IFileMapProvider fileMapProvider) {
+ super(fileMapProvider);
+ }
- @Override
+ @Override
public void finalize(Object lsmComponent) throws HyracksDataException {
LSMRTreeComponent component = (LSMRTreeComponent) lsmComponent;
- treeIndexFinalizer.finalize(component.getRTree());
- treeIndexFinalizer.finalize(component.getBTree());
+ super.finalize(component.getRTree());
+ super.finalize(component.getBTree());
}
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
index b30b48a..fe28a50 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
@@ -26,14 +26,29 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.control.nc.io.IOManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFinalizer;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeFileManager;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTree.LSMRTreeComponent;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class LSMRTreeFileManager extends LSMTreeFileManager {
private static final String RTREE_STRING = "r";
private static final String BTREE_STRING = "b";
- public LSMRTreeFileManager(IOManager ioManager, String baseDir) {
- super(ioManager, baseDir);
+ private static FilenameFilter btreeFilter = new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return !name.startsWith(".") && name.endsWith(BTREE_STRING);
+ }
+ };
+
+ private static FilenameFilter rtreeFilter = new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return !name.startsWith(".") && name.endsWith(RTREE_STRING);
+ }
+ };
+
+ public LSMRTreeFileManager(IOManager ioManager, IFileMapProvider fileMapProvider, String baseDir) {
+ super(ioManager, fileMapProvider, baseDir);
}
@Override
@@ -51,56 +66,44 @@
+ BTREE_STRING);
}
- private boolean searchForFileName(HashSet<String> stringSet, String file) {
- if (stringSet.contains(file)) {
- return true;
- }
- return false;
-
- }
-
- public List<Object> cleanupAndGetValidFiles() throws HyracksDataException {
+ @Override
+ public List<Object> cleanupAndGetValidFiles(Object lsmComponent, ILSMComponentFinalizer componentFinalizer) throws HyracksDataException {
List<Object> validFiles = new ArrayList<Object>();
ArrayList<ComparableFileName> allRTreeFiles = new ArrayList<ComparableFileName>();
ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<ComparableFileName>();
-
+ LSMRTreeComponent component = (LSMRTreeComponent) lsmComponent;
+
// Gather files from all IODeviceHandles.
- for (IODeviceHandle dev : ioManager.getIODevices()) {
- File dir = new File(dev.getPath(), baseDir);
- FilenameFilter btreeFilter = new FilenameFilter() {
- public boolean accept(File dir, String name) {
- return !name.startsWith(".") && name.endsWith(BTREE_STRING);
- }
- };
- String[] btreeFiles = dir.list(btreeFilter);
- for (String file : btreeFiles) {
- allBTreeFiles.add(new ComparableFileName(dir.getPath() + File.separator + file));
- }
+ for (IODeviceHandle dev : ioManager.getIODevices()) {
+ getValidFiles(dev, btreeFilter, component.getBTree(), componentFinalizer, allBTreeFiles);
HashSet<String> btreeFilesSet = new HashSet<String>();
- for (String file : btreeFiles) {
- int index = file.lastIndexOf(SPLIT_STRING);
- btreeFilesSet.add(file.substring(0, index));
+ for (ComparableFileName cmpFileName : allBTreeFiles) {
+ int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
+ btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
}
-
- FilenameFilter rtreeFilter = new FilenameFilter() {
- public boolean accept(File dir, String name) {
- return !name.startsWith(".") && name.endsWith(RTREE_STRING);
- }
- };
- String[] rtreeFiles = dir.list(rtreeFilter);
- for (String file : rtreeFiles) {
- int index = file.lastIndexOf(SPLIT_STRING);
- file = file.substring(0, index);
- if (searchForFileName(btreeFilesSet, file)) {
- allRTreeFiles.add(new ComparableFileName(dir.getPath() + File.separator + file));
+ // List of valid RTree files that may or may not have a BTree buddy. Will check for buddies below.
+ ArrayList<ComparableFileName> tmpAllRTreeFiles = new ArrayList<ComparableFileName>();
+ getValidFiles(dev, rtreeFilter, component.getRTree(), componentFinalizer, tmpAllRTreeFiles);
+ // Look for buddy BTrees for all valid RTrees.
+ // If no buddy is found, delete the file, otherwise add the RTree to allRTreeFiles.
+ for (ComparableFileName cmpFileName : tmpAllRTreeFiles) {
+ int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
+ String file = cmpFileName.fileName.substring(0, index);
+ if (btreeFilesSet.contains(file)) {
+ allRTreeFiles.add(cmpFileName);
} else {
// Couldn't find the corresponding BTree file; thus, delete
// the RTree file.
- File invalidRTreeFile = new File(dir.getPath() + File.separator + file);
+ File invalidRTreeFile = new File(cmpFileName.fullPath);
invalidRTreeFile.delete();
}
}
}
+ // Sanity check.
+ if (allRTreeFiles.size() != allBTreeFiles.size()) {
+ throw new HyracksDataException("Unequal number of valid RTree and BTree files found. Aborting cleanup.");
+ }
+
// Trivial cases.
if (allRTreeFiles.isEmpty() || allBTreeFiles.isEmpty()) {
return validFiles;
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
index eacf23f..3e19097 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
@@ -16,14 +16,13 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeFactory;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-public class RTreeFactory extends TreeFactory {
+public class RTreeFactory extends TreeFactory<RTree> {
public RTreeFactory(IBufferCache bufferCache, LinkedListFreePageManagerFactory freePageManagerFactory, IBinaryComparatorFactory[] cmpFactories,
int fieldCount, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
@@ -31,8 +30,8 @@
}
@Override
- public ITreeIndex createIndexInstance(int fileId) {
- return new RTree(bufferCache, fieldCount, cmpFactories, freePageManagerFactory.createFreePageManager(fileId),
+ public RTree createIndexInstance() {
+ return new RTree(bufferCache, fieldCount, cmpFactories, freePageManagerFactory.createFreePageManager(),
interiorFrameFactory, leafFrameFactory);
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
index 6b96843..163ce5c 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
@@ -64,7 +64,7 @@
BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, btreeCmpFactories,
typeTraits.length, btreeInteriorFrameFactory, btreeLeafFrameFactory);
- ILSMFileManager fileNameManager = new LSMRTreeFileManager(ioManager, onDiskDir);
+ ILSMFileManager fileNameManager = new LSMRTreeFileManager(ioManager, diskFileMapProvider, onDiskDir);
LSMRTree lsmTree = new LSMRTree(memBufferCache, memFreePageManager, rtreeInteriorFrameFactory,
rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager,
diskRTreeFactory, diskBTreeFactory, diskFileMapProvider, typeTraits.length, rtreeCmpFactories,
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java
index 75328aa..447c0cb 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java
@@ -38,9 +38,9 @@
public ITreeIndex createIndexInstance() throws HyracksDataException {
IBufferCache bufferCache = treeOpDesc.getStorageManager().getBufferCache(ctx);
ITreeIndexMetaDataFrameFactory metaDataFrameFactory = new LIFOMetaDataFrameFactory();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, indexFileId, 0,
- metaDataFrameFactory);
- return new RTree(bufferCache, treeOpDesc.getTreeIndexTypeTraits().length, treeOpDesc.getTreeIndexComparatorFactories(), freePageManager,
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaDataFrameFactory);
+ return new RTree(bufferCache, treeOpDesc.getTreeIndexTypeTraits().length,
+ treeOpDesc.getTreeIndexComparatorFactories(), freePageManager,
treeOpDesc.getTreeIndexInteriorFactory(), treeOpDesc.getTreeIndexLeafFactory());
}
}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
index 09096ef..0e1dd23 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
@@ -166,6 +166,7 @@
try {
ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
+ freePageManager.open(fileId);
freePageManager.init(metaFrame, rootPage);
// initialize root page
@@ -187,11 +188,13 @@
@Override
public void open(int fileId) {
this.fileId = fileId;
+ freePageManager.open(fileId);
}
@Override
public void close() {
fileId = -1;
+ freePageManager.close();
}
@Override
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/util/RTreeUtils.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/util/RTreeUtils.java
index f17d1ab..13c4386 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/util/RTreeUtils.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/util/RTreeUtils.java
@@ -45,7 +45,7 @@
valueProviderFactories);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, rtreeFileId, 0, metaFrameFactory);
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
RTree rtree = new RTree(bufferCache, typeTraits.length, cmpFactories, freePageManager, interiorFrameFactory,
leafFrameFactory);
return rtree;
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeSearchCursorTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeSearchCursorTest.java
index 7afea00..5b039d1 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeSearchCursorTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeSearchCursorTest.java
@@ -97,7 +97,7 @@
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
BTree btree = new BTree(bufferCache, fieldCount, cmpFactories, freePageManager, interiorFrameFactory, leafFrameFactory);
btree.create(btreeFileId);
@@ -170,7 +170,7 @@
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
BTree btree = new BTree(bufferCache, fieldCount, cmpFactories, freePageManager, interiorFrameFactory, leafFrameFactory);
btree.create(btreeFileId);
@@ -240,7 +240,7 @@
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
BTree btree = new BTree(bufferCache, fieldCount, cmpFactories, freePageManager, interiorFrameFactory, leafFrameFactory);
btree.create(btreeFileId);
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java
index 28ae335..75041ab 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java
@@ -84,7 +84,7 @@
IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
BTree btree = new BTree(bufferCache, fieldCount, cmpFactories, freePageManager, interiorFrameFactory, leafFrameFactory);
btree.create(fileId);
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateSearchTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
index 1ef90bc..9b66d07 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
@@ -63,7 +63,7 @@
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
BTree btree = new BTree(bufferCache, fieldCount, cmpFactories, freePageManager, interiorFrameFactory, leafFrameFactory);
btree.create(btreeFileId);
btree.open(btreeFileId);
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java
index 2d350c6..4dba14a 100644
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java
@@ -168,7 +168,7 @@
btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
- freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
+ freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
btree = new BTree(bufferCache, btreeTypeTraits.length, btreeCmpFactories, freePageManager, interiorFrameFactory,
leafFrameFactory);
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java
index 955da5a..8a40a29 100644
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java
@@ -119,7 +119,7 @@
ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
BTree btree = new BTree(bufferCache, btreeTypeTraits.length, cmpFactories, freePageManager, interiorFrameFactory,
leafFrameFactory);
diff --git a/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/DummyLSMComponentFinalizer.java b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/DummyLSMComponentFinalizer.java
new file mode 100644
index 0000000..4a43f9b
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/DummyLSMComponentFinalizer.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.common;
+
+import java.io.File;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFinalizer;
+
+public class DummyLSMComponentFinalizer implements ILSMComponentFinalizer {
+
+ @Override
+ public boolean isValid(File file, Object lsmComponent)
+ throws HyracksDataException {
+ return true;
+ }
+
+ @Override
+ public void finalize(Object lsmComponent) throws HyracksDataException {
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/LSMTreeFileManagerTest.java b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/LSMTreeFileManagerTest.java
index 4ae8734..6c78afb 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/LSMTreeFileManagerTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/LSMTreeFileManagerTest.java
@@ -38,8 +38,10 @@
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.control.nc.io.IOManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFinalizer;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeFileManager;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
public class LSMTreeFileManagerTest {
@@ -49,12 +51,16 @@
protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
protected final static String sep = System.getProperty("file.separator");
protected IOManager ioManager;
+ protected IFileMapProvider fileMapProvider;
protected String baseDir;
+ // Dummy finalizer that always considers components valid. Just for testing.
+ protected ILSMComponentFinalizer componentFinalizer = new DummyLSMComponentFinalizer();
@Before
public void setUp() throws HyracksException {
TestStorageManagerComponentHolder.init(DEFAULT_PAGE_SIZE, DEFAULT_NUM_PAGES, DEFAULT_MAX_OPEN_FILES);
ioManager = TestStorageManagerComponentHolder.getIOManager();
+ fileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(null);
baseDir = "lsm_tree" + simpleDateFormat.format(new Date()) + sep;
File f = new File(baseDir);
f.mkdirs();
@@ -67,7 +73,7 @@
}
public void sortOrderTest(boolean testFlushFileName) throws InterruptedException, HyracksDataException {
- ILSMFileManager fileManager = new LSMTreeFileManager(ioManager, baseDir);
+ ILSMFileManager fileManager = new LSMTreeFileManager(ioManager, fileMapProvider, baseDir);
LinkedList<String> fileNames = new LinkedList<String>();
int numFileNames = 100;
@@ -105,7 +111,7 @@
}
public void cleanInvalidFilesTest(IOManager ioManager) throws InterruptedException, IOException {
- ILSMFileManager fileManager = new LSMTreeFileManager(ioManager, baseDir);
+ ILSMFileManager fileManager = new LSMTreeFileManager(ioManager, fileMapProvider, baseDir);
fileManager.createDirs();
List<FileReference> flushFiles = new ArrayList<FileReference>();
@@ -168,7 +174,8 @@
// Sort expected files.
Collections.sort(expectedValidFiles, fileManager.getFileNameComparator());
- List<Object> validFiles = fileManager.cleanupAndGetValidFiles();
+ // Pass null and a dummy component finalizer. We don't test for physical consistency in this test.
+ List<Object> validFiles = fileManager.cleanupAndGetValidFiles(null, componentFinalizer);
// Check actual files against expected files.
assertEquals(expectedValidFiles.size(), validFiles.size());
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeSearchCursorTest.java b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
index d167538..0631c20 100644
--- a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
+++ b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
@@ -115,7 +115,7 @@
IRTreeInteriorFrame interiorFrame = (IRTreeInteriorFrame) interiorFrameFactory.createFrame();
IRTreeLeafFrame leafFrame = (IRTreeLeafFrame) leafFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, rtreeFileId, 0, metaFrameFactory);
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
RTree rtree = new RTree(bufferCache, fieldCount, cmpFactories, freePageManager, interiorFrameFactory,
leafFrameFactory);