Major cleanup of lsm-btree. Added first simple insert+search test.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_lsm_tree@1044 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
index 19ab91f..49c58b3 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
@@ -809,7 +809,7 @@
}
}
- public final class BulkLoadContext implements IIndexBulkLoadContext {
+ public class BulkLoadContext implements IIndexBulkLoadContext {
public final int slotSize;
public final int leafMaxBytes;
public final int interiorMaxBytes;
@@ -1035,6 +1035,10 @@
return IndexType.BTREE;
}
+ public int getFileId() {
+ return fileId;
+ }
+
public byte getTreeHeight(IBTreeLeafFrame leafFrame) throws HyracksDataException {
ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
rootNode.acquireReadLatch();
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
index 8122314..1b92b85 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
@@ -62,7 +62,7 @@
}
@Override
- public void close() throws Exception {
+ public void close() throws HyracksDataException {
if (page != null) {
if (exclusiveLatchNodes) {
page.releaseWriteLatch();
@@ -100,7 +100,7 @@
}
@Override
- public boolean hasNext() throws Exception {
+ public boolean hasNext() throws HyracksDataException {
if (pred.isForward()) {
if (tupleIndex >= frame.getTupleCount()) {
int nextLeafPage = frame.getNextLeaf();
@@ -145,7 +145,7 @@
}
@Override
- public void next() throws Exception {
+ public void next() throws HyracksDataException {
tupleIndex += tupleIndexInc;
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexCursor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexCursor.java
index d3ce3867..24e552d 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexCursor.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexCursor.java
@@ -23,16 +23,16 @@
public interface ITreeIndexCursor {
public void reset();
- public boolean hasNext() throws Exception;
+ public boolean hasNext() throws HyracksDataException;
- public void next() throws Exception;
+ public void next() throws HyracksDataException;
public void open(ICursorInitialState initialState,
ISearchPredicate searchPred) throws HyracksDataException;
public ICachedPage getPage();
- public void close() throws Exception;
+ public void close() throws HyracksDataException;
public void setBufferCache(IBufferCache bufferCache);
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/TreeDiskOrderScanCursor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/TreeDiskOrderScanCursor.java
index 1a81c3f..15b63ec 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/TreeDiskOrderScanCursor.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/TreeDiskOrderScanCursor.java
@@ -43,7 +43,7 @@
}
@Override
- public void close() throws Exception {
+ public void close() throws HyracksDataException {
page.releaseReadLatch();
bufferCache.unpin(page);
page = null;
@@ -84,7 +84,7 @@
}
@Override
- public boolean hasNext() throws Exception {
+ public boolean hasNext() throws HyracksDataException {
if (currentPageId > maxPageId) {
return false;
}
@@ -102,7 +102,7 @@
}
@Override
- public void next() throws Exception {
+ public void next() throws HyracksDataException {
tupleIndex++;
}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/BTreeFactory.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/BTreeFactory.java
index 202ef68..cbe531a 100644
--- a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/BTreeFactory.java
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/BTreeFactory.java
@@ -30,5 +30,8 @@
interiorFrameFactory, leafFrameFactory);
}
-
+ public IBufferCache getBufferCache() {
+ return bufferCache;
+ }
+
}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/InDiskTreeInfo.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/InDiskTreeInfo.java
deleted file mode 100644
index e8d1137..0000000
--- a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/InDiskTreeInfo.java
+++ /dev/null
@@ -1,16 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.lsmtree.impls;
-
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-
-public class InDiskTreeInfo {
-
- private BTree bTree;
-
- public InDiskTreeInfo(BTree bTree) {
- this.bTree = bTree;
- }
-
- public BTree getBTree() {
- return bTree;
- }
-}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMPriorityQueueComparator.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMPriorityQueueComparator.java
index 1bb5abb..04281ac 100644
--- a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMPriorityQueueComparator.java
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMPriorityQueueComparator.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
package edu.uci.ics.hyracks.storage.am.lsmtree.impls;
import java.util.Comparator;
@@ -6,7 +22,7 @@
public class LSMPriorityQueueComparator implements Comparator<LSMPriorityQueueElement> {
- private MultiComparator cmp;
+ private final MultiComparator cmp;
public LSMPriorityQueueComparator(MultiComparator cmp) {
this.cmp = cmp;
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTree.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTree.java
index 7991a37..dd3d6b4 100644
--- a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTree.java
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTree.java
@@ -1,18 +1,27 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.lsmtree.impls;
-import java.io.ByteArrayInputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
import java.io.File;
import java.util.LinkedList;
import java.util.ListIterator;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeDuplicateKeyException;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
@@ -29,112 +38,107 @@
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsmtree.common.api.ILSMTree;
import edu.uci.ics.hyracks.storage.am.lsmtree.common.freepage.InMemoryFreePageManager;
-import edu.uci.ics.hyracks.storage.am.lsmtree.tuples.LSMTypeAwareTupleReference;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapManager;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-public class LSMTree implements ITreeIndex {
+public class LSMTree implements ITreeIndex, ILSMTree {
+ // In-memory components.
+ private final BTree memBTree;
+ private final InMemoryFreePageManager memFreePageManager;
- private final IBufferCache bufferCache;
- private BTree memBTree;
- private String fileName;
- private int fileId;
- private boolean created;
-
- private final InMemoryFreePageManager memFreePageManager;
+ // On-disk components.
+ private final String onDiskDir;
+ private final BTreeFactory diskBTreeFactory;
+ private final IBufferCache diskBufferCache;
+ private final IFileMapProvider diskFileMapProvider;
+ private LinkedList<BTree> onDiskBTrees = new LinkedList<BTree>();
+ private LinkedList<BTree> mergedBTrees = new LinkedList<BTree>();
+ private int onDiskBTreeCount;
+
+ // Common for in-memory and on-disk components.
private final ITreeIndexFrameFactory interiorFrameFactory;
private final ITreeIndexFrameFactory insertLeafFrameFactory;
private final ITreeIndexFrameFactory deleteLeafFrameFactory;
private final MultiComparator cmp;
-
- // TODO: change to private, it's public only for LSMTreeSearchTest
- public LinkedList<InDiskTreeInfo> inDiskTreeInfoList;
- private LinkedList<InDiskTreeInfo> mergedInDiskTreeInfoList;
- private int inDiskTreeCounter;
- private final BTreeFactory bTreeFactory;
- private final IFileMapManager fileMapManager;
+
+ // For dealing with concurrent accesses.
private int threadRefCount;
private boolean flushFlag;
- public LSMTree(IBufferCache memCache, IBufferCache bufferCache, int fieldCount, MultiComparator cmp,
- InMemoryFreePageManager memFreePageManager, ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory,
- BTreeFactory bTreeFactory, IFileMapManager fileMapManager) {
- this.bufferCache = bufferCache;
- this.cmp = cmp;
+ public LSMTree(IBufferCache memBufferCache, InMemoryFreePageManager memFreePageManager,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory,
+ ITreeIndexFrameFactory deleteLeafFrameFactory, String onDiskDir, BTreeFactory diskBTreeFactory,
+ IFileMapProvider diskFileMapProvider, int fieldCount, MultiComparator cmp) {
+ memBTree = new BTree(memBufferCache, fieldCount, cmp, memFreePageManager, interiorFrameFactory,
+ insertLeafFrameFactory);
+ this.memFreePageManager = memFreePageManager;
this.interiorFrameFactory = interiorFrameFactory;
this.insertLeafFrameFactory = insertLeafFrameFactory;
this.deleteLeafFrameFactory = deleteLeafFrameFactory;
- this.memFreePageManager = memFreePageManager;
- this.bTreeFactory = bTreeFactory;
- this.inDiskTreeInfoList = new LinkedList<InDiskTreeInfo>();
- this.inDiskTreeCounter = 0;
- this.fileMapManager = fileMapManager;
+ this.diskBufferCache = diskBTreeFactory.getBufferCache();
+ this.diskFileMapProvider = diskFileMapProvider;
+ this.diskBTreeFactory = diskBTreeFactory;
+ this.cmp = cmp;
+ this.onDiskBTrees = new LinkedList<BTree>();
+ this.onDiskBTreeCount = 0;
this.threadRefCount = 0;
- this.created = false;
this.flushFlag = false;
-
- try {
- this.fileName = this.fileMapManager.lookupFileName(this.fileId).toString();
- } catch (Exception e) {
- e.printStackTrace();
+ if (!onDiskDir.endsWith(System.getProperty("file.separator"))) {
+ onDiskDir += System.getProperty("file.separator");
}
-
- memBTree = new BTree(memCache, fieldCount, cmp, memFreePageManager, interiorFrameFactory,
- insertLeafFrameFactory);
+ this.onDiskDir = onDiskDir;
}
@Override
public void create(int indexFileId) throws HyracksDataException {
- if (created) {
- return;
- } else {
- memBTree.create(indexFileId);
- this.fileId = indexFileId;
- created = true;
- }
+ memBTree.create(indexFileId);
}
@Override
public void open(int indexFileId) {
memBTree.open(indexFileId);
- this.fileId = indexFileId;
}
@Override
public void close() {
memBTree.close();
- this.fileId = -1;
}
- private void lsmPerformOp(ITupleReference tuple, LSMTreeOpContext ctx) throws Exception {
- boolean waitForFlush = false;
- do {
- synchronized (this) {
- if (!flushFlag) {
- threadEnter();
- waitForFlush = false;
- }
- }
- } while (waitForFlush == true);
- try {
- ctx.memBtreeAccessor.insert(tuple);
- } catch (BTreeDuplicateKeyException e) {
- ctx.reset(IndexOp.UPDATE);
- // We don't need to deal with a nonexistent key here, because a
- // deleter will actually update the key and it's value, and not
- // delete it from the BTree.
- ctx.memBtreeAccessor.update(tuple);
- }
- threadExit();
- }
+ private void lsmPerformOp(ITupleReference tuple, LSMTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
+ boolean waitForFlush = true;
+ do {
+ // Wait for ongoing flush to complete.
+ synchronized (this) {
+ if (!flushFlag) {
+ // Increments threadRefCount, to force a flush to wait for this operation to finish.
+ // (a flush can only begin once threadRefCount == 0).
+ threadEnter();
+ // Proceed with operation.
+ waitForFlush = false;
+ }
+ }
+ } while (waitForFlush);
+ try {
+ ctx.memBtreeAccessor.insert(tuple);
+ } catch (BTreeDuplicateKeyException e) {
+ // We don't need to deal with a nonexistent key here, because a
+ // deleter will actually update the key and it's value, and not
+ // delete it from the BTree.
+ // Also notice that a flush must wait for the current operation to
+ // finish (threadRefCount must reach zero).
+ ctx.reset(IndexOp.UPDATE);
+ ctx.memBtreeAccessor.update(tuple);
+ }
+ threadExit();
+ }
public void threadEnter() {
threadRefCount++;
}
- public void threadExit() throws Exception {
+ public void threadExit() throws HyracksDataException, TreeIndexException {
synchronized (this) {
threadRefCount--;
// Check if we've reached or exceeded the maximum number of pages.
@@ -143,199 +147,135 @@
}
// Flush will only be handled by last exiting thread.
if (flushFlag && threadRefCount == 0) {
- flushInMemoryBtree();
+ flush();
flushFlag = false;
}
}
}
- private String getNextFileName() {
- int newDiskBTreeId = inDiskTreeCounter++;
- String LSMFileName = new String(this.fileName);
- return LSMFileName.concat("-" + Integer.toString(newDiskBTreeId));
- }
-
- public void flushInMemoryBtree() throws Exception {
- // read the tuple from InMemoryBtree, and bulkload into the disk
-
- // scan
+ @Override
+ public void flush() throws HyracksDataException, TreeIndexException {
+ System.out.println("FLUSHING!");
+ // Bulk load a new on-disk BTree from the in-memory BTree.
ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(
(IBTreeLeafFrame) insertLeafFrameFactory.createFrame(), false);
RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
- ITreeIndexAccessor memBtreeAccessor = memBTree.createAccessor();
- memBtreeAccessor.search(scanCursor, nullPred);
-
- // Create a new in-Disk BTree, which have full fillfactor.
-
- // Register the BTree information into system.
- FileReference file = new FileReference(new File(getNextFileName()));
- // TODO: Delete the file during cleanup.
- bufferCache.createFile(file);
- int newDiskBTreeId = fileMapManager.lookupFileId(file);
- // TODO: Close the file during cleanup.
- bufferCache.openFile(newDiskBTreeId);
-
- // Create new in-Disk Btree.
- BTree inDiskBTree = this.bTreeFactory.createBTreeInstance(newDiskBTreeId);
- inDiskBTree.create(newDiskBTreeId);
- // TODO: Close the BTree during cleanup.
- inDiskBTree.open(newDiskBTreeId);
-
- // BulkLoad the tuples from the in-memory tree into the new disk BTree.
- IIndexBulkLoadContext bulkLoadCtx = inDiskBTree.beginBulkLoad(1.0f);
+ ITreeIndexAccessor memBTreeAccessor = memBTree.createAccessor();
+ memBTreeAccessor.search(scanCursor, nullPred);
+ BTree diskBTree = createDiskBTree();
+ // Bulk load the tuples from the in-memory BTree into the new disk BTree.
+ IIndexBulkLoadContext bulkLoadCtx = diskBTree.beginBulkLoad(1.0f);
try {
while (scanCursor.hasNext()) {
scanCursor.next();
ITupleReference frameTuple = scanCursor.getTuple();
- inDiskBTree.bulkLoadAddTuple(frameTuple, bulkLoadCtx);
+ diskBTree.bulkLoadAddTuple(frameTuple, bulkLoadCtx);
}
} finally {
scanCursor.close();
}
- inDiskBTree.endBulkLoad(bulkLoadCtx);
-
- // After BulkLoading, Clear the in-memTree
- resetInMemoryTree();
-
- InDiskTreeInfo newLinkedListNode = new InDiskTreeInfo(inDiskBTree);
- synchronized (inDiskTreeInfoList) {
- inDiskTreeInfoList.addFirst(newLinkedListNode);
- }
+ diskBTree.endBulkLoad(bulkLoadCtx);
+ resetMemBTree();
+ onDiskBTrees.addFirst(diskBTree);
}
- private void insert(ITupleReference tuple, LSMTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
- try {
- lsmPerformOp(tuple, ctx);
- } catch (Exception e) {
- e.printStackTrace();
+ private void resetMemBTree() throws HyracksDataException {
+ memFreePageManager.reset();
+ memBTree.create(memBTree.getFileId());
+ }
+
+ private String getNextFileName() {
+ return onDiskDir + "btree_" + onDiskBTreeCount++;
+ }
+
+ private BTree createDiskBTree() throws HyracksDataException {
+ // Register the new BTree file.
+ FileReference file = new FileReference(new File(getNextFileName()));
+ // TODO: Delete the file during cleanup.
+ diskBufferCache.createFile(file);
+ int diskBTreeFileId = diskFileMapProvider.lookupFileId(file);
+ // TODO: Close the file during cleanup.
+ diskBufferCache.openFile(diskBTreeFileId);
+ // Create new BTree instance.
+ BTree diskBTree = diskBTreeFactory.createBTreeInstance(diskBTreeFileId);
+ diskBTree.create(diskBTreeFileId);
+ // TODO: Close the BTree during cleanup.
+ diskBTree.open(diskBTreeFileId);
+ return diskBTree;
+ }
+
+ private void search(ITreeIndexCursor cursor, RangePredicate pred, LSMTreeOpContext ctx, boolean includeMemBTree) throws HyracksDataException, TreeIndexException {
+ boolean waitForFlush = true;
+ do {
+ synchronized (this) {
+ if (!flushFlag) {
+ // The corresponding threadExit() is in LSMTreeRangeSearchCursor.close().
+ threadEnter();
+ waitForFlush = false;
+ }
+ }
+ } while (waitForFlush);
+
+ // TODO: Think about what happens with possibly concurrent merges.
+ LSMTreeRangeSearchCursor lsmTreeCursor = (LSMTreeRangeSearchCursor) cursor;
+ int numDiskBTrees = onDiskBTrees.size();
+ int numBTrees = (includeMemBTree) ? numDiskBTrees + 1 : numDiskBTrees;
+ ListIterator<BTree> diskBTreesIter = onDiskBTrees.listIterator();
+ LSMTreeCursorInitialState initialState = new LSMTreeCursorInitialState(numBTrees,
+ insertLeafFrameFactory, cmp, this);
+ lsmTreeCursor.open(initialState, pred);
+
+ int cursorIx;
+ if (includeMemBTree) {
+ // Open cursor of in-memory BTree at index 0.
+ ctx.memBtreeAccessor.search(lsmTreeCursor.getCursor(0), pred);
+ // Skip 0 because it is the in-memory BTree.
+ cursorIx = 1;
+ } else {
+ cursorIx = 0;
}
+
+ // Open cursors of on-disk BTrees.
+ ITreeIndexAccessor[] diskBTreeAccessors = new ITreeIndexAccessor[numDiskBTrees];
+ int diskBTreeIx = 0;
+ while(diskBTreesIter.hasNext()) {
+ BTree diskBTree = diskBTreesIter.next();
+ diskBTreeAccessors[diskBTreeIx] = diskBTree.createAccessor();
+ diskBTreeAccessors[diskBTreeIx].search(lsmTreeCursor.getCursor(cursorIx), pred);
+ cursorIx++;
+ diskBTreeIx++;
+ }
+ LSMPriorityQueueComparator LSMPriorityQueueCmp = new LSMPriorityQueueComparator(cmp);
+ lsmTreeCursor.initPriorityQueue(LSMPriorityQueueCmp);
+ }
+
+ private void insert(ITupleReference tuple, LSMTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
+ lsmPerformOp(tuple, ctx);
}
private void delete(ITupleReference tuple, LSMTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
- try {
- lsmPerformOp(tuple, ctx);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public IIndexBulkLoadContext beginBulkLoad(float fillFactor) throws TreeIndexException, HyracksDataException {
- return null;
- }
-
- @Override
- public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException {
- }
-
- @Override
- public void endBulkLoad(IIndexBulkLoadContext ictx) throws HyracksDataException {
- }
-
- public void search(ITreeIndexCursor cursor, RangePredicate pred, LSMTreeOpContext ctx) throws Exception {
- int numberOfInDiskTrees;
- ListIterator<InDiskTreeInfo> inDiskTreeInfoListIterator;
- boolean continuePerformOp = false;
-
- ctx.reset(IndexOp.SEARCH);
-
- while (continuePerformOp == false) {
- synchronized (this) {
- if (!flushFlag) {
- threadRefCount++;
- continuePerformOp = true;
- }
- }
- }
-
- // in-disk
- synchronized (inDiskTreeInfoList) {
- numberOfInDiskTrees = inDiskTreeInfoList.size();
- inDiskTreeInfoListIterator = inDiskTreeInfoList.listIterator();
- }
-
- LSMTreeCursorInitialState initialState = new LSMTreeCursorInitialState(numberOfInDiskTrees + 1,
- insertLeafFrameFactory, cmp, this);
- cursor.open(initialState, pred);
-
- BTree[] onDiskBtrees = new BTree[numberOfInDiskTrees];
- ITreeIndexAccessor[] onDiskBtreeAccessors = new ITreeIndexAccessor[numberOfInDiskTrees];
-
- for (int i = 0; i < numberOfInDiskTrees; i++) {
- // get btree instances for in-disk trees
- if (inDiskTreeInfoListIterator.hasNext()) {
- onDiskBtrees[i] = ((InDiskTreeInfo) inDiskTreeInfoListIterator.next()).getBTree();
- } else {
- throw new HyracksDataException("Cannot find in-disk tree instance");
- }
- onDiskBtreeAccessors[i] = onDiskBtrees[i].createAccessor();
- onDiskBtreeAccessors[i].search(((LSMTreeRangeSearchCursor) cursor).getCursor(i + 1), pred);
- }
-
- // in-memory
- ctx.memBtreeAccessor.search(((LSMTreeRangeSearchCursor) cursor).getCursor(0), pred);
-
- LSMPriorityQueueComparator LSMPriorityQueueCmp = new LSMPriorityQueueComparator(cmp);
- ((LSMTreeRangeSearchCursor) cursor).initPriorityQueue(numberOfInDiskTrees + 1, LSMPriorityQueueCmp);
+ lsmPerformOp(tuple, ctx);
}
public void merge() throws Exception {
- ITreeIndexCursor rangeCursor = new LSMTreeRangeSearchCursor();
+ LSMTreeOpContext ctx = createOpContext();
+ ITreeIndexCursor cursor = new LSMTreeRangeSearchCursor();
RangePredicate rangePred = new RangePredicate(true, null, null, true, true, null, null);
-
- // Cursor setting -- almost the same as search, only difference is
- // "no cursor for in-memory tree"
- int numberOfInDiskTrees;
- ListIterator<InDiskTreeInfo> inDiskTreeInfoListIterator;
- boolean continuePerformOp = false;
- while (continuePerformOp == false) {
- synchronized (this) {
- if (!flushFlag) {
- threadRefCount++;
- continuePerformOp = true;
- }
- }
- }
-
- synchronized (inDiskTreeInfoList) {
- numberOfInDiskTrees = inDiskTreeInfoList.size();
- inDiskTreeInfoListIterator = inDiskTreeInfoList.listIterator();
- }
-
- LSMTreeCursorInitialState initialState = new LSMTreeCursorInitialState(numberOfInDiskTrees,
- insertLeafFrameFactory, cmp, this);
- rangeCursor.open(initialState, rangePred);
-
- BTree[] onDiskBtrees = new BTree[numberOfInDiskTrees];
- ITreeIndexAccessor[] onDiskBtreeAccessors = new ITreeIndexAccessor[numberOfInDiskTrees];
-
- for (int i = 0; i < numberOfInDiskTrees; i++) {
- // get btree instances for in-disk trees
- if (inDiskTreeInfoListIterator.hasNext()) {
- onDiskBtrees[i] = ((InDiskTreeInfo) inDiskTreeInfoListIterator.next()).getBTree();
- } else {
- throw new Exception("Cannot find in-disk tree instance");
- }
- onDiskBtreeAccessors[i] = onDiskBtrees[i].createAccessor();
- onDiskBtreeAccessors[i].search(((LSMTreeRangeSearchCursor) rangeCursor).getCursor(i), rangePred);
- }
-
- LSMPriorityQueueComparator LSMPriorityQueueCmp = new LSMPriorityQueueComparator(cmp);
- ((LSMTreeRangeSearchCursor) rangeCursor).initPriorityQueue(numberOfInDiskTrees, LSMPriorityQueueCmp);
- // End of Cursor setting
+ // Ordered scan, ignoring the in-memory BTree.
+ search(cursor, (RangePredicate) rangePred, ctx, false);
// Create a new Merged BTree, which have full fillfactor.
// Register the BTree information into system.
// TODO: change the naming schema for the new tree
FileReference file = new FileReference(new File(getNextFileName()));
// TODO: Delete the file during cleanup.
- bufferCache.createFile(file);
- int mergedBTreeId = fileMapManager.lookupFileId(file);
+ diskBufferCache.createFile(file);
+ int mergedBTreeId = diskFileMapProvider.lookupFileId(file);
// TODO: Close the file during cleanup.
- bufferCache.openFile(mergedBTreeId);
+ diskBufferCache.openFile(mergedBTreeId);
// Create new in-Disk BTree.
- BTree mergedBTree = this.bTreeFactory.createBTreeInstance(mergedBTreeId);
+ BTree mergedBTree = this.diskBTreeFactory.createBTreeInstance(mergedBTreeId);
mergedBTree.create(mergedBTreeId);
// TODO: Close the BTree during cleanup.
mergedBTree.open(mergedBTreeId);
@@ -344,34 +284,32 @@
IIndexBulkLoadContext bulkLoadCtx = mergedBTree.beginBulkLoad(1.0f);
try {
- while (rangeCursor.hasNext()) {
- rangeCursor.next();
- ITupleReference frameTuple = rangeCursor.getTuple();
+ while (cursor.hasNext()) {
+ cursor.next();
+ ITupleReference frameTuple = cursor.getTuple();
mergedBTree.bulkLoadAddTuple(frameTuple, bulkLoadCtx);
}
} finally {
- rangeCursor.close();
+ cursor.close();
}
-
mergedBTree.endBulkLoad(bulkLoadCtx);
- InDiskTreeInfo newLinkedListNode = new InDiskTreeInfo(mergedBTree);
- LinkedList<InDiskTreeInfo> tempInDiskTreeInfo;
- synchronized (inDiskTreeInfoList) {
- mergedInDiskTreeInfoList = (LinkedList<InDiskTreeInfo>) inDiskTreeInfoList.clone();
+ /*
+ synchronized (onDiskBTrees) {
+ mergedBTrees = (LinkedList<BTree>) onDiskBTrees.clone();
// Remove the redundant trees, and add the new merged tree in the
// last off the list
for (int i = 0; i < numberOfInDiskTrees; i++) {
- mergedInDiskTreeInfoList.removeLast();
+ mergedBTrees.removeLast();
}
- mergedInDiskTreeInfoList.addLast(newLinkedListNode);
+ mergedBTrees.addLast(mergedBTree);
// TODO: to swap the linkedlists
- /*
- * tempInDiskTreeInfo = inDiskTreeInfoList; inDiskTreeInfoList =
- * mergedInDiskTreeInfoList; mergedInDiskTreeInfoList =
- * tempInDiskTreeInfo;
- */
+ //
+ // tempInDiskTreeInfo = inDiskTreeInfoList; inDiskTreeInfoList =
+ // mergedInDiskTreeInfoList; mergedInDiskTreeInfoList =
+ // tempInDiskTreeInfo;
+ //
// TODO: to swap the searchThreadCounters
// 1. should add the searcherReferenceCounter
@@ -382,6 +320,49 @@
}
// TODO: to wake up the cleaning thread
+ */
+ }
+
+ public class LSMTreeBulkLoadContext implements IIndexBulkLoadContext {
+ private final BTree btree;
+ private IIndexBulkLoadContext bulkLoadCtx;
+
+ public LSMTreeBulkLoadContext(BTree btree) {
+ this.btree = btree;
+ }
+
+ public void beginBulkLoad(float fillFactor) throws HyracksDataException, TreeIndexException {
+ bulkLoadCtx = btree.beginBulkLoad(fillFactor);
+ }
+
+ public BTree getBTree() {
+ return btree;
+ }
+
+ public IIndexBulkLoadContext getBulkLoadCtx() {
+ return bulkLoadCtx;
+ }
+ }
+
+ @Override
+ public IIndexBulkLoadContext beginBulkLoad(float fillFactor) throws TreeIndexException, HyracksDataException {
+ BTree diskBTree = createDiskBTree();
+ LSMTreeBulkLoadContext bulkLoadCtx = new LSMTreeBulkLoadContext(diskBTree);
+ bulkLoadCtx.beginBulkLoad(fillFactor);
+ return bulkLoadCtx;
+ }
+
+ @Override
+ public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException {
+ LSMTreeBulkLoadContext bulkLoadCtx = (LSMTreeBulkLoadContext) ictx;
+ bulkLoadCtx.getBTree().bulkLoadAddTuple(tuple, bulkLoadCtx.getBulkLoadCtx());
+ }
+
+ @Override
+ public void endBulkLoad(IIndexBulkLoadContext ictx) throws HyracksDataException {
+ LSMTreeBulkLoadContext bulkLoadCtx = (LSMTreeBulkLoadContext) ictx;
+ bulkLoadCtx.getBTree().endBulkLoad(bulkLoadCtx.getBulkLoadCtx());
+ onDiskBTrees.addFirst(bulkLoadCtx.getBTree());
}
@Override
@@ -414,96 +395,6 @@
return memBTree.getIndexType();
}
- public void resetInMemoryTree() throws HyracksDataException {
- ((InMemoryFreePageManager) memFreePageManager).reset();
- memBTree.create(fileId);
- }
-
- // This function is just for testing flushInMemoryBtree()
- public void scanDiskTree(int treeIndex) throws Exception {
- BTree onDiskBtree = inDiskTreeInfoList.get(treeIndex).getBTree();
- ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(
- (IBTreeLeafFrame) this.insertLeafFrameFactory.createFrame(), false);
- RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
- ITreeIndexAccessor onDiskBtreeAccessor = onDiskBtree.createAccessor();
- onDiskBtreeAccessor.search(scanCursor, nullPred);
- try {
- int scanTupleIndex = 0;
- while (scanCursor.hasNext()) {
- scanCursor.hasNext();
- scanCursor.next();
- ITupleReference frameTuple = scanCursor.getTuple();
- int numPrintFields = Math.min(frameTuple.getFieldCount(), recDescSers.length);
-
- for (int i = 0; i < numPrintFields; i++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(frameTuple.getFieldData(i),
- frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
- DataInput dataIn = new DataInputStream(inStream);
- Object o = recDescSers[i].deserialize(dataIn);
-
- if (i == 1)
- System.out.printf("LSMTree.scanDiskTree(): scanTupleIndex[%d]; Value is [%d]; ",
- scanTupleIndex, Integer.parseInt(o.toString()));
-
- }
-
- if (((LSMTypeAwareTupleReference) frameTuple).isDelete()) {
- System.out.printf(" DELETE\n");
- } else {
- System.out.printf(" INSERT\n");
- }
- scanTupleIndex++;
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- scanCursor.close();
- }
- }
-
- // This function is just for testing flushInMemoryBtree()
- public void scanInMemoryTree() throws Exception {
- ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(
- (IBTreeLeafFrame) this.insertLeafFrameFactory.createFrame(), false);
- RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
- ITreeIndexAccessor inMemBtreeAccessor = memBTree.createAccessor();
- inMemBtreeAccessor.search(scanCursor, nullPred);
- try {
- int scanTupleIndex = 0;
- while (scanCursor.hasNext()) {
- scanCursor.hasNext();
- scanCursor.next();
- ITupleReference frameTuple = scanCursor.getTuple();
- int numPrintFields = Math.min(frameTuple.getFieldCount(), recDescSers.length);
- for (int i = 0; i < numPrintFields; i++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(frameTuple.getFieldData(i),
- frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
- DataInput dataIn = new DataInputStream(inStream);
- Object o = recDescSers[i].deserialize(dataIn);
- if (i == 1)
- System.out.printf("LSMTree.scanMemoryTree(): scanTupleIndex[%d]; Value is [%d]; ",
- scanTupleIndex, Integer.parseInt(o.toString()));
- }
- if (((LSMTypeAwareTupleReference) frameTuple).isDelete()) {
- System.out.printf(" DELETE\n");
- } else {
- System.out.printf(" INSERT\n");
- }
- scanTupleIndex++;
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- scanCursor.close();
- }
- }
-
public LSMTreeOpContext createOpContext() {
return new LSMTreeOpContext((BTree.BTreeAccessor) memBTree.createAccessor(), insertLeafFrameFactory,
deleteLeafFrameFactory, interiorFrameFactory, memFreePageManager.getMetaDataFrameFactory()
@@ -545,12 +436,7 @@
public void search(ITreeIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException,
TreeIndexException {
ctx.reset(IndexOp.SEARCH);
- // TODO: fix exception handling throughout LSM tree.
- try {
- lsmTree.search(cursor, (RangePredicate) searchPred, ctx);
- } catch (Exception e) {
- throw new HyracksDataException(e);
- }
+ lsmTree.search(cursor, (RangePredicate) searchPred, ctx, true);
}
@Override
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTreeCursorInitialState.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTreeCursorInitialState.java
index 5391d9c..6f2cde2 100644
--- a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTreeCursorInitialState.java
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTreeCursorInitialState.java
@@ -7,20 +7,20 @@
public class LSMTreeCursorInitialState implements ICursorInitialState {
- private int numberOfTrees;
+ private int numBTrees;
private ITreeIndexFrameFactory leafFrameFactory;
private MultiComparator cmp;
private LSMTree lsm;
- public LSMTreeCursorInitialState(int numberOfTrees, ITreeIndexFrameFactory leafFrameFactory, MultiComparator cmp, LSMTree lsm) {
- this.numberOfTrees = numberOfTrees;
+ public LSMTreeCursorInitialState(int numBTrees, ITreeIndexFrameFactory leafFrameFactory, MultiComparator cmp, LSMTree lsm) {
+ this.numBTrees = numBTrees;
this.leafFrameFactory = leafFrameFactory;
this.cmp = cmp;
this.lsm = lsm;
}
- public int getNumberOfTrees() {
- return numberOfTrees;
+ public int getNumBTrees() {
+ return numBTrees;
}
public ITreeIndexFrameFactory getLeafFrameFactory() {
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTreeRangeSearchCursor.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTreeRangeSearchCursor.java
index 03253c3..2e9a500 100644
--- a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTreeRangeSearchCursor.java
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTreeRangeSearchCursor.java
@@ -9,6 +9,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.lsmtree.tuples.LSMTypeAwareTupleReference;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
@@ -21,19 +22,18 @@
private MultiComparator cmp;
private LSMPriorityQueueElement outputElement;
private LSMPriorityQueueElement reusedElement;
- private int numberOfTrees;
private boolean needPush;
- private LSMTree lsm;
+ private LSMTree lsmTree;
public LSMTreeRangeSearchCursor() {
outputElement = null;
needPush = false;
}
- public void initPriorityQueue(int numberOfTrees, LSMPriorityQueueComparator LSMPriorityQueueCmp) throws Exception {
- this.numberOfTrees = numberOfTrees;
- outputPriorityQueue = new PriorityQueue<LSMPriorityQueueElement>(numberOfTrees, LSMPriorityQueueCmp);
- for (int i = 0; i < numberOfTrees; i++) {
+ public void initPriorityQueue(LSMPriorityQueueComparator LSMPriorityQueueCmp)
+ throws HyracksDataException {
+ outputPriorityQueue = new PriorityQueue<LSMPriorityQueueElement>(rangeCursors.length, LSMPriorityQueueCmp);
+ for (int i = 0; i < rangeCursors.length; i++) {
LSMPriorityQueueElement element;
if (rangeCursors[i].hasNext()) {
rangeCursors[i].next();
@@ -54,32 +54,30 @@
}
@Override
- public boolean hasNext() throws Exception {
+ public boolean hasNext() throws HyracksDataException {
checkPriorityQueue();
return !outputPriorityQueue.isEmpty();
}
@Override
- public void next() throws Exception {
-
+ public void next() throws HyracksDataException {
outputElement = outputPriorityQueue.poll();
needPush = true;
if (outputElement == null) {
- throw new Exception("The outputPriorityQueue is empty");
+ throw new HyracksDataException("The outputPriorityQueue is empty");
}
}
@Override
public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
-
- lsm = ((LSMTreeCursorInitialState) initialState).getLsm();
- cmp = ((LSMTreeCursorInitialState) initialState).getCmp();
-
- rangeCursors = new BTreeRangeSearchCursor[((LSMTreeCursorInitialState) initialState).getNumberOfTrees()];
-
- for (int i = 0; i < ((LSMTreeCursorInitialState) initialState).getNumberOfTrees(); i++) {
- rangeCursors[i] = new BTreeRangeSearchCursor((IBTreeLeafFrame) ((LSMTreeCursorInitialState) initialState)
- .getLeafFrameFactory().createFrame(), false);
+ LSMTreeCursorInitialState lsmInitialState = (LSMTreeCursorInitialState) initialState;
+ lsmTree = lsmInitialState.getLsm();
+ cmp = lsmInitialState.getCmp();
+ int numBTrees = lsmInitialState.getNumBTrees();
+ rangeCursors = new BTreeRangeSearchCursor[numBTrees];
+ for (int i = 0; i < numBTrees; i++) {
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) lsmInitialState.getLeafFrameFactory().createFrame();
+ rangeCursors[i] = new BTreeRangeSearchCursor(leafFrame, false);
}
}
@@ -90,13 +88,17 @@
}
@Override
- public void close() throws Exception {
- lsm.threadExit();
+ public void close() throws HyracksDataException {
outputPriorityQueue.clear();
- for (int i = 0; i < numberOfTrees; i++) {
+ for (int i = 0; i < rangeCursors.length; i++) {
rangeCursors[i].close();
}
rangeCursors = null;
+ try {
+ lsmTree.threadExit();
+ } catch (TreeIndexException e) {
+ throw new HyracksDataException(e);
+ }
}
@Override
@@ -114,8 +116,7 @@
return (ITupleReference) outputElement.getTuple();
}
- private void pushIntoPriorityQueue(int cursorIndex) throws Exception {
-
+ private void pushIntoPriorityQueue(int cursorIndex) throws HyracksDataException {
if (rangeCursors[cursorIndex].hasNext()) {
rangeCursors[cursorIndex].next();
reusedElement.reset(rangeCursors[cursorIndex].getTuple(), cursorIndex);
@@ -123,7 +124,7 @@
}
}
- private void checkPriorityQueue() throws Exception {
+ private void checkPriorityQueue() throws HyracksDataException {
while (!outputPriorityQueue.isEmpty() || needPush == true) {
if (!outputPriorityQueue.isEmpty()) {
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/ILSMTreeTupleReference.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/ILSMTreeTupleReference.java
index 68f2672..54b9fcc 100644
--- a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/ILSMTreeTupleReference.java
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/ILSMTreeTupleReference.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.lsmtree.tuples;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
index ee0cb20..32138db 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
@@ -56,7 +56,7 @@
}
@Override
- public void close() throws Exception {
+ public void close() throws HyracksDataException {
if (readLatched) {
page.releaseReadLatch();
bufferCache.unpin(page);
@@ -142,7 +142,7 @@
}
@Override
- public boolean hasNext() throws Exception {
+ public boolean hasNext() throws HyracksDataException {
if (page == null) {
return false;
}
@@ -172,7 +172,7 @@
}
@Override
- public void next() throws Exception {
+ public void next() throws HyracksDataException {
tupleIndex = tupleIndexInc;
}
diff --git a/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/AbstractLSMTreeTest.java b/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/AbstractLSMTreeTest.java
index 53ce238..3c8f9f7 100644
--- a/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/AbstractLSMTreeTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/AbstractLSMTreeTest.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.lsmtree.btree;
import java.io.File;
@@ -11,7 +26,8 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.storage.am.lsmtree.common.freepage.InMemoryBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
@@ -19,58 +35,65 @@
public abstract class AbstractLSMTreeTest {
protected static final Logger LOGGER = Logger.getLogger(AbstractLSMTreeTest.class.getName());
- public static final long RANDOM_SEED = 50;
-
- private static final int PAGE_SIZE = 256;
- private static final int NUM_PAGES = 10;
- private static final int MAX_OPEN_FILES = 10;
+ private static final long RANDOM_SEED = 50;
+ private static final int DISK_PAGE_SIZE = 256;
+ private static final int DISK_NUM_PAGES = 10;
+ private static final int DISK_MAX_OPEN_FILES = 100;
+ private static final int MEM_PAGE_SIZE = 256;
+ private static final int MEM_NUM_PAGES = 10;
private static final int HYRACKS_FRAME_SIZE = 128;
-
+ protected static final int dummyFileId = -1;
+
+ protected IBufferCache diskBufferCache;
+ protected IFileMapProvider diskFileMapProvider;
+ protected InMemoryBufferCache memBufferCache;
protected IHyracksTaskContext ctx;
- protected IBufferCache bufferCache;
- protected int btreeFileId;
protected final Random rnd = new Random();
protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
protected final static String tmpDir = System.getProperty("java.io.tmpdir");
protected final static String sep = System.getProperty("file.separator");
- protected String fileName;
+ protected String onDiskDir;
@Before
public void setUp() throws HyracksDataException {
- fileName = tmpDir + sep + simpleDateFormat.format(new Date());
+ onDiskDir = tmpDir + sep + "lsm_btree" + simpleDateFormat.format(new Date()) + sep;
ctx = TestUtils.create(getHyracksFrameSize());
- TestStorageManagerComponentHolder.init(getPageSize(), getNumPages(), getMaxOpenFiles());
- bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- btreeFileId = fmp.lookupFileId(file);
- bufferCache.openFile(btreeFileId);
+ TestStorageManagerComponentHolder.init(getDiskPageSize(), getDiskNumPages(), getDiskMaxOpenFiles());
+ diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ diskFileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), getMemPageSize(), getDiskPageSize());
rnd.setSeed(RANDOM_SEED);
}
@After
public void tearDown() throws HyracksDataException {
- bufferCache.closeFile(btreeFileId);
- bufferCache.close();
- File f = new File(fileName);
+ diskBufferCache.close();
+ File f = new File(onDiskDir);
f.deleteOnExit();
}
- public int getPageSize() {
- return PAGE_SIZE;
+ public int getDiskPageSize() {
+ return DISK_PAGE_SIZE;
}
- public int getNumPages() {
- return NUM_PAGES;
+ public int getDiskNumPages() {
+ return DISK_NUM_PAGES;
+ }
+
+ public int getDiskMaxOpenFiles() {
+ return DISK_MAX_OPEN_FILES;
+ }
+
+ public int getMemPageSize() {
+ return MEM_PAGE_SIZE;
+ }
+
+ public int getMemNumPages() {
+ return MEM_NUM_PAGES;
}
public int getHyracksFrameSize() {
return HYRACKS_FRAME_SIZE;
}
-
- public int getMaxOpenFiles() {
- return MAX_OPEN_FILES;
- }
}
diff --git a/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/LSMTreeDeleteTest.java b/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/LSMTreeDeleteTest.java
deleted file mode 100644
index 7fd913f..0000000
--- a/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/LSMTreeDeleteTest.java
+++ /dev/null
@@ -1,1090 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.lsmtree.btree;
-
-import static org.junit.Assert.fail;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.File;
-import java.nio.ByteBuffer;
-
-import org.junit.Test;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
-import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
-import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.lsmtree.common.freepage.InMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsmtree.common.freepage.InMemoryFreePageManager;
-import edu.uci.ics.hyracks.storage.am.lsmtree.impls.BTreeFactory;
-import edu.uci.ics.hyracks.storage.am.lsmtree.impls.LSMTree;
-import edu.uci.ics.hyracks.storage.am.lsmtree.impls.LSMTreeRangeSearchCursor;
-import edu.uci.ics.hyracks.storage.am.lsmtree.tuples.LSMTypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapManager;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
-import edu.uci.ics.hyracks.test.support.TestUtils;
-
-public class LSMTreeDeleteTest extends AbstractLSMTreeTest {
-
- private static final int PAGE_SIZE = 256;
- private static final int NUM_PAGES = 100;
- private static final int MAX_OPEN_FILES = 100;
- private static final int HYRACKS_FRAME_SIZE = 128;
- private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
-
- // BASIC DELETE TEST
- // create a fix-length lsm tree, and do 100 deletes. That is insert 100
- // delete nodes into the in-memory tree.
- @Test
- public void Test1() throws Exception {
- // in disk
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- IBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), PAGE_SIZE, NUM_PAGES);
-
- // declare fields
- int fieldCount = 2;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = IntegerPointable.TYPE_TRAITS;
- typeTraits[1] = IntegerPointable.TYPE_TRAITS;
-
- // declare keys
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
-
- MultiComparator cmp = new MultiComparator(cmps);
-
- LSMTypeAwareTupleWriterFactory insertTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
- LSMTypeAwareTupleWriterFactory deleteTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
-
- ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
- ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-
- InMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(100, metaFrameFactory);
-
- LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(bufferCache, metaFrameFactory);
- BTreeFactory bTreeFactory = new BTreeFactory(bufferCache, freePageManagerFactory, cmp, fieldCount,
- interiorFrameFactory, insertLeafFrameFactory);
-
- LSMTree lsmtree = new LSMTree(memBufferCache, bufferCache, fieldCount, cmp, memFreePageManager,
- interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, bTreeFactory,
- (IFileMapManager) fmp);
-
- lsmtree.create(fileId);
- lsmtree.open(fileId);
-
- ByteBuffer frame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
-
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- DataOutput dos = tb.getDataOutput();
-
- ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
-
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(frame);
-
- FrameTupleReference tuple = new FrameTupleReference();
-
- ITreeIndexAccessor lsmTreeAccessor = lsmtree.createAccessor();
-
- int resultSize = 50;
- int[][] resultArray = new int[resultSize][3];
-
- for (int i = 0; i < resultSize; i++) {
- resultArray[i][0] = i;
- resultArray[i][1] = i + 1;
- resultArray[i][2] = 1;
- }
-
- // delete
- for (int i = 0; i < resultSize; i++) {
-
- int f0 = resultArray[i][0];
- int f1 = resultArray[i][1];
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.delete(t);
- } catch (TreeIndexException e) {
- System.out.println("test01:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- // scan
- ITreeIndexCursor scanCursor = new LSMTreeRangeSearchCursor();
- RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
- lsmTreeAccessor.search(scanCursor, nullPred);
-
- try {
- int scanTupleIndex = 0;
- int arrayIndex = 0;
- Object o = null;
- while (scanCursor.hasNext()) {
- scanCursor.next();
- ITupleReference frameTuple = scanCursor.getTuple();
- int numPrintFields = Math.min(frameTuple.getFieldCount(), recDescSers.length);
-
- for (int i = 0; i < numPrintFields; i++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(frameTuple.getFieldData(i),
- frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
- DataInput dataIn = new DataInputStream(inStream);
- o = recDescSers[i].deserialize(dataIn);
-
- }
- while (resultArray[arrayIndex][2] != 0) {
- arrayIndex++;
- }
- if (Integer.parseInt(o.toString()) != resultArray[arrayIndex][1]) {
- fail("Input value and Output value doesn't match on the " + scanTupleIndex + " tuple\n");
- }
- scanTupleIndex++;
- arrayIndex++;
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- scanCursor.close();
- }
-
- lsmtree.close();
- bufferCache.closeFile(fileId);
- memBufferCache.close();
- }
-
- // INSERT-DELETE TEST
- // create a fix-length lsm tree. First, do 100 insertions,
- // and then do 50 deletions which has the same 50 keys which are part of the
- // insertions.
- @Test
- public void Test2() throws Exception {
- // in disk
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- IBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), PAGE_SIZE, NUM_PAGES);
-
- // declare fields
- int fieldCount = 2;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = IntegerPointable.TYPE_TRAITS;
- typeTraits[1] = IntegerPointable.TYPE_TRAITS;
-
- // declare keys
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
-
- MultiComparator cmp = new MultiComparator(cmps);
-
- LSMTypeAwareTupleWriterFactory insertTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
- LSMTypeAwareTupleWriterFactory deleteTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
-
- ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
- ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-
- InMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(100, metaFrameFactory);
-
- LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(bufferCache, metaFrameFactory);
- BTreeFactory bTreeFactory = new BTreeFactory(bufferCache, freePageManagerFactory, cmp, fieldCount,
- interiorFrameFactory, insertLeafFrameFactory);
-
- LSMTree lsmtree = new LSMTree(memBufferCache, bufferCache, fieldCount, cmp, memFreePageManager,
- interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, bTreeFactory,
- (IFileMapManager) fmp);
-
- lsmtree.create(fileId);
- lsmtree.open(fileId);
-
- ByteBuffer frame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
-
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- DataOutput dos = tb.getDataOutput();
-
- ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
-
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(frame);
-
- FrameTupleReference tuple = new FrameTupleReference();
-
- int resultSize = 100;
- int deleteStartPosition = 50;
- int[][] resultArray = new int[resultSize][3];
-
- for (int i = 0; i < resultSize; i++) {
- resultArray[i][0] = i;
- resultArray[i][1] = i + 1;
- resultArray[i][2] = 0;
- }
-
- // insert
- ITreeIndexAccessor lsmTreeAccessor = lsmtree.createAccessor();
- for (int i = 0; i < resultSize; i++) {
-
- int f0 = resultArray[i][0];
- int f1 = resultArray[i][1];
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.insert(t);
- } catch (TreeIndexException e) {
- System.out.println("test02:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- // delete
- for (int i = deleteStartPosition; i < resultSize; i++) {
-
- int f0 = resultArray[i][0];
- int f1 = ++resultArray[i][1];
- resultArray[i][2] = 1;
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.delete(t);
- } catch (TreeIndexException e) {
- System.out.println("test02:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- // scan
- ITreeIndexCursor scanCursor = new LSMTreeRangeSearchCursor();
- RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
- lsmTreeAccessor.search(scanCursor, nullPred);
-
- try {
- int scanTupleIndex = 0;
- int arrayIndex = 0;
- Object o = null;
- while (scanCursor.hasNext()) {
- scanCursor.next();
- ITupleReference frameTuple = scanCursor.getTuple();
- int numPrintFields = Math.min(frameTuple.getFieldCount(), recDescSers.length);
-
- for (int i = 0; i < numPrintFields; i++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(frameTuple.getFieldData(i),
- frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
- DataInput dataIn = new DataInputStream(inStream);
- o = recDescSers[i].deserialize(dataIn);
-
- }
- while (resultArray[arrayIndex][2] != 0) {
- arrayIndex++;
- }
- if (Integer.parseInt(o.toString()) != resultArray[arrayIndex][1]) {
- fail("Input value and Output value doesn't match on the " + scanTupleIndex + " tuple\n");
- }
-
- scanTupleIndex++;
- arrayIndex++;
-
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- scanCursor.close();
- }
-
- lsmtree.close();
- bufferCache.closeFile(fileId);
- memBufferCache.close();
- }
-
- // DELETE->INSERT TEST
- // create a fix-length lsm tree. First, do 100 deletions,
- // and then do 50 insertions which has the same 50 keys which are part of
- // the deletions.
- @Test
- public void Test3() throws Exception {
- System.out.println("TEST3");
- // in disk
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- IBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), PAGE_SIZE, NUM_PAGES);
-
- // declare fields
- int fieldCount = 2;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = IntegerPointable.TYPE_TRAITS;
- typeTraits[1] = IntegerPointable.TYPE_TRAITS;
-
- // declare keys
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
-
- MultiComparator cmp = new MultiComparator(cmps);
-
- LSMTypeAwareTupleWriterFactory insertTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
- LSMTypeAwareTupleWriterFactory deleteTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
-
- ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
- ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
- // change
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-
- InMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(100, metaFrameFactory);
-
- LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(bufferCache, metaFrameFactory);
- BTreeFactory bTreeFactory = new BTreeFactory(bufferCache, freePageManagerFactory, cmp, fieldCount,
- interiorFrameFactory, insertLeafFrameFactory);
-
- LSMTree lsmtree = new LSMTree(memBufferCache, bufferCache, fieldCount, cmp, memFreePageManager,
- interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, bTreeFactory,
- (IFileMapManager) fmp);
-
- lsmtree.create(fileId);
- lsmtree.open(fileId);
-
- ByteBuffer frame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
-
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- DataOutput dos = tb.getDataOutput();
-
- ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
-
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(frame);
-
- FrameTupleReference tuple = new FrameTupleReference();
-
- int resultSize = 100;
- int insertStartPosition = 50;
- int[][] resultArray = new int[resultSize][3];
-
- for (int i = 0; i < resultSize; i++) {
- resultArray[i][0] = i;
- resultArray[i][1] = i + 1;
- resultArray[i][2] = 1;
- }
-
- // delete
- ITreeIndexAccessor lsmTreeAccessor = lsmtree.createAccessor();
- for (int i = 0; i < resultSize; i++) {
-
- int f0 = resultArray[i][0];
- int f1 = resultArray[i][1];
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.delete(t);
- } catch (TreeIndexException e) {
- System.out.println("test03:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- // insert
- for (int i = insertStartPosition; i < resultSize; i++) {
-
- int f0 = resultArray[i][0];
- int f1 = ++resultArray[i][1];
- resultArray[i][2] = 0;
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.insert(t);
- } catch (TreeIndexException e) {
- System.out.println("test03:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- // scan
- ITreeIndexCursor scanCursor = new LSMTreeRangeSearchCursor();
- RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
- lsmTreeAccessor.search(scanCursor, nullPred);
-
- try {
- int scanTupleIndex = 0;
- int arrayIndex = 0;
- Object o = null;
- while (scanCursor.hasNext()) {
- scanCursor.next();
- ITupleReference frameTuple = scanCursor.getTuple();
- int numPrintFields = Math.min(frameTuple.getFieldCount(), recDescSers.length);
-
- for (int i = 0; i < numPrintFields; i++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(frameTuple.getFieldData(i),
- frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
- DataInput dataIn = new DataInputStream(inStream);
- o = recDescSers[i].deserialize(dataIn);
- }
- while (resultArray[arrayIndex][2] != 0) {
- arrayIndex++;
- }
- if (Integer.parseInt(o.toString()) != resultArray[arrayIndex][1]) {
- fail("Input value and Output value doesn't match on the " + scanTupleIndex + " tuple\n");
- }
-
- scanTupleIndex++;
- arrayIndex++;
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- scanCursor.close();
- }
-
- lsmtree.close();
- bufferCache.closeFile(fileId);
- memBufferCache.close();
- }
-
- // TEST DELETION and PageAllocationException
- // create a fix-length lsm tree. First, do 811 deletions,
- // the page will be run out on the 810th deletions, if there is any
- // exception returns, the test case fails.
- @Test
- public void Test4() throws Exception {
- // in disk
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- IBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), PAGE_SIZE, NUM_PAGES);
-
- // declare fields
- int fieldCount = 2;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = IntegerPointable.TYPE_TRAITS;
- typeTraits[1] = IntegerPointable.TYPE_TRAITS;
-
- // declare keys
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
-
- MultiComparator cmp = new MultiComparator(cmps);
-
- LSMTypeAwareTupleWriterFactory insertTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
- LSMTypeAwareTupleWriterFactory deleteTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
-
- ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
- ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-
- InMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(100, metaFrameFactory);
-
- LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(bufferCache, metaFrameFactory);
- BTreeFactory bTreeFactory = new BTreeFactory(bufferCache, freePageManagerFactory, cmp, fieldCount,
- interiorFrameFactory, insertLeafFrameFactory);
-
- // For the Flush Mechanism
- LSMTree lsmtree = new LSMTree(memBufferCache, bufferCache, fieldCount, cmp, memFreePageManager,
- interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, bTreeFactory,
- (IFileMapManager) fmp);
-
- lsmtree.create(fileId);
- lsmtree.open(fileId);
-
- ByteBuffer frame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
-
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- DataOutput dos = tb.getDataOutput();
-
- ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
-
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(frame);
-
- FrameTupleReference tuple = new FrameTupleReference();
-
- int resultSize = 811;
- int[][] resultArray = new int[resultSize][2];
-
- for (int i = 0; i < resultSize; i++) {
- resultArray[i][0] = i;
- resultArray[i][1] = i + 1;
- }
-
- // delete
- ITreeIndexAccessor lsmTreeAccessor = lsmtree.createAccessor();
- for (int i = 0; i < resultSize; i++) {
-
- int f0 = resultArray[i][0];
- int f1 = resultArray[i][1];
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.delete(t);
- } catch (TreeIndexException e) {
- System.out.println("test04:" + e);
- e.printStackTrace();
- fail("test04: Catch TreeIndexException" + e);
- } catch (Exception e) {
- e.printStackTrace();
- fail("test04: Catch Other Exceptions" + e);
- }
- }
- }
-
- // DELETE -> DELETE
- // create a fix-length lsm tree. First, do 100 deletions,
- // and then do 50 deletions which has the same 50 keys which are part of the
- // first deletions.
- @Test
- public void Test5() throws Exception {
- // in disk
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- IBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), PAGE_SIZE, NUM_PAGES);
-
- // declare fields
- int fieldCount = 2;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = IntegerPointable.TYPE_TRAITS;
- typeTraits[1] = IntegerPointable.TYPE_TRAITS;
-
- // declare keys
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
-
- MultiComparator cmp = new MultiComparator(cmps);
-
- LSMTypeAwareTupleWriterFactory insertTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
- LSMTypeAwareTupleWriterFactory deleteTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
-
- ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
- ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-
- InMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(100, metaFrameFactory);
-
- LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(bufferCache, metaFrameFactory);
- BTreeFactory bTreeFactory = new BTreeFactory(bufferCache, freePageManagerFactory, cmp, fieldCount,
- interiorFrameFactory, insertLeafFrameFactory);
-
- LSMTree lsmtree = new LSMTree(memBufferCache, bufferCache, fieldCount, cmp, memFreePageManager,
- interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, bTreeFactory,
- (IFileMapManager) fmp);
-
- lsmtree.create(fileId);
- lsmtree.open(fileId);
-
- ByteBuffer frame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
-
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- DataOutput dos = tb.getDataOutput();
-
- ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
-
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(frame);
-
- FrameTupleReference tuple = new FrameTupleReference();
-
- int resultSize = 100;
- int insertStartPosition = 50;
- int[][] resultArray = new int[resultSize][3];
-
- for (int i = 0; i < resultSize; i++) {
- resultArray[i][0] = i;
- resultArray[i][1] = i + 1;
- resultArray[i][2] = 1;
- }
-
- // First deletion part
- ITreeIndexAccessor lsmTreeAccessor = lsmtree.createAccessor();
- for (int i = 0; i < resultSize; i++) {
-
- int f0 = resultArray[i][0];
- int f1 = resultArray[i][1];
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.delete(t);
- } catch (TreeIndexException e) {
- System.out.println("test05:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- // Second delete part
- for (int i = insertStartPosition; i < resultSize; i++) {
-
- int f0 = resultArray[i][0];
- int f1 = ++resultArray[i][1];
- resultArray[i][2] = 1;
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.insert(t);
- } catch (TreeIndexException e) {
- System.out.println("test05:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- // scan
- ITreeIndexCursor scanCursor = new LSMTreeRangeSearchCursor();
- RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
- lsmTreeAccessor.search(scanCursor, nullPred);
-
- try {
- int scanTupleIndex = 0;
- int arrayIndex = 0;
- Object o = null;
- while (scanCursor.hasNext()) {
- scanCursor.next();
- ITupleReference frameTuple = scanCursor.getTuple();
- int numPrintFields = Math.min(frameTuple.getFieldCount(), recDescSers.length);
-
- for (int i = 0; i < numPrintFields; i++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(frameTuple.getFieldData(i),
- frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
- DataInput dataIn = new DataInputStream(inStream);
- o = recDescSers[i].deserialize(dataIn);
-
- }
- while (resultArray[arrayIndex][2] != 0) {
- arrayIndex++;
- }
- if (Integer.parseInt(o.toString()) != resultArray[arrayIndex][1]) {
- fail("Input value and Output value doesn't match on the " + scanTupleIndex + " tuple\n");
- }
-
- scanTupleIndex++;
- arrayIndex++;
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- scanCursor.close();
- }
-
- lsmtree.close();
- bufferCache.closeFile(fileId);
- memBufferCache.close();
- }
-
- // INSERT -> DELETE -> INSERT
- // create a fix-length lsm tree. Do the insertion, deletion and insertion.
- // the final result will be
- // | 0~9 | 10~19 | 20~39 | 40~59 | 60~79 | 80~99 |
- // | f1=10 | f1=9 | f1=8 | f1=7 | f1=6 | f1=5 |
- // | Insert| Insert| Delete| Delete| Insert| Insert|
- @Test
- public void Test6() throws Exception {
-
- // in disk
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- IBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), PAGE_SIZE, NUM_PAGES);
-
- // declare fields
- int fieldCount = 2;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = IntegerPointable.TYPE_TRAITS;
- typeTraits[1] = IntegerPointable.TYPE_TRAITS;
-
- // declare keys
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
-
- MultiComparator cmp = new MultiComparator(cmps);
-
- LSMTypeAwareTupleWriterFactory insertTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
- LSMTypeAwareTupleWriterFactory deleteTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
-
- ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
- ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-
- InMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(100, metaFrameFactory);
-
- LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(bufferCache, metaFrameFactory);
- BTreeFactory bTreeFactory = new BTreeFactory(bufferCache, freePageManagerFactory, cmp, fieldCount,
- interiorFrameFactory, insertLeafFrameFactory);
-
- LSMTree lsmtree = new LSMTree(memBufferCache, bufferCache, fieldCount, cmp, memFreePageManager,
- interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, bTreeFactory,
- (IFileMapManager) fmp);
-
- lsmtree.create(fileId);
- lsmtree.open(fileId);
-
- ByteBuffer frame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
-
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- DataOutput dos = tb.getDataOutput();
-
- ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
-
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(frame);
-
- FrameTupleReference tuple = new FrameTupleReference();
-
- ITreeIndexAccessor lsmTreeAccessor = lsmtree.createAccessor();
-
- int resultSize = 180;
- int[][] resultArray = new int[resultSize][3];
-
- // insert
- for (int i = 0; i < 180; i++) {
- int f0 = i % 100;
- int f1;
- if (i >= 100) {
- f1 = 6;
- } else {
- f1 = 5;
- }
-
- resultArray[f0][0] = f0;
- resultArray[f0][1] = f1;
- resultArray[f0][2] = 0;
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.insert(t);
- } catch (TreeIndexException e) {
- System.out.println("test06:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- // delete
- for (int i = 0; i < 100; i++) {
- int f0 = i % 60;
- int f1;
- if (i >= 60) {
- f1 = 8;
- } else {
- f1 = 7;
- }
-
- resultArray[f0][0] = f0;
- resultArray[f0][1] = f1;
- resultArray[f0][2] = 1;
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.delete(t);
- } catch (TreeIndexException e) {
- System.out.println("test06:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- }
-
- // reinsert
- for (int i = 0; i < 30; i++) {
- int f0 = i % 20;
- int f1;
- if (i >= 20) {
- f1 = 10;
- } else {
- f1 = 9;
- }
-
- resultArray[f0][0] = f0;
- resultArray[f0][1] = f1;
- resultArray[f0][2] = 0;
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.insert(t);
- } catch (TreeIndexException e) {
- System.out.println("test06:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- // scan
- ITreeIndexCursor scanCursor = new LSMTreeRangeSearchCursor();
- RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
- lsmTreeAccessor.search(scanCursor, nullPred);
-
- try {
- int scanTupleIndex = 0;
- int arrayIndex = 0;
- Object o = null;
- while (scanCursor.hasNext()) {
- scanCursor.next();
- ITupleReference frameTuple = scanCursor.getTuple();
- int numPrintFields = Math.min(frameTuple.getFieldCount(), recDescSers.length);
-
- for (int i = 0; i < numPrintFields; i++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(frameTuple.getFieldData(i),
- frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
- DataInput dataIn = new DataInputStream(inStream);
- o = recDescSers[i].deserialize(dataIn);
- }
- while (resultArray[arrayIndex][2] != 0) {
- arrayIndex++;
- }
- if (Integer.parseInt(o.toString()) != resultArray[arrayIndex][1]) {
- fail("Input value and Output value doesn't match on the " + scanTupleIndex + " tuple\n");
- }
-
- scanTupleIndex++;
- arrayIndex++;
- }
-
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- scanCursor.close();
- }
-
- lsmtree.close();
- bufferCache.closeFile(fileId);
- memBufferCache.close();
- }
-}
diff --git a/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/LSMTreeFlushTest.java b/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/LSMTreeFlushTest.java
deleted file mode 100644
index 0c40485..0000000
--- a/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/LSMTreeFlushTest.java
+++ /dev/null
@@ -1,754 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.lsmtree.btree;
-
-import java.io.DataOutput;
-import java.io.File;
-import java.nio.ByteBuffer;
-
-import org.junit.Test;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
-import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
-import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.lsmtree.common.freepage.InMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsmtree.common.freepage.InMemoryFreePageManager;
-import edu.uci.ics.hyracks.storage.am.lsmtree.impls.BTreeFactory;
-import edu.uci.ics.hyracks.storage.am.lsmtree.impls.LSMTree;
-import edu.uci.ics.hyracks.storage.am.lsmtree.tuples.LSMEntireTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.am.lsmtree.tuples.LSMTypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapManager;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
-import edu.uci.ics.hyracks.test.support.TestUtils;
-
-public class LSMTreeFlushTest extends AbstractLSMTreeTest {
- private static final int PAGE_SIZE = 256;
- private static final int NUM_PAGES = 100;
- private static final int MAX_OPEN_FILES = 10000;
- private static final int HYRACKS_FRAME_SIZE = 128;
- private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
-
- // BASIC TEST
- // @Test
- // public void Test1() throws Exception {
- // System.out.printf("TEST1 START\n");
- // //in disk
- // TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES,
- // MAX_OPEN_FILES);
- // IBufferCache bufferCache =
- // TestStorageManagerComponentHolder.getBufferCache(ctx);
- // IFileMapProvider fmp =
- // TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- // FileReference file = new FileReference(new File(fileName));
- // bufferCache.createFile(file);
- // int fileId = fmp.lookupFileId(file);
- // bufferCache.openFile(fileId);
- //
- // //in memory
- // InMemoryBufferCacheFactory InMemBufferCacheFactory = new
- // InMemoryBufferCacheFactory(PAGE_SIZE, NUM_PAGES);
- // IBufferCache memBufferCache =
- // InMemBufferCacheFactory.createInMemoryBufferCache();
- //
- // // declare fields
- // int fieldCount = 2;
- // ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- // typeTraits[0] = new TypeTrait(4);
- // typeTraits[1] = new TypeTrait(4);
- //
- // // declare keys
- // int keyFieldCount = 1;
- // IBinaryComparatorFactory[] cmpFactories = new
- // IBinaryComparatorFactory[keyFieldCount];
- // cmpFactories[0] = IntegerBinaryComparatorFactory.INSTANCE;
- //
- // MultiComparator cmp = BTreeUtils.createMultiComparator(cmpFactories);
- //
- // LSMTypeAwareTupleWriterFactory insertTupleWriterFactory = new
- // LSMTypeAwareTupleWriterFactory(typeTraits, false);
- // LSMTypeAwareTupleWriterFactory deleteTupleWriterFactory = new
- // LSMTypeAwareTupleWriterFactory(typeTraits, true);
- //
- // ITreeIndexFrameFactory insertLeafFrameFactory = new
- // BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
- // ITreeIndexFrameFactory deleteLeafFrameFactory = new
- // BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
- // ITreeIndexFrameFactory interiorFrameFactory = new
- // BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
- // ITreeIndexMetaDataFrameFactory metaFrameFactory = new
- // LIFOMetaDataFrameFactory();
- //
- // IBTreeLeafFrame insertLeafFrame = (IBTreeLeafFrame)
- // insertLeafFrameFactory.createFrame();
- //
- // IFreePageManager freePageManager = new
- // LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
- // IFreePageManager memFreePageManager = new InMemoryFreePageManager(100,
- // metaFrameFactory);
- //
- // // For the Flush Mechanism
- // LSMEntireTupleWriterFactory flushTupleWriterFactory = new
- // LSMEntireTupleWriterFactory(typeTraits);
- // ITreeIndexFrameFactory flushLeafFrameFactory = new
- // BTreeNSMLeafFrameFactory(flushTupleWriterFactory);
- // FreePageManagerFactory freePageManagerFactory = new
- // FreePageManagerFactory(bufferCache, metaFrameFactory);
- // BTreeFactory bTreeFactory = new BTreeFactory(bufferCache,
- // freePageManagerFactory, cmp, fieldCount, interiorFrameFactory,
- // flushLeafFrameFactory);
- //
- //
- //
- // // LSMTree lsmtree = new LSMTree(3, 100, 2, memBufferCache, bufferCache,
- // fieldCount, cmp, memFreePageManager,
- // // freePageManager, interiorFrameFactory, insertLeafFrameFactory,
- // deleteLeafFrameFactory, bTreeFactory, flushLeafFrameFactory,
- // (IFileMapManager)fmp);
- // //
- // LSMTree lsmtree = LSMTreeUtils.createLSMTree(memBufferCache, bufferCache,
- // fileId, typeTraits, cmp.getComparators(), BTreeLeafFrameType.REGULAR_NSM,
- // (IFileMapManager)fmp);
- // lsmtree.create(fileId);
- // lsmtree.open(fileId);
- //
- // ByteBuffer frame = ctx.allocateFrame();
- // FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- //
- // ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- // DataOutput dos = tb.getDataOutput();
- //
- // ISerializerDeserializer[] recDescSers = {
- // IntegerSerializerDeserializer.INSTANCE,
- // IntegerSerializerDeserializer.INSTANCE };
- // RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- //
- // IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(),
- // recDesc);
- // accessor.reset(frame);
- //
- // FrameTupleReference tuple = new FrameTupleReference();
- //
- // int resultSize = 100;
- // int[][] resultArray = new int[resultSize][2];
- //
- //
- // //insert 100 tuples
- // for (int i = 0; i < resultSize; i++){
- // resultArray[i][0] = i;
- // resultArray[i][1] = 1;
- // }
- //
- //
- // LSMTreeOpContext insertOpCtx = lsmtree.createOpContext(IndexOp.INSERT);
- // for (int i = 0; i < resultSize; i++) {
- //
- // int f0 = resultArray[i][0];
- // int f1 = resultArray[i][1];
- //
- // tb.reset();
- // IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- // tb.addFieldEndOffset();
- // IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- // tb.addFieldEndOffset();
- //
- // appender.reset(frame, true);
- // appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0,
- // tb.getSize());
- //
- // tuple.reset(accessor, 0);
- //
- // ArrayTupleReference t = new ArrayTupleReference();
- // t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
- //
- // try {
- // lsmtree.insert(t, insertOpCtx);
- // } catch (TreeIndexException e) {
- // System.out.println("test01:" + e);
- // e.printStackTrace();
- // } catch (Exception e) {
- // e.printStackTrace();
- // }
- // }
- // // Delete the first 50 keys in the in-memory tree
- // insertOpCtx = lsmtree.createOpContext(IndexOp.DELETE);
- // for (int i = 0; i < 50; i++){
- // resultArray[i][0] = i;
- // resultArray[i][1] = 1;
- // }
- //
- // for (int i = 0; i < 50; i++) {
- //
- // int f0 = resultArray[i][0];
- // int f1 = resultArray[i][1];
- //
- // tb.reset();
- // IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- // tb.addFieldEndOffset();
- // IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- // tb.addFieldEndOffset();
- //
- // appender.reset(frame, true);
- // appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0,
- // tb.getSize());
- //
- // tuple.reset(accessor, 0);
- //
- // ArrayTupleReference t = new ArrayTupleReference();
- // t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
- //
- // try {
- // lsmtree.delete(t, insertOpCtx);
- // } catch (TreeIndexException e) {
- // System.out.println("test01:" + e);
- // e.printStackTrace();
- // } catch (Exception e) {
- // e.printStackTrace();
- // }
- // }
- //
- //
- // //Flush the tree into the first in Disk tree
- // lsmtree.flushInMemoryBtree();
- //
- // //insert 50 delete nodes
- // insertOpCtx = lsmtree.createOpContext(IndexOp.DELETE);
- // for (int i = 0; i < 50; i++){
- // resultArray[i][0] = i;
- // resultArray[i][1] = 2;
- // }
- //
- // for (int i = 0; i < 50; i++) {
- //
- // int f0 = resultArray[i][0];
- // int f1 = resultArray[i][1];
- //
- // tb.reset();
- // IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- // tb.addFieldEndOffset();
- // IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- // tb.addFieldEndOffset();
- //
- // appender.reset(frame, true);
- // appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0,
- // tb.getSize());
- //
- // tuple.reset(accessor, 0);
- //
- // ArrayTupleReference t = new ArrayTupleReference();
- // t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
- //
- // try {
- // lsmtree.delete(t, insertOpCtx);
- // } catch (TreeIndexException e) {
- // System.out.println("test01:" + e);
- // e.printStackTrace();
- // } catch (Exception e) {
- // e.printStackTrace();
- // }
- // }
- //
- // // insert 25 nodes
- // insertOpCtx = lsmtree.createOpContext(IndexOp.INSERT);
- // for (int i = 0; i < resultSize; i++){
- // resultArray[i][0] = i;
- // resultArray[i][1] = 2;
- // }
- // for (int i = 0; i < 25; i++) {
- //
- // int f0 = resultArray[i][0];
- // int f1 = resultArray[i][1];
- //
- // tb.reset();
- // IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- // tb.addFieldEndOffset();
- // IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- // tb.addFieldEndOffset();
- //
- // appender.reset(frame, true);
- // appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0,
- // tb.getSize());
- //
- // tuple.reset(accessor, 0);
- //
- // ArrayTupleReference t = new ArrayTupleReference();
- // t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
- //
- // try {
- // lsmtree.insert(t, insertOpCtx);
- // } catch (TreeIndexException e) {
- // System.out.println("test01:" + e);
- // e.printStackTrace();
- // } catch (Exception e) {
- // e.printStackTrace();
- // }
- // }
- //
- // //Flush the tree into the fist in Disk tree, which have fieldId as "1"
- // lsmtree.flushInMemoryBtree();
- //
- // //Print out the first in Disk Btree
- // System.out.println("LSMTreeFlushTest: start print the first tree");
- // lsmtree.scanDiskTree(0);
- // //Print out the second in Disk Btree
- // System.out.println("LSMTreeFlushTest: start print the second tree");
- // lsmtree.scanDiskTree(1);
- //
- //
- // lsmtree.close();
- // bufferCache.closeFile(fileId);
- // memBufferCache.close();
- //
- // System.out.printf("End of TEST1()\n");
- //
- // }
- // TEST auto Flush
- @Test
- public void Test2() throws Exception {
- System.out.printf("TEST2 START\n");
- // in disk
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- IBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), PAGE_SIZE, NUM_PAGES);
-
- // declare fields
- int fieldCount = 2;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = IntegerPointable.TYPE_TRAITS;
- typeTraits[1] = IntegerPointable.TYPE_TRAITS;
-
- // declare keys
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
-
- MultiComparator cmp = new MultiComparator(cmps);
-
- LSMTypeAwareTupleWriterFactory insertTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
- LSMTypeAwareTupleWriterFactory deleteTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
-
- ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
- ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-
- IBTreeLeafFrame insertLeafFrame = (IBTreeLeafFrame) insertLeafFrameFactory.createFrame();
-
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
- InMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(NUM_PAGES, metaFrameFactory);
-
- // For the Flush Mechanism
- LSMEntireTupleWriterFactory flushTupleWriterFactory = new LSMEntireTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory flushLeafFrameFactory = new BTreeNSMLeafFrameFactory(flushTupleWriterFactory);
- LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(bufferCache, metaFrameFactory);
- BTreeFactory bTreeFactory = new BTreeFactory(bufferCache, freePageManagerFactory, cmp, fieldCount,
- interiorFrameFactory, flushLeafFrameFactory);
-
- LSMTree lsmtree = new LSMTree(memBufferCache, bufferCache, fieldCount, cmp, memFreePageManager,
- interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, bTreeFactory,
- (IFileMapManager) fmp);
-
- lsmtree.create(fileId);
- lsmtree.open(fileId);
-
- ByteBuffer frame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
-
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- DataOutput dos = tb.getDataOutput();
-
- ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
-
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(frame);
-
- FrameTupleReference tuple = new FrameTupleReference();
-
- int resultSize = 820;
- int[][] resultArray = new int[resultSize][2];
-
- // insert 820 tuples
- for (int i = 0; i < resultSize; i++) {
- resultArray[i][0] = i;
- resultArray[i][1] = i;
- }
-
- ITreeIndexAccessor lsmTreeAccessor = lsmtree.createAccessor();
- for (int i = 0; i < resultSize; i++) {
-
- int f0 = resultArray[i][0];
- int f1 = resultArray[i][1];
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.insert(t);
- } catch (TreeIndexException e) {
- System.out.println("test02:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- // Print out the third in Disk Btree
- System.out.println("LSMTreeFlushTest: start print the first tree");
- // lsmtree.scanDiskTree(2);
- // Print out the second in Disk Btree
- System.out.println("LSMTreeFlushTest: start print the first tree");
- // lsmtree.scanDiskTree(1);
- // Print out the first in Disk Btree
- System.out.println("LSMTreeFlushTest: start print the first tree");
- lsmtree.scanDiskTree(0);
-
- lsmtree.close();
- bufferCache.closeFile(fileId);
- memBufferCache.close();
-
- System.out.printf("End of TEST2()\n");
-
- }
-
- // @Test
- // public void Test3() throws Exception {
- // System.out.printf("TEST3 START\n");
- // //in disk
- // TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES,
- // MAX_OPEN_FILES);
- // IBufferCache bufferCache =
- // TestStorageManagerComponentHolder.getBufferCache(ctx);
- // IFileMapProvider fmp =
- // TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- // FileReference file = new FileReference(new File(fileName));
- // bufferCache.createFile(file);
- // int fileId = fmp.lookupFileId(file);
- // bufferCache.openFile(fileId);
- //
- // //in memory
- // InMemoryBufferCacheFactory InMemBufferCacheFactory = new
- // InMemoryBufferCacheFactory(PAGE_SIZE, NUM_PAGES);
- // IBufferCache memBufferCache =
- // InMemBufferCacheFactory.createInMemoryBufferCache();
- //
- // // declare fields
- // int fieldCount = 2;
- // ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- // typeTraits[0] = new TypeTrait(4);
- // typeTraits[1] = new TypeTrait(4);
- //
- // // declare keys
- // int keyFieldCount = 1;
- // IBinaryComparatorFactory[] cmpFactories = new
- // IBinaryComparatorFactory[keyFieldCount];
- // cmpFactories[0] = IntegerBinaryComparatorFactory.INSTANCE;
- //
- // MultiComparator cmp = BTreeUtils.createMultiComparator(cmpFactories);
- //
- // LSMTypeAwareTupleWriterFactory insertTupleWriterFactory = new
- // LSMTypeAwareTupleWriterFactory(typeTraits, false);
- // LSMTypeAwareTupleWriterFactory deleteTupleWriterFactory = new
- // LSMTypeAwareTupleWriterFactory(typeTraits, true);
- //
- // ITreeIndexFrameFactory insertLeafFrameFactory = new
- // BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
- // ITreeIndexFrameFactory deleteLeafFrameFactory = new
- // BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
- // ITreeIndexFrameFactory interiorFrameFactory = new
- // BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
- // ITreeIndexMetaDataFrameFactory metaFrameFactory = new
- // LIFOMetaDataFrameFactory();
- //
- // IBTreeLeafFrame insertLeafFrame = (IBTreeLeafFrame)
- // insertLeafFrameFactory.createFrame();
- //
- // IFreePageManager freePageManager = new
- // LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
- // IFreePageManager memFreePageManager = new InMemoryFreePageManager(30,
- // metaFrameFactory);
- //
- // // For the Flush Mechanism
- // LSMEntireTupleWriterFactory flushTupleWriterFactory = new
- // LSMEntireTupleWriterFactory(typeTraits);
- // ITreeIndexFrameFactory flushLeafFrameFactory = new
- // BTreeNSMLeafFrameFactory(flushTupleWriterFactory);
- // FreePageManagerFactory freePageManagerFactory = new
- // FreePageManagerFactory(bufferCache, metaFrameFactory);
- // BTreeFactory bTreeFactory = new BTreeFactory(bufferCache,
- // freePageManagerFactory, cmp, fieldCount, interiorFrameFactory,
- // flushLeafFrameFactory);
- //
- //
- //
- // LSMTree lsmtree = new LSMTree(memBufferCache, bufferCache, fieldCount,
- // cmp, memFreePageManager, interiorFrameFactory, insertLeafFrameFactory,
- // deleteLeafFrameFactory, bTreeFactory, (IFileMapManager)fmp);
- //
- // lsmtree.create(fileId);
- // lsmtree.open(fileId);
- //
- // ByteBuffer frame = ctx.allocateFrame();
- // FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- //
- // ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- // DataOutput dos = tb.getDataOutput();
- //
- // ISerializerDeserializer[] recDescSers = {
- // IntegerSerializerDeserializer.INSTANCE,
- // IntegerSerializerDeserializer.INSTANCE };
- // RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- //
- // IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(),
- // recDesc);
- // accessor.reset(frame);
- //
- // FrameTupleReference tuple = new FrameTupleReference();
- //
- // int resultSize = 500;
- // int[][] resultArray = new int[resultSize][2];
- //
- //
- // //insert 250 tuples
- // System.out.printf("Start for 1st Insert\n");
- // LSMTreeOpContext insertOpCtx = lsmtree.createOpContext(IndexOp.INSERT);
- // for (int i = 0; i < 252; i++){
- // resultArray[i][0] = i;
- // resultArray[i][1] = i;
- // }
- // for (int i = 0; i < 252; i++) {
- //
- // int f0 = resultArray[i][0];
- // int f1 = resultArray[i][1];
- //
- // tb.reset();
- // IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- // tb.addFieldEndOffset();
- // IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- // tb.addFieldEndOffset();
- //
- // appender.reset(frame, true);
- // appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0,
- // tb.getSize());
- //
- // tuple.reset(accessor, 0);
- //
- // ArrayTupleReference t = new ArrayTupleReference();
- // t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
- //
- // try {
- // lsmtree.insert(t, insertOpCtx);
- // } catch (TreeIndexException e) {
- // System.out.println("test03:" + e);
- // e.printStackTrace();
- // } catch (Exception e) {
- // e.printStackTrace();
- // }
- // }
- // System.out.printf("Start for 2nd Insert\n");
- // //delete 126~251. Deletion of 251 cause the flush
- // insertOpCtx.reset(IndexOp.DELETE);
- // // LSMTreeOpContext insertOpCtx =
- // lsmtree.createOpContext(IndexOp.DELETE);
- // for (int i = 125; i < 253; i++){
- // resultArray[i][0] = i;
- // resultArray[i][1] = i;
- // }
- // for (int i = 126; i < 253; i++) {
- // int f0 = resultArray[i][0];
- // int f1 = resultArray[i][1];
- //
- // tb.reset();
- // IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- // tb.addFieldEndOffset();
- // IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- // tb.addFieldEndOffset();
- //
- // appender.reset(frame, true);
- // appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0,
- // tb.getSize());
- //
- // tuple.reset(accessor, 0);
- //
- // ArrayTupleReference t = new ArrayTupleReference();
- // t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
- //
- // try {
- // lsmtree.delete(t, insertOpCtx);
- // } catch (TreeIndexException e) {
- // System.out.println("test03:" + e);
- // e.printStackTrace();
- // } catch (Exception e) {
- // e.printStackTrace();
- // }
- // }
- // //delete 0~250
- // insertOpCtx = lsmtree.createOpContext(IndexOp.INSERT);
- // for (int i = 130; i > 0; i--){
- // resultArray[i][0] = i;
- // resultArray[i][1] = i;
- // }
- // for (int i = 130; i > 0; i--) {
- //
- // int f0 = resultArray[i][0];
- // int f1 = resultArray[i][1];
- //
- // tb.reset();
- // IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- // tb.addFieldEndOffset();
- // IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- // tb.addFieldEndOffset();
- //
- // appender.reset(frame, true);
- // appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0,
- // tb.getSize());
- //
- // tuple.reset(accessor, 0);
- //
- // ArrayTupleReference t = new ArrayTupleReference();
- // t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
- //
- // try {
- // lsmtree.insert(t, insertOpCtx);
- // } catch (TreeIndexException e) {
- // System.out.println("test03:" + e);
- // e.printStackTrace();
- // } catch (Exception e) {
- // e.printStackTrace();
- // }
- // }
- //
- // //
- // //
- // //
- // // //Print out the second in Disk Btree
- // // System.out.println("LSMTreeFlushTest: start print the second tree");
- // // lsmtree.scanDiskTree(1);
- // // //Print out the first in Disk Btree
- // // System.out.println("LSMTreeFlushTest: start print the first tree");
- // // lsmtree.scanDiskTree(0);
- // //
- // // //Print out the In-memory Tree
- // //
- // System.out.println("LSMTreeFlushTest: start print the In-memory tree");
- // // lsmtree.scanInMemoryTree();
- // // //TODO: scan whole tree
- //
- // LOGGER.info("RANGE SEARCH:");
- //
- // BTreeOpContext searchOpCtx = lsmtree.createOpContext(IndexOp.SEARCH);
- // ITreeIndexCursor rangeCursor = new LSMTreeRangeSearchCursor();
- //
- // // build low and high keys
- // ArrayTupleBuilder ktb = new ArrayTupleBuilder(cmp.getKeyFieldCount());
- // DataOutput kdos = ktb.getDataOutput();
- //
- // ISerializerDeserializer[] keyDescSers = {
- // IntegerSerializerDeserializer.INSTANCE };
- // RecordDescriptor keyDesc = new RecordDescriptor(keyDescSers);
- // IFrameTupleAccessor keyAccessor = new
- // FrameTupleAccessor(ctx.getFrameSize(), keyDesc);
- // keyAccessor.reset(frame);
- //
- // appender.reset(frame, true);
- //
- // // build and append low key
- // ktb.reset();
- // IntegerSerializerDeserializer.INSTANCE.serialize(-1, kdos);
- // ktb.addFieldEndOffset();
- // appender.append(ktb.getFieldEndOffsets(), ktb.getByteArray(), 0,
- // ktb.getSize());
- //
- // // build and append high key
- // ktb.reset();
- // IntegerSerializerDeserializer.INSTANCE.serialize(300, kdos);
- // ktb.addFieldEndOffset();
- // appender.append(ktb.getFieldEndOffsets(), ktb.getByteArray(), 0,
- // ktb.getSize());
- //
- // // create tuplereferences for search keys
- // FrameTupleReference lowKey = new FrameTupleReference();
- // lowKey.reset(keyAccessor, 0);
- //
- // FrameTupleReference highKey = new FrameTupleReference();
- // highKey.reset(keyAccessor, 1);
- //
- // IBinaryComparator[] searchCmps = new IBinaryComparator[1];
- // searchCmps[0] =
- // IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
- // MultiComparator searchCmp = new MultiComparator(searchCmps);
- //
- // RangePredicate rangePred = new RangePredicate(true, lowKey, highKey,
- // true, true, searchCmp, searchCmp);
- // lsmtree.search(rangeCursor, rangePred, searchOpCtx);
- //
- // try {
- // while (rangeCursor.hasNext()) {
- // rangeCursor.next();
- // ITupleReference frameTuple = rangeCursor.getTuple();
- // String rec = TupleUtils.printTuple(frameTuple, recDescSers);
- // if(((LSMTypeAwareTupleReference)frameTuple).isDelete()) {
- // System.out.println("del " + rec);
- // }
- // else {
- // System.out.println("ins " + rec);
- // }
- // // System.out.println("------------------");
- // }
- // } catch (Exception e) {
- // e.printStackTrace();
- // } finally {
- // rangeCursor.close();
- // }
- //
- // lsmtree.close();
- // bufferCache.closeFile(fileId);
- // memBufferCache.close();
- //
- // System.out.printf("End of TEST3()\n");
- //
- // }
-
-}
\ No newline at end of file
diff --git a/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/LSMTreeMergeTest.java b/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/LSMTreeMergeTest.java
deleted file mode 100644
index 24677f7..0000000
--- a/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/LSMTreeMergeTest.java
+++ /dev/null
@@ -1,377 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.lsmtree.btree;
-
-import java.io.DataOutput;
-import java.io.File;
-import java.nio.ByteBuffer;
-
-import org.junit.Test;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
-import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
-import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.lsmtree.common.freepage.InMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsmtree.common.freepage.InMemoryFreePageManager;
-import edu.uci.ics.hyracks.storage.am.lsmtree.impls.BTreeFactory;
-import edu.uci.ics.hyracks.storage.am.lsmtree.impls.LSMTree;
-import edu.uci.ics.hyracks.storage.am.lsmtree.tuples.LSMEntireTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.am.lsmtree.tuples.LSMTypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapManager;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
-import edu.uci.ics.hyracks.test.support.TestUtils;
-
-public class LSMTreeMergeTest extends AbstractLSMTreeTest {
- private static final int PAGE_SIZE = 256;
- private static final int NUM_PAGES = 30;
- private static final int MAX_OPEN_FILES = 100;
- private static final int HYRACKS_FRAME_SIZE = 128;
- private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
-
- @Test
- public void Test1() throws Exception {
- System.out.printf("TEST1 START\n");
- // in disk
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- IBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), PAGE_SIZE, NUM_PAGES);
-
- // declare fields
- int fieldCount = 2;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = IntegerPointable.TYPE_TRAITS;
- typeTraits[1] = IntegerPointable.TYPE_TRAITS;
-
- // declare keys
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
-
- MultiComparator cmp = new MultiComparator(cmps);
-
- LSMTypeAwareTupleWriterFactory insertTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
- LSMTypeAwareTupleWriterFactory deleteTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
-
- ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
- ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-
- IBTreeLeafFrame insertLeafFrame = (IBTreeLeafFrame) insertLeafFrameFactory.createFrame();
-
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
- InMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(NUM_PAGES, metaFrameFactory);
-
- // For the Flush Mechanism
- LSMEntireTupleWriterFactory flushTupleWriterFactory = new LSMEntireTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory flushLeafFrameFactory = new BTreeNSMLeafFrameFactory(flushTupleWriterFactory);
- LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(bufferCache, metaFrameFactory);
- BTreeFactory bTreeFactory = new BTreeFactory(bufferCache, freePageManagerFactory, cmp, fieldCount,
- interiorFrameFactory, flushLeafFrameFactory);
-
- LSMTree lsmtree = new LSMTree(memBufferCache, bufferCache, fieldCount, cmp, memFreePageManager,
- interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, bTreeFactory,
- (IFileMapManager) fmp);
-
- // LSMTree lsmtree = LSMTreeUtils.createLSMTree(10, 30, 2,
- // memBufferCache, bufferCache, fileId, typeTraits,
- // cmp.getComparators(), BTreeLeafFrameType.REGULAR_NSM,
- // (IFileMapManager)fmp);
-
- lsmtree.create(fileId);
- lsmtree.open(fileId);
-
- ByteBuffer frame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
-
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- DataOutput dos = tb.getDataOutput();
-
- ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
-
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(frame);
-
- FrameTupleReference tuple = new FrameTupleReference();
-
- int resultSize = 100000;
- int[][] resultArray = new int[resultSize][2];
-
- // insert 0~250 tuples
- System.out.printf("Start for 1st Insert\n");
- ITreeIndexAccessor lsmTreeAccessor = lsmtree.createAccessor();
- for (int i = 0; i < 251; i++) {
- resultArray[i][0] = i;
- resultArray[i][1] = i;
- }
- for (int i = 0; i < 251; i++) {
-
- int f0 = resultArray[i][0];
- int f1 = resultArray[i][1];
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.insert(t);
- } catch (TreeIndexException e) {
- System.out.println("test03:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- System.out.printf("Start for 2nd Insert\n");
- // delete 126~250.
- for (int i = 126; i < 251; i++) {
- resultArray[i][0] = i;
- resultArray[i][1] = i;
- }
- for (int i = 126; i < 251; i++) {
- int f0 = resultArray[i][0];
- int f1 = resultArray[i][1];
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.delete(t);
- } catch (TreeIndexException e) {
- System.out.println("test03:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- // insert 251~1
- for (int i = 251; i > 0; i--) {
- resultArray[i][0] = i;
- resultArray[i][1] = i;
- }
- for (int i = 251; i > 0; i--) {
-
- int f0 = resultArray[i][0];
- int f1 = resultArray[i][1];
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.insert(t);
- } catch (TreeIndexException e) {
- System.out.println("test03:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- // delete 100~0
- for (int i = 100; i >= 0; i--) {
- resultArray[i][0] = i;
- resultArray[i][1] = i;
- }
- for (int i = 100; i >= 0; i--) {
-
- int f0 = resultArray[i][0];
- int f1 = resultArray[i][1];
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.delete(t);
- } catch (TreeIndexException e) {
- System.out.println("test03:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- // insert 1~50
- for (int i = 1; i < 51; i++) {
- resultArray[i][0] = i;
- resultArray[i][1] = i;
- }
- for (int i = 1; i < 51; i++) {
-
- int f0 = resultArray[i][0];
- int f1 = resultArray[i][1];
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.insert(t);
- } catch (TreeIndexException e) {
- System.out.println("test03:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- lsmtree.merge();
-
- // Output should be:
- // In memory tree = 0->del, 1~50 ins
- // MergedTree = 0->ins, 1~100->del, 101~251->ins
- // Whole search = 1~50,101~251
-
- // System.out.println("LSMTreeFlushTest: start print the first tree");
- // lsmtree.scanDiskTree(1);
- //
- // Print out the first in Disk Btree
- // System.out.println("LSMTreeFlushTest: start print the first tree");
- // lsmtree.scanDiskTree(0);
- // Print out the In-memory Tree
- System.out.println("LSMTreeFlushTest: start print the In-memory tree");
- lsmtree.scanInMemoryTree();
- // TODO: scan whole tree
- /*
- * System.out.println("Range SEARCH:");
- *
- * BTreeOpContext searchOpCtx = lsmtree.createOpContext(IndexOp.SEARCH);
- * ITreeIndexCursor rangeCursor = new LSMTreeRangeSearchCursor();
- *
- * // build low and high keys ArrayTupleBuilder ktb = new
- * ArrayTupleBuilder(cmp.getKeyFieldCount()); DataOutput kdos =
- * ktb.getDataOutput();
- *
- * ISerializerDeserializer[] keyDescSers = {
- * IntegerSerializerDeserializer.INSTANCE }; RecordDescriptor keyDesc =
- * new RecordDescriptor(keyDescSers); IFrameTupleAccessor keyAccessor =
- * new FrameTupleAccessor( ctx.getFrameSize(), keyDesc);
- * keyAccessor.reset(frame);
- *
- * appender.reset(frame, true);
- *
- * // build and append low key ktb.reset();
- * IntegerSerializerDeserializer.INSTANCE.serialize(-1, kdos);
- * ktb.addFieldEndOffset(); appender.append(ktb.getFieldEndOffsets(),
- * ktb.getByteArray(), 0, ktb.getSize());
- *
- * // build and append high key ktb.reset();
- * IntegerSerializerDeserializer.INSTANCE.serialize(300, kdos);
- * ktb.addFieldEndOffset(); appender.append(ktb.getFieldEndOffsets(),
- * ktb.getByteArray(), 0, ktb.getSize());
- *
- * // create tuplereferences for search keys FrameTupleReference lowKey
- * = new FrameTupleReference(); lowKey.reset(keyAccessor, 0);
- *
- * FrameTupleReference highKey = new FrameTupleReference();
- * highKey.reset(keyAccessor, 1);
- *
- * IBinaryComparator[] searchCmps = new IBinaryComparator[1];
- * searchCmps[0] = IntegerBinaryComparatorFactory.INSTANCE
- * .createBinaryComparator(); MultiComparator searchCmp = new
- * MultiComparator(searchCmps);
- *
- * RangePredicate rangePred = new RangePredicate(true, lowKey, highKey,
- * true, true, searchCmp, searchCmp); lsmtree.search(rangeCursor,
- * rangePred, searchOpCtx);
- *
- * try { while (rangeCursor.hasNext()) { rangeCursor.next();
- * ITupleReference frameTuple = rangeCursor.getTuple(); String rec =
- * TupleUtils.printTuple(frameTuple, recDescSers); if
- * (((LSMTypeAwareTupleReference) frameTuple).isDelete()) {
- * System.out.println("del " + rec); } else { System.out.println("ins "
- * + rec); } // System.out.println("------------------"); } } catch
- * (Exception e) { e.printStackTrace(); } finally { rangeCursor.close();
- * }
- */
- lsmtree.close();
- bufferCache.closeFile(fileId);
- memBufferCache.close();
-
- System.out.printf("End of TEST1()\n");
-
- }
-}
diff --git a/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/LSMTreeSearchTest.java b/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/LSMTreeSearchTest.java
deleted file mode 100644
index a342076..0000000
--- a/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/LSMTreeSearchTest.java
+++ /dev/null
@@ -1,418 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.lsmtree.btree;
-
-import java.io.DataOutput;
-import java.io.File;
-import java.nio.ByteBuffer;
-import java.util.Date;
-import java.util.Random;
-
-import org.junit.Test;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
-import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
-import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
-import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.am.lsmtree.common.freepage.InMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsmtree.common.freepage.InMemoryFreePageManager;
-import edu.uci.ics.hyracks.storage.am.lsmtree.impls.BTreeFactory;
-import edu.uci.ics.hyracks.storage.am.lsmtree.impls.InDiskTreeInfo;
-import edu.uci.ics.hyracks.storage.am.lsmtree.impls.LSMTree;
-import edu.uci.ics.hyracks.storage.am.lsmtree.impls.LSMTreeRangeSearchCursor;
-import edu.uci.ics.hyracks.storage.am.lsmtree.tuples.LSMEntireTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.am.lsmtree.tuples.LSMTypeAwareTupleReference;
-import edu.uci.ics.hyracks.storage.am.lsmtree.tuples.LSMTypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapManager;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
-import edu.uci.ics.hyracks.test.support.TestUtils;
-
-// TODO: needs a big cleanup phase.
-public class LSMTreeSearchTest extends AbstractLSMTreeTest {
-
- private static final int PAGE_SIZE = 256;
- private static final int NUM_PAGES = 10;
- private static final int MAX_OPEN_FILES = 100;
- private static final int HYRACKS_FRAME_SIZE = 128;
- private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
-
- @Test
- public void Test1() throws Exception {
- // in disk
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- IBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), PAGE_SIZE, NUM_PAGES);
-
- // declare fields
- int fieldCount = 2;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = IntegerPointable.TYPE_TRAITS;
- typeTraits[1] = IntegerPointable.TYPE_TRAITS;
-
- // declare keys
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
-
- MultiComparator cmp = new MultiComparator(cmps);
-
- LSMTypeAwareTupleWriterFactory insertTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
- LSMTypeAwareTupleWriterFactory deleteTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
-
- ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
- ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-
- IBTreeLeafFrame insertLeafFrame = (IBTreeLeafFrame) insertLeafFrameFactory.createFrame();
-
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
- InMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(100, metaFrameFactory);
-
- LSMEntireTupleWriterFactory flushTupleWriterFactory = new LSMEntireTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory flushLeafFrameFactory = new BTreeNSMLeafFrameFactory(flushTupleWriterFactory);
- LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(bufferCache, metaFrameFactory);
- BTreeFactory bTreeFactory = new BTreeFactory(bufferCache, freePageManagerFactory, cmp, fieldCount,
- interiorFrameFactory, flushLeafFrameFactory);
-
- LSMTree lsmtree = new LSMTree(memBufferCache, bufferCache, fieldCount, cmp, memFreePageManager,
- interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, bTreeFactory,
- (IFileMapManager) fmp);
-
- lsmtree.create(fileId);
- lsmtree.open(fileId);
-
- ByteBuffer frame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
-
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- DataOutput dos = tb.getDataOutput();
-
- ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
-
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(frame);
-
- FrameTupleReference tuple = new FrameTupleReference();
-
- ITreeIndexAccessor lsmTreeAccessor = lsmtree.createAccessor();
-
- // delete
- for (int i = 26; i < 36; i++) {
-
- int f0 = i;
- int f1 = -1;
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.delete(t);
- } catch (TreeIndexException e) {
- System.out.println("test01:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- for (int i = 21; i < 31; i++) {
- int f0 = i;
- int f1 = 0;
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- try {
- lsmTreeAccessor.insert(t);
- } catch (TreeIndexException e) {
- System.out.println("test01:" + e);
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- // In disk insertions 1
-
- LOGGER.info("Start in-disk insertions");
-
- fileName = tmpDir + sep + simpleDateFormat.format(new Date());
- FileReference file_1 = new FileReference(new File(fileName));
- bufferCache.createFile(file_1);
- int fileId_1 = fmp.lookupFileId(file_1);
- bufferCache.openFile(fileId_1);
-
- TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
-
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
- IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
-
- IFreePageManager freePageManager_1 = new LinkedListFreePageManager(bufferCache, fileId_1, 0, metaFrameFactory);
-
- BTree btree_1 = new BTree(bufferCache, fieldCount, cmp, freePageManager_1, interiorFrameFactory,
- leafFrameFactory);
- btree_1.create(fileId_1);
- btree_1.open(fileId_1);
-
- // TODO: rename this one.
- InDiskTreeInfo info_1 = new InDiskTreeInfo(btree_1);
- lsmtree.inDiskTreeInfoList.add(info_1);
-
- Random rnd = new Random();
- rnd.setSeed(50);
-
- LOGGER.info("INSERTING INTO TREE");
-
- // ByteBuffer
- frame = ctx.allocateFrame();
- // FrameTupleAppender
- appender = new FrameTupleAppender(ctx.getFrameSize());
- // ArrayTupleBuilder
- tb = new ArrayTupleBuilder(fieldCount);
- // DataOutput
- dos = tb.getDataOutput();
-
- recDesc = new RecordDescriptor(recDescSers);
-
- accessor.reset(frame);
-
- tuple = new FrameTupleReference();
-
- ITreeIndexAccessor indexAccessor_1 = btree_1.createAccessor();
-
- // 10000
- for (int i = 16; i < 41; i++) {
-
- int f0 = i;
- int f1 = 1;
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- if (i % 10 == 0) {
- System.out.println("INSERTING " + i + " : " + f0 + " " + f1);
- }
-
- try {
- indexAccessor_1.insert(t);
- } catch (TreeIndexException e) {
- e.printStackTrace();
- System.out.println("Error: " + e);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- // btree_1.close();
-
- // In disk insertions 2
-
- LOGGER.info("Start in-disk insertions");
-
- fileName = tmpDir + sep + simpleDateFormat.format(new Date());
- FileReference file_2 = new FileReference(new File(fileName));
- bufferCache.createFile(file_2);
- int fileId_2 = fmp.lookupFileId(file_2);
- bufferCache.openFile(fileId_2);
-
- IFreePageManager freePageManager_2 = new LinkedListFreePageManager(bufferCache, fileId_2, 0, metaFrameFactory);
- BTree btree_2 = new BTree(bufferCache, fieldCount, cmp, freePageManager_2, interiorFrameFactory,
- leafFrameFactory);
- btree_2.create(fileId_2);
- btree_2.open(fileId_2);
-
- InDiskTreeInfo info_2 = new InDiskTreeInfo(btree_2);
- lsmtree.inDiskTreeInfoList.add(info_2);
-
- LOGGER.info("INSERTING INTO TREE");
-
- // ByteBuffer
- frame = ctx.allocateFrame();
- // FrameTupleAppender
- appender = new FrameTupleAppender(ctx.getFrameSize());
- // ArrayTupleBuilder
- tb = new ArrayTupleBuilder(fieldCount);
- // DataOutput
- dos = tb.getDataOutput();
-
- recDesc = new RecordDescriptor(recDescSers);
-
- accessor.reset(frame);
-
- tuple = new FrameTupleReference();
-
- ITreeIndexAccessor indexAccessor_2 = btree_2.createAccessor();
-
- // 10000
- for (int i = 11; i < 51; i++) {
-
- int f0 = i;
- int f1 = 2;
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- ArrayTupleReference t = new ArrayTupleReference();
- t.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- if (i % 10 == 0) {
- System.out.println("INSERTING " + i + " : " + f0 + " " + f1);
- }
-
- try {
- indexAccessor_2.insert(t);
- } catch (TreeIndexException e) {
- e.printStackTrace();
- System.out.println("Error: " + e);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- // btree_2.close();
-
- // range search in [-1000, 1000]
- LOGGER.info("RANGE SEARCH:");
-
- ITreeIndexCursor rangeCursor = new LSMTreeRangeSearchCursor();
-
- // build low and high keys
- ArrayTupleBuilder ktb = new ArrayTupleBuilder(cmp.getKeyFieldCount());
- DataOutput kdos = ktb.getDataOutput();
-
- ISerializerDeserializer[] keyDescSers = { IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor keyDesc = new RecordDescriptor(keyDescSers);
- IFrameTupleAccessor keyAccessor = new FrameTupleAccessor(ctx.getFrameSize(), keyDesc);
- keyAccessor.reset(frame);
-
- appender.reset(frame, true);
-
- // build and append low key
- ktb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(-100, kdos);
- ktb.addFieldEndOffset();
- appender.append(ktb.getFieldEndOffsets(), ktb.getByteArray(), 0, ktb.getSize());
-
- // build and append high key
- ktb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(100, kdos);
- ktb.addFieldEndOffset();
- appender.append(ktb.getFieldEndOffsets(), ktb.getByteArray(), 0, ktb.getSize());
-
- // create tuplereferences for search keys
- FrameTupleReference lowKey = new FrameTupleReference();
- lowKey.reset(keyAccessor, 0);
-
- FrameTupleReference highKey = new FrameTupleReference();
- highKey.reset(keyAccessor, 1);
-
- IBinaryComparator[] searchCmps = cmps;
- MultiComparator searchCmp = new MultiComparator(searchCmps);
-
- RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, true, true, searchCmp, searchCmp);
- lsmTreeAccessor.search(rangeCursor, rangePred);
-
- try {
- while (rangeCursor.hasNext()) {
- rangeCursor.next();
- ITupleReference frameTuple = rangeCursor.getTuple();
- String rec = TupleUtils.printTuple(frameTuple, recDescSers);
- if (((LSMTypeAwareTupleReference) frameTuple).isDelete()) {
- System.out.println("del " + rec);
- } else {
- System.out.println("ins " + rec);
- }
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- rangeCursor.close();
- }
-
- lsmtree.close();
- bufferCache.closeFile(fileId);
- memBufferCache.close();
- }
-}
\ No newline at end of file
diff --git a/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/perf/LSMTreeRunner.java b/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/perf/LSMTreeRunner.java
index 3890929..852511e 100644
--- a/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/perf/LSMTreeRunner.java
+++ b/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/perf/LSMTreeRunner.java
@@ -1,15 +1,12 @@
package edu.uci.ics.hyracks.storage.am.lsmtree.btree.perf;
-import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Date;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
@@ -17,9 +14,9 @@
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.lsmtree.common.freepage.InMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsmtree.impls.LSMTree;
+import edu.uci.ics.hyracks.storage.am.lsmtree.util.LSMBTreeUtils;
import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapManager;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
import edu.uci.ics.hyracks.test.support.TestUtils;
@@ -37,7 +34,7 @@
protected final static String tmpDir = System.getProperty("java.io.tmpdir");
protected final static String sep = System.getProperty("file.separator");
protected final static String classDir = "/lsmtree/";
- protected String fileName;
+ protected String onDiskDir;
protected final int numBatches;
protected final LSMTree lsmtree;
@@ -51,21 +48,16 @@
this.onDiskPageSize = onDiskPageSize;
this.onDiskNumPages = onDiskNumPages;
- fileName = tmpDir + classDir + sep + simpleDateFormat.format(new Date());
+ onDiskDir = tmpDir + classDir + sep + simpleDateFormat.format(new Date()) + sep;
ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
-
TestStorageManagerComponentHolder.init(this.onDiskPageSize, this.onDiskNumPages, MAX_OPEN_FILES);
bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- lsmtreeFileId = fmp.lookupFileId(file);
- bufferCache.openFile(lsmtreeFileId);
- IBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), inMemPageSize, inMemNumPages);
+ InMemoryBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), inMemPageSize, inMemNumPages);
- lsmtree = LSMTreeUtils.createLSMTree(memBufferCache, bufferCache, lsmtreeFileId, typeTraits, cmp.getComparators(), BTreeLeafFrameType.REGULAR_NSM, (IFileMapManager)fmp);
+ lsmtree = LSMBTreeUtils.createLSMTree(memBufferCache, onDiskDir, bufferCache, fmp, typeTraits, cmp.getComparators());
}
@Override
public void init() throws Exception {
diff --git a/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/perf/LSMTreeUtils.java b/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/perf/LSMTreeUtils.java
deleted file mode 100644
index cf66991..0000000
--- a/hyracks-tests/hyracks-storage-am-lsmtree-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/btree/perf/LSMTreeUtils.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.lsmtree.btree.perf;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.lsmtree.common.freepage.InMemoryFreePageManager;
-import edu.uci.ics.hyracks.storage.am.lsmtree.impls.BTreeFactory;
-import edu.uci.ics.hyracks.storage.am.lsmtree.impls.LSMTree;
-import edu.uci.ics.hyracks.storage.am.lsmtree.tuples.LSMEntireTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.am.lsmtree.tuples.LSMTypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapManager;
-
-public class LSMTreeUtils {
- public static LSMTree createLSMTree(IBufferCache memCache, IBufferCache bufferCache, int lsmtreeFileId, ITypeTraits[] typeTraits, IBinaryComparator[] cmps, BTreeLeafFrameType leafType, IFileMapManager fileMapManager) throws BTreeException {
- MultiComparator cmp = new MultiComparator(cmps);
-
- LSMTypeAwareTupleWriterFactory insertTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
- LSMTypeAwareTupleWriterFactory deleteTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
-
- ITreeIndexFrameFactory insertLeafFrameFactory = BTreeUtils.getLeafFrameFactory(insertTupleWriterFactory,leafType);
- ITreeIndexFrameFactory deleteLeafFrameFactory = BTreeUtils.getLeafFrameFactory(deleteTupleWriterFactory,leafType);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- InMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(memCache.getNumPages(), metaFrameFactory);
-
- // For the Flush Mechanism
- LSMEntireTupleWriterFactory flushTupleWriterFactory = new LSMEntireTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory flushLeafFrameFactory = BTreeUtils.getLeafFrameFactory(flushTupleWriterFactory,leafType);
- LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(bufferCache, metaFrameFactory);
- BTreeFactory bTreeFactory = new BTreeFactory(bufferCache, freePageManagerFactory, cmp, typeTraits.length, interiorFrameFactory, flushLeafFrameFactory);
-
- LSMTree lsmtree = new LSMTree(memCache, bufferCache, typeTraits.length, cmp, memFreePageManager,
- interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, bTreeFactory, fileMapManager);
- return lsmtree;
- }
-}