Merged hyracks_dev_next r1064:r1078.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_lsm_tree@1079 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/SerdeUtils.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/SerdeUtils.java
index 87d9b35..00575f4 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/SerdeUtils.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/SerdeUtils.java
@@ -35,13 +35,41 @@
@SuppressWarnings("rawtypes")
public class SerdeUtils {
- public static ITypeTraits[] serdesToTypeTraits(ISerializerDeserializer[] serdes, int numSerdes) {
- ITypeTraits[] typeTraits = new ITypeTraits[numSerdes];
- for (int i = 0; i < numSerdes; i++) {
+ public static class PayloadTypeTraits implements ITypeTraits {
+ private static final long serialVersionUID = 1L;
+ final int payloadSize;
+
+ public PayloadTypeTraits(int payloadSize) {
+ this.payloadSize = payloadSize;
+ }
+
+ @Override
+ public boolean isFixedLength() {
+ return true;
+ }
+
+ @Override
+ public int getFixedLength() {
+ return payloadSize;
+ }
+ }
+
+ public static ITypeTraits[] serdesToTypeTraits(ISerializerDeserializer[] serdes) {
+ ITypeTraits[] typeTraits = new ITypeTraits[serdes.length];
+ for (int i = 0; i < serdes.length; i++) {
typeTraits[i] = serdeToTypeTrait(serdes[i]);
}
return typeTraits;
}
+
+ public static ITypeTraits[] serdesToTypeTraits(ISerializerDeserializer[] serdes, int payloadSize) {
+ ITypeTraits[] typeTraits = new ITypeTraits[serdes.length + 1];
+ for (int i = 0; i < serdes.length; i++) {
+ typeTraits[i] = serdeToTypeTrait(serdes[i]);
+ }
+ typeTraits[serdes.length] = new PayloadTypeTraits(payloadSize);
+ return typeTraits;
+ }
public static ITypeTraits serdeToTypeTrait(ISerializerDeserializer serde) {
if (serde instanceof IntegerSerializerDeserializer) {
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/TupleUtils.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/TupleUtils.java
index 14c4d66..df3ee67 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/TupleUtils.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/TupleUtils.java
@@ -83,4 +83,14 @@
}
return strBuilder.toString();
}
+
+ public static ITupleReference copyTuple(ITupleReference tuple) throws HyracksDataException {
+ ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(tuple.getFieldCount());
+ for (int i = 0; i < tuple.getFieldCount(); i++) {
+ tupleBuilder.addField(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
+ }
+ ArrayTupleReference tupleCopy = new ArrayTupleReference();
+ tupleCopy.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
+ return tupleCopy;
+ }
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeFrame.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeFrame.java
index a24c4d7..affc8ce 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeFrame.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeFrame.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.btree.api;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java
index 4486205..f78b6e4 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java
@@ -247,7 +247,7 @@
// buf.getInt(prevRec.getFieldOff()) + " " +
// buf.getInt(prevRec.getFieldOff()+4));
prefixFreeSpace += tupleWriter.writeTupleFields(prevTuple, 0, fieldCountToCompress,
- byteBuffer, prefixFreeSpace);
+ byteBuffer.array(), prefixFreeSpace);
// System.out.println("WRITING PREFIX RECORD " +
// prefixSlotNum + " AT " + tmp + " " +
// freeSpace);
@@ -265,7 +265,7 @@
newTupleSlots[tupleCount - 1 - currTupleIndex] = slotManager.encodeSlotFields(
prefixTupleIndex, tupleFreeSpace);
tupleFreeSpace += tupleWriter.writeTupleFields(tupleToWrite, fieldCountToCompress,
- fieldCount - fieldCountToCompress, byteBuffer, tupleFreeSpace);
+ fieldCount - fieldCountToCompress, byteBuffer.array(), tupleFreeSpace);
}
prefixTupleIndex++;
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
index 8a8c5ce..200c861 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
@@ -90,7 +90,7 @@
accessor = new FrameTupleAccessor(treeIndexHelper.getHyracksTaskContext().getFrameSize(), recDesc);
cursorFrame = opDesc.getTreeIndexLeafFactory().createFrame();
- setCursor();
+ setCursor();
writer.open();
try {
@@ -98,8 +98,9 @@
btree = (BTree) treeIndexHelper.getIndex();
// Construct range predicate.
- lowKeySearchCmp = BTreeUtils.getSearchMultiComparator(btree.getMultiComparator(), lowKey);
- highKeySearchCmp = BTreeUtils.getSearchMultiComparator(btree.getMultiComparator(), highKey);
+ lowKeySearchCmp = BTreeUtils.getSearchMultiComparator(btree.getMultiComparator().getComparators(), lowKey);
+ highKeySearchCmp = BTreeUtils
+ .getSearchMultiComparator(btree.getMultiComparator().getComparators(), highKey);
rangePred = new RangePredicate(isForward, null, null, lowKeyInclusive, highKeyInclusive, lowKeySearchCmp,
highKeySearchCmp);
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
index 0aba70b..6e9ad81 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
@@ -252,7 +252,7 @@
int freeSpace = buf.getInt(freeSpaceOff);
int bytesWritten = tupleWriter.writeTupleFields(tuple, numPrefixFields,
- tuple.getFieldCount() - numPrefixFields, buf, freeSpace);
+ tuple.getFieldCount() - numPrefixFields, buf.array(), freeSpace);
buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
buf.putInt(freeSpaceOff, buf.getInt(freeSpaceOff) + bytesWritten);
@@ -315,11 +315,11 @@
if (inPlace) {
// Overwrite the old tuple suffix in place.
- bytesWritten = tupleWriter.writeTupleFields(newTuple, numPrefixFields, fieldCount - numPrefixFields, buf, suffixTupleStartOff);
+ bytesWritten = tupleWriter.writeTupleFields(newTuple, numPrefixFields, fieldCount - numPrefixFields, buf.array(), suffixTupleStartOff);
} else {
// Insert the new tuple suffix at the end of the free space, and change the slot value (effectively "deleting" the old tuple).
int newSuffixTupleStartOff = buf.getInt(freeSpaceOff);
- bytesWritten = tupleWriter.writeTupleFields(newTuple, numPrefixFields, fieldCount - numPrefixFields, buf, newSuffixTupleStartOff);
+ bytesWritten = tupleWriter.writeTupleFields(newTuple, numPrefixFields, fieldCount - numPrefixFields, buf.array(), newSuffixTupleStartOff);
// Update slot value using the same prefix slot num.
slotManager.setSlot(tupleSlotOff, slotManager.encodeSlotFields(prefixSlotNum, newSuffixTupleStartOff));
// Update contiguous free space pointer.
@@ -493,7 +493,7 @@
}
int bytesWritten = tupleWriter.writeTupleFields(tuple, fieldsToTruncate, tuple.getFieldCount()
- - fieldsToTruncate, buf, freeSpace);
+ - fieldsToTruncate, buf.array(), freeSpace);
// insert slot
int prefixSlotNum = FieldPrefixSlotManager.TUPLE_UNCOMPRESSED;
@@ -634,7 +634,7 @@
int splitKeySize = tupleWriter.bytesRequired(frameTuple, 0, cmp.getKeyFieldCount());
splitKey.initData(splitKeySize);
- tupleWriter.writeTupleFields(frameTuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer(), 0);
+ tupleWriter.writeTupleFields(frameTuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer().array(), 0);
splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer(), 0);
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
index 6173440..1836da2 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
@@ -77,7 +77,7 @@
public void insert(ITupleReference tuple, int tupleIndex) {
int slotOff = slotManager.insertSlot(tupleIndex, buf.getInt(freeSpaceOff));
int freeSpace = buf.getInt(freeSpaceOff);
- int bytesWritten = tupleWriter.writeTupleFields(tuple, 0, tuple.getFieldCount(), buf, freeSpace);
+ int bytesWritten = tupleWriter.writeTupleFields(tuple, 0, tuple.getFieldCount(), buf.array(), freeSpace);
System.arraycopy(tuple.getFieldData(tuple.getFieldCount() - 1), getLeftChildPageOff(tuple), buf.array(),
freeSpace + bytesWritten, childPtrSize);
int tupleSize = bytesWritten + childPtrSize;
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
index 4856595..40a7670 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
@@ -117,7 +117,7 @@
@Override
public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey) throws TreeIndexException {
- ByteBuffer right = rightFrame.getBuffer();
+ ByteBuffer right = rightFrame.getBuffer();
int tupleCount = getTupleCount();
// Find split point, and determine into which frame the new tuple should be inserted into.
@@ -162,7 +162,7 @@
frameTuple.resetByTupleOffset(buf, tupleOff);
int splitKeySize = tupleWriter.bytesRequired(frameTuple, 0, cmp.getKeyFieldCount());
splitKey.initData(splitKeySize);
- tupleWriter.writeTupleFields(frameTuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer(), 0);
+ tupleWriter.writeTupleFields(frameTuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer().array(), 0);
splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer(), 0);
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
index 4113e9f..b3870b8 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
@@ -39,7 +39,6 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
-import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
@@ -138,7 +137,7 @@
}
private void search(ITreeIndexCursor cursor, ISearchPredicate searchPred, BTreeOpContext ctx)
- throws TreeIndexException, HyracksDataException, PageAllocationException {
+ throws TreeIndexException, HyracksDataException {
ctx.reset();
ctx.pred = (RangePredicate) searchPred;
ctx.cursor = cursor;
@@ -199,7 +198,7 @@
}
}
- private void createNewRoot(BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void createNewRoot(BTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
// Make sure the root is always in the same page.
ICachedPage leftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, ctx.splitKey.getLeftPage()),
false);
@@ -245,7 +244,7 @@
}
}
- private void insertUpdateOrDelete(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void insertUpdateOrDelete(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
ctx.reset();
ctx.pred.setLowKeyComparator(cmp);
ctx.pred.setHighKeyComparator(cmp);
@@ -281,11 +280,11 @@
}
}
- private void insert(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void insert(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
insertUpdateOrDelete(tuple, ctx);
}
- private void update(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void update(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
// This call only allows updating of non-key fields.
// Updating a tuple's key necessitates deleting the old entry, and inserting the new entry.
// The user of the BTree is responsible for dealing with non-key updates (i.e., doing a delete + insert).
@@ -295,7 +294,7 @@
insertUpdateOrDelete(tuple, ctx);
}
- private void delete(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void delete(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
insertUpdateOrDelete(tuple, ctx);
}
@@ -666,7 +665,7 @@
return isConsistent;
}
- private void performOp(int pageId, ICachedPage parent, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void performOp(int pageId, ICachedPage parent, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
ctx.interiorFrame.setPage(node);
@@ -801,13 +800,6 @@
ctx.exceptionHandled = true;
}
throw e;
- } catch (PageAllocationException e) {
- if (!ctx.exceptionHandled) {
- releaseLatch(node, ctx, unsafeIsLeaf);
- bufferCache.unpin(node);
- ctx.exceptionHandled = true;
- }
- throw e;
} catch (Exception e) {
e.printStackTrace();
releaseLatch(node, ctx, unsafeIsLeaf);
@@ -817,7 +809,7 @@
}
}
- public final class BulkLoadContext implements IIndexBulkLoadContext {
+ public class BulkLoadContext implements IIndexBulkLoadContext {
public final int slotSize;
public final int leafMaxBytes;
public final int interiorMaxBytes;
@@ -831,7 +823,7 @@
private final ITreeIndexTupleWriter tupleWriter;
public BulkLoadContext(float fillFactor, IBTreeLeafFrame leafFrame, IBTreeInteriorFrame interiorFrame,
- ITreeIndexMetaDataFrame metaFrame, MultiComparator cmp) throws HyracksDataException, PageAllocationException {
+ ITreeIndexMetaDataFrame metaFrame, MultiComparator cmp) throws HyracksDataException {
leafFrame.setMultiComparator(cmp);
interiorFrame.setMultiComparator(cmp);
@@ -862,7 +854,7 @@
nodeFrontiers.add(leafFrontier);
}
- private void addLevel() throws HyracksDataException, PageAllocationException {
+ private void addLevel() throws HyracksDataException {
NodeFrontier frontier = new NodeFrontier(tupleWriter.createTupleReference());
frontier.pageId = freePageManager.getFreePage(metaFrame);
frontier.page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, frontier.pageId), true);
@@ -874,7 +866,7 @@
}
}
- private void propagateBulk(BulkLoadContext ctx, int level) throws HyracksDataException, PageAllocationException {
+ private void propagateBulk(BulkLoadContext ctx, int level) throws HyracksDataException {
if (ctx.splitKey.getBuffer() == null)
return;
@@ -897,7 +889,7 @@
int splitKeySize = ctx.tupleWriter.bytesRequired(frontier.lastTuple, 0, cmp.getKeyFieldCount());
ctx.splitKey.initData(splitKeySize);
ctx.tupleWriter
- .writeTupleFields(frontier.lastTuple, 0, cmp.getKeyFieldCount(), ctx.splitKey.getBuffer(), 0);
+ .writeTupleFields(frontier.lastTuple, 0, cmp.getKeyFieldCount(), ctx.splitKey.getBuffer().array(), 0);
ctx.splitKey.getTuple().resetByTupleOffset(ctx.splitKey.getBuffer(), 0);
ctx.splitKey.setLeftPage(frontier.pageId);
@@ -920,7 +912,7 @@
// assumes btree has been created and opened
@Override
- public IIndexBulkLoadContext beginBulkLoad(float fillFactor) throws TreeIndexException, HyracksDataException, PageAllocationException {
+ public IIndexBulkLoadContext beginBulkLoad(float fillFactor) throws TreeIndexException, HyracksDataException {
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame)leafFrameFactory.createFrame();
if (!isEmptyTree(leafFrame)) {
throw new BTreeException("Trying to Bulk-load a non-empty BTree.");
@@ -933,7 +925,7 @@
}
@Override
- public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException, PageAllocationException {
+ public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException {
BulkLoadContext ctx = (BulkLoadContext) ictx;
NodeFrontier leafFrontier = ctx.nodeFrontiers.get(0);
IBTreeLeafFrame leafFrame = ctx.leafFrame;
@@ -952,7 +944,7 @@
int splitKeySize = ctx.tupleWriter.bytesRequired(leafFrontier.lastTuple, 0, cmp.getKeyFieldCount());
ctx.splitKey.initData(splitKeySize);
ctx.tupleWriter.writeTupleFields(leafFrontier.lastTuple, 0, cmp.getKeyFieldCount(),
- ctx.splitKey.getBuffer(), 0);
+ ctx.splitKey.getBuffer().array(), 0);
ctx.splitKey.getTuple().resetByTupleOffset(ctx.splitKey.getBuffer(), 0);
ctx.splitKey.setLeftPage(leafFrontier.pageId);
int prevPageId = leafFrontier.pageId;
@@ -1043,6 +1035,10 @@
return IndexType.BTREE;
}
+ public int getFileId() {
+ return fileId;
+ }
+
public byte getTreeHeight(IBTreeLeafFrame leafFrame) throws HyracksDataException {
ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
rootNode.acquireReadLatch();
@@ -1128,7 +1124,9 @@
return new BTreeAccessor(this);
}
- private class BTreeAccessor implements ITreeIndexAccessor {
+ // TODO: Class should be private. But currently we need to expose the
+ // setOpContext() API to the LSM Tree for it to work correctly.
+ public class BTreeAccessor implements ITreeIndexAccessor {
private BTree btree;
private BTreeOpContext ctx;
@@ -1138,34 +1136,53 @@
}
@Override
- public void insert(ITupleReference tuple) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ public void insert(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
ctx.reset(IndexOp.INSERT);
btree.insert(tuple, ctx);
}
@Override
- public void update(ITupleReference tuple) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ public void update(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
ctx.reset(IndexOp.UPDATE);
btree.update(tuple, ctx);
}
@Override
- public void delete(ITupleReference tuple) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ public void delete(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
ctx.reset(IndexOp.DELETE);
btree.delete(tuple, ctx);
}
@Override
+ public ITreeIndexCursor createSearchCursor() {
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
+ return new BTreeRangeSearchCursor(leafFrame, false);
+ }
+
+ @Override
public void search(ITreeIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException,
- TreeIndexException, PageAllocationException {
+ TreeIndexException {
ctx.reset(IndexOp.SEARCH);
btree.search(cursor, searchPred, ctx);
}
@Override
+ public ITreeIndexCursor createDiskOrderScanCursor() {
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
+ return new TreeDiskOrderScanCursor(leafFrame);
+ }
+
+ @Override
public void diskOrderScan(ITreeIndexCursor cursor) throws HyracksDataException {
ctx.reset(IndexOp.DISKORDERSCAN);
btree.diskOrderScan(cursor, ctx);
}
+
+ // TODO: Ideally, this method should not exist. But we need it for
+ // the changing the leafFrame and leafFrameFactory of the op context for
+ // the LSM-BTree to work correctly.
+ public BTreeOpContext getOpContext() {
+ return ctx;
+ }
}
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeOpContext.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeOpContext.java
index 07c645c..8f056cc 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeOpContext.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeOpContext.java
@@ -28,8 +28,8 @@
public class BTreeOpContext implements IIndexOpContext {
private final int INIT_ARRAYLIST_SIZE = 6;
- protected ITreeIndexFrameFactory leafFrameFactory;
- protected ITreeIndexFrameFactory interiorFrameFactory;
+ public ITreeIndexFrameFactory leafFrameFactory;
+ public ITreeIndexFrameFactory interiorFrameFactory;
public IBTreeLeafFrame leafFrame;
public IBTreeInteriorFrame interiorFrame;
public ITreeIndexMetaDataFrame metaFrame;
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
index 8122314..1b92b85 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
@@ -62,7 +62,7 @@
}
@Override
- public void close() throws Exception {
+ public void close() throws HyracksDataException {
if (page != null) {
if (exclusiveLatchNodes) {
page.releaseWriteLatch();
@@ -100,7 +100,7 @@
}
@Override
- public boolean hasNext() throws Exception {
+ public boolean hasNext() throws HyracksDataException {
if (pred.isForward()) {
if (tupleIndex >= frame.getTupleCount()) {
int nextLeafPage = frame.getNextLeaf();
@@ -145,7 +145,7 @@
}
@Override
- public void next() throws Exception {
+ public void next() throws HyracksDataException {
tupleIndex += tupleIndexInc;
}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/CheckTuple.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/CheckTuple.java
similarity index 97%
rename from hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/CheckTuple.java
rename to hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/CheckTuple.java
index f945ab9..d7a24a6 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/CheckTuple.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/CheckTuple.java
@@ -13,7 +13,7 @@
* limitations under the License.
*/
-package edu.uci.ics.hyracks.storage.am.btree.util;
+package edu.uci.ics.hyracks.storage.am.btree.tests;
@SuppressWarnings({"rawtypes", "unchecked"})
public class CheckTuple<T extends Comparable<T>> implements Comparable<T> {
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/IOrderedIndexTestContext.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/IOrderedIndexTestContext.java
new file mode 100644
index 0000000..78a860a
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/IOrderedIndexTestContext.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.tests;
+
+import java.util.TreeSet;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+
+@SuppressWarnings("rawtypes")
+public interface IOrderedIndexTestContext {
+ public int getFieldCount();
+
+ public int getKeyFieldCount();
+
+ public ISerializerDeserializer[] getFieldSerdes();
+
+ public ITreeIndexAccessor getIndexAccessor();
+
+ public ITreeIndex getIndex();
+
+ public ArrayTupleReference getTuple();
+
+ public ArrayTupleBuilder getTupleBuilder();
+
+ public IBinaryComparator[] getComparators();
+
+ public void insertIntCheckTuple(int[] fieldValues);
+
+ public void insertStringCheckTuple(String[] fieldValues);
+
+ public TreeSet<CheckTuple> getCheckTuples();
+}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexBulkLoadTest.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexBulkLoadTest.java
new file mode 100644
index 0000000..105008e
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexBulkLoadTest.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.tests;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+
+@SuppressWarnings("rawtypes")
+public abstract class OrderedIndexBulkLoadTest extends OrderedIndexTestDriver {
+
+ public OrderedIndexBulkLoadTest(BTreeLeafFrameType[] leafFrameTypesToTest) {
+ super(leafFrameTypesToTest);
+ }
+
+ @Override
+ protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType,
+ ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey)
+ throws Exception {
+ IOrderedIndexTestContext ctx = createTestContext(fieldSerdes, numKeys, leafType);
+
+ // We assume all fieldSerdes are of the same type. Check the first one
+ // to determine which field types to generate.
+ if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
+ OrderedIndexTestUtils.bulkLoadIntTuples(ctx, numTuplesToInsert, getRandom());
+ } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
+ OrderedIndexTestUtils.bulkLoadStringTuples(ctx, numTuplesToInsert, getRandom());
+ }
+
+ OrderedIndexTestUtils.checkPointSearches(ctx);
+ OrderedIndexTestUtils.checkOrderedScan(ctx);
+ OrderedIndexTestUtils.checkDiskOrderScan(ctx);
+ OrderedIndexTestUtils.checkRangeSearch(ctx, lowKey, highKey, true, true);
+ if (prefixLowKey != null && prefixHighKey != null) {
+ OrderedIndexTestUtils.checkRangeSearch(ctx, prefixLowKey, prefixHighKey, true, true);
+ }
+ }
+
+ @Override
+ protected String getTestOpName() {
+ return "BulkLoad";
+ }
+}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexDeleteTest.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexDeleteTest.java
new file mode 100644
index 0000000..a472934
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexDeleteTest.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.tests;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+
+@SuppressWarnings("rawtypes")
+public abstract class OrderedIndexDeleteTest extends OrderedIndexTestDriver {
+
+ public OrderedIndexDeleteTest(BTreeLeafFrameType[] leafFrameTypesToTest) {
+ super(leafFrameTypesToTest);
+ }
+
+ private static final int numInsertRounds = 3;
+ private static final int numDeleteRounds = 3;
+
+ @Override
+ protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType,
+ ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey)
+ throws Exception {
+ IOrderedIndexTestContext ctx = createTestContext(fieldSerdes, numKeys, leafType);
+ for (int i = 0; i < numInsertRounds; i++) {
+ // We assume all fieldSerdes are of the same type. Check the first
+ // one to determine which field types to generate.
+ if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
+ OrderedIndexTestUtils.insertIntTuples(ctx, numTuplesToInsert, getRandom());
+ } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
+ OrderedIndexTestUtils.insertStringTuples(ctx, numTuplesToInsert, getRandom());
+ }
+ int numTuplesPerDeleteRound = (int) Math.ceil((float) ctx.getCheckTuples().size() / (float) numDeleteRounds);
+ for (int j = 0; j < numDeleteRounds; j++) {
+ OrderedIndexTestUtils.deleteTuples(ctx, numTuplesPerDeleteRound, getRandom());
+ OrderedIndexTestUtils.checkPointSearches(ctx);
+ OrderedIndexTestUtils.checkOrderedScan(ctx);
+ OrderedIndexTestUtils.checkDiskOrderScan(ctx);
+ OrderedIndexTestUtils.checkRangeSearch(ctx, lowKey, highKey, true, true);
+ if (prefixLowKey != null && prefixHighKey != null) {
+ OrderedIndexTestUtils.checkRangeSearch(ctx, prefixLowKey, prefixHighKey, true, true);
+ }
+ }
+ }
+ }
+
+ @Override
+ protected String getTestOpName() {
+ return "Delete";
+ }
+}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexExamplesTest.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexExamplesTest.java
new file mode 100644
index 0000000..bdbd9e9
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexExamplesTest.java
@@ -0,0 +1,635 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.tests;
+
+import java.util.Random;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+
+@SuppressWarnings("rawtypes")
+public abstract class OrderedIndexExamplesTest {
+ protected static final Logger LOGGER = Logger.getLogger(OrderedIndexExamplesTest.class.getName());
+ protected final Random rnd = new Random(50);
+
+ protected abstract ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparator[] cmps)
+ throws TreeIndexException;
+
+ protected abstract int getIndexFileId();
+
+ /**
+ * Fixed-Length Key,Value Example.
+ *
+ * Create a tree index with one fixed-length key field and one fixed-length value
+ * field. Fill index with random values using insertions (not bulk load).
+ * Perform scans and range search.
+ */
+ @Test
+ public void fixedLengthKeyValueExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Fixed-Length Key,Value Example.");
+ }
+
+ // Declare fields.
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE };
+
+ // Declare keys.
+ int keyFieldCount = 1;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+
+ int indexFileId = getIndexFileId();
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmps);
+ treeIndex.create(indexFileId);
+ treeIndex.open(indexFileId);
+
+ long start = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Inserting into tree...");
+ }
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ ITreeIndexAccessor indexAccessor = treeIndex.createAccessor();
+ int numInserts = 10000;
+ for (int i = 0; i < numInserts; i++) {
+ int f0 = rnd.nextInt() % numInserts;
+ int f1 = 5;
+ TupleUtils.createIntegerTuple(tb, tuple, f0, f1);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + i + " : " + f0 + " " + f1);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ }
+ long end = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
+ }
+
+ orderedScan(indexAccessor, fieldSerdes);
+ diskOrderScan(indexAccessor, fieldSerdes);
+
+ // Build low key.
+ ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(keyFieldCount);
+ ArrayTupleReference lowKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(lowKeyTb, lowKey, -1000);
+
+ // Build high key.
+ ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(keyFieldCount);
+ ArrayTupleReference highKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(highKeyTb, highKey, 1000);
+
+ rangeSearch(cmps, indexAccessor, fieldSerdes, lowKey, highKey);
+
+ treeIndex.close();
+ }
+
+ /**
+ * Composite Key Example (Non-Unique Index).
+ *
+ * Create a tree index with two fixed-length key fields and one fixed-length
+ * value field. Fill index with random values using insertions (not bulk
+ * load) Perform scans and range search.
+ */
+ @Test
+ public void twoFixedLengthKeysOneFixedLengthValueExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Composite Key Test");
+ }
+
+ // Declare fields.
+ int fieldCount = 3;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+
+ // declare keys
+ int keyFieldCount = 2;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+ cmps[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+
+ int indexFileId = getIndexFileId();
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmps);
+ treeIndex.create(indexFileId);
+ treeIndex.open(indexFileId);
+
+ long start = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Inserting into tree...");
+ }
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ ITreeIndexAccessor indexAccessor = treeIndex.createAccessor();
+ int numInserts = 10000;
+ for (int i = 0; i < 10000; i++) {
+ int f0 = rnd.nextInt() % 2000;
+ int f1 = rnd.nextInt() % 1000;
+ int f2 = 5;
+ TupleUtils.createIntegerTuple(tb, tuple, f0, f1, f2);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + i + " : " + f0 + " " + f1 + " " + f2);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ }
+ long end = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
+ }
+
+ orderedScan(indexAccessor, fieldSerdes);
+ diskOrderScan(indexAccessor, fieldSerdes);
+
+ // Build low key.
+ ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference lowKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(lowKeyTb, lowKey, -3);
+
+ // Build high key.
+ ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference highKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(highKeyTb, highKey, 3);
+
+ // Prefix-Range search in [-3, 3]
+ rangeSearch(cmps, indexAccessor, fieldSerdes, lowKey, highKey);
+
+ treeIndex.close();
+ }
+
+ /**
+ * Variable-Length Example. Create a BTree with one variable-length key
+ * field and one variable-length value field. Fill BTree with random values
+ * using insertions (not bulk load) Perform ordered scans and range search.
+ */
+ @Test
+ public void varLenKeyValueExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Variable-Length Key,Value Example");
+ }
+
+ // Declare fields.
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+
+ // Declare keys.
+ int keyFieldCount = 1;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY).createBinaryComparator();
+
+ int indexFileId = getIndexFileId();
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmps);
+ treeIndex.create(indexFileId);
+ treeIndex.open(indexFileId);
+
+ long start = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Inserting into tree...");
+ }
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ ITreeIndexAccessor indexAccessor = treeIndex.createAccessor();
+ // Max string length to be generated.
+ int maxLength = 10;
+ int numInserts = 10000;
+ for (int i = 0; i < 10000; i++) {
+ String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + f0 + " " + f1);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ }
+ long end = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
+ }
+
+ orderedScan(indexAccessor, fieldSerdes);
+ diskOrderScan(indexAccessor, fieldSerdes);
+
+ // Build low key.
+ ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference lowKey = new ArrayTupleReference();
+ TupleUtils.createTuple(lowKeyTb, lowKey, fieldSerdes, "cbf");
+
+ // Build high key.
+ ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference highKey = new ArrayTupleReference();
+ TupleUtils.createTuple(highKeyTb, highKey, fieldSerdes, "cc7");
+
+ rangeSearch(cmps, indexAccessor, fieldSerdes, lowKey, highKey);
+
+ treeIndex.close();
+ }
+
+ /**
+ * Deletion Example.
+ *
+ * Create a BTree with one variable-length key field and one variable-length
+ * value field. Fill B-tree with random values using insertions, then delete
+ * entries one-by-one. Repeat procedure a few times on same BTree.
+ */
+ @Test
+ public void deleteExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Deletion Example");
+ }
+
+ // Declare fields.
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+
+ // Declare keys.
+ int keyFieldCount = 1;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY).createBinaryComparator();
+
+ int indexFileId = getIndexFileId();
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmps);
+ treeIndex.create(indexFileId);
+ treeIndex.open(indexFileId);
+
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ ITreeIndexAccessor indexAccessor = treeIndex.createAccessor();
+ // Max string length to be generated.
+ int runs = 3;
+ for (int run = 0; run < runs; run++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Deletion example run: " + (run + 1) + "/" + runs);
+ LOGGER.info("Inserting into tree...");
+ }
+ int maxLength = 10;
+ int ins = 10000;
+ String[] f0s = new String[ins];
+ String[] f1s = new String[ins];
+ int insDone = 0;
+ int[] insDoneCmp = new int[ins];
+ for (int i = 0; i < ins; i++) {
+ String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
+ f0s[i] = f0;
+ f1s[i] = f1;
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + i);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ insDone++;
+ } catch (TreeIndexException e) {
+ }
+ insDoneCmp[i] = insDone;
+ }
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Deleting from tree...");
+ }
+ int delDone = 0;
+ for (int i = 0; i < ins; i++) {
+ TupleUtils.createTuple(tb, tuple, fieldSerdes, f0s[i], f1s[i]);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Deleting " + i);
+ }
+ }
+ try {
+ indexAccessor.delete(tuple);
+ delDone++;
+ } catch (TreeIndexException e) {
+ }
+ if (insDoneCmp[i] != delDone) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("INCONSISTENT STATE, ERROR IN DELETION EXAMPLE.");
+ LOGGER.info("INSDONECMP: " + insDoneCmp[i] + " " + delDone);
+ }
+ break;
+ }
+ }
+ if (insDone != delDone) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("ERROR! INSDONE: " + insDone + " DELDONE: " + delDone);
+ }
+ break;
+ }
+ }
+ treeIndex.close();
+ }
+
+ /**
+ * Update example.
+ *
+ * Create a BTree with one variable-length key field and one variable-length
+ * value field. Fill B-tree with random values using insertions, then update
+ * entries one-by-one. Repeat procedure a few times on same BTree.
+ */
+ @Test
+ public void updateExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Update example");
+ }
+
+ // Declare fields.
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+
+ // Declare keys.
+ int keyFieldCount = 1;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY).createBinaryComparator();
+
+ int indexFileId = getIndexFileId();
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmps);
+ treeIndex.create(indexFileId);
+ treeIndex.open(indexFileId);
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Inserting into tree...");
+ }
+ ITreeIndexAccessor indexAccessor = treeIndex.createAccessor();
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ int maxLength = 10;
+ int ins = 10000;
+ String[] keys = new String[10000];
+ for (int i = 0; i < ins; i++) {
+ String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
+ keys[i] = f0;
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + i);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ }
+ // Print before doing any updates.
+ orderedScan(indexAccessor, fieldSerdes);
+
+ int runs = 3;
+ for (int run = 0; run < runs; run++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Update test run: " + (run + 1) + "/" + runs);
+ LOGGER.info("Updating BTree");
+ }
+ for (int i = 0; i < ins; i++) {
+ // Generate a new random value for f1.
+ String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ TupleUtils.createTuple(tb, tuple, fieldSerdes, keys[i], f1);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Updating " + i);
+ }
+ }
+ try {
+ indexAccessor.update(tuple);
+ } catch (TreeIndexException e) {
+ } catch (UnsupportedOperationException e) {
+ }
+ }
+ // Do another scan after a round of updates.
+ orderedScan(indexAccessor, fieldSerdes);
+ }
+ treeIndex.close();
+ }
+
+ /**
+ * Bulk load example.
+ *
+ * Load a tree with 100,000 tuples. BTree has a composite key to "simulate"
+ * non-unique index creation.
+ *
+ */
+ @Test
+ public void bulkLoadExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Bulk load example");
+ }
+ // Declare fields.
+ int fieldCount = 3;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+
+ // declare keys
+ int keyFieldCount = 2;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+ cmps[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+
+ int indexFileId = getIndexFileId();
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmps);
+ treeIndex.create(indexFileId);
+ treeIndex.open(indexFileId);
+
+ // Load sorted records.
+ int ins = 100000;
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Bulk loading " + ins + " tuples");
+ }
+ long start = System.currentTimeMillis();
+ IIndexBulkLoadContext bulkLoadCtx = treeIndex.beginBulkLoad(0.7f);
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ for (int i = 0; i < ins; i++) {
+ TupleUtils.createIntegerTuple(tb, tuple, i, i, 5);
+ treeIndex.bulkLoadAddTuple(tuple, bulkLoadCtx);
+ }
+ treeIndex.endBulkLoad(bulkLoadCtx);
+ long end = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(ins + " tuples loaded in " + (end - start) + "ms");
+ }
+
+ ITreeIndexAccessor indexAccessor = treeIndex.createAccessor();
+
+ // Build low key.
+ ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference lowKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(lowKeyTb, lowKey, 44444);
+
+ // Build high key.
+ ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference highKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(highKeyTb, highKey, 44500);
+
+ // Prefix-Range search in [44444, 44500]
+ rangeSearch(cmps, indexAccessor, fieldSerdes, lowKey, highKey);
+
+ treeIndex.close();
+ }
+
+ private void orderedScan(ITreeIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes)
+ throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Ordered Scan:");
+ }
+ ITreeIndexCursor scanCursor = indexAccessor.createSearchCursor();
+ RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
+ indexAccessor.search(scanCursor, nullPred);
+ try {
+ while (scanCursor.hasNext()) {
+ scanCursor.next();
+ ITupleReference frameTuple = scanCursor.getTuple();
+ String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rec);
+ }
+ }
+ } finally {
+ scanCursor.close();
+ }
+ }
+
+ private void diskOrderScan(ITreeIndexAccessor indexAccessor,
+ ISerializerDeserializer[] fieldSerdes) throws Exception {
+ try {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Disk-Order Scan:");
+ }
+ TreeDiskOrderScanCursor diskOrderCursor = (TreeDiskOrderScanCursor) indexAccessor
+ .createDiskOrderScanCursor();
+ indexAccessor.diskOrderScan(diskOrderCursor);
+ try {
+ while (diskOrderCursor.hasNext()) {
+ diskOrderCursor.next();
+ ITupleReference frameTuple = diskOrderCursor.getTuple();
+ String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rec);
+ }
+ }
+ } finally {
+ diskOrderCursor.close();
+ }
+ } catch (UnsupportedOperationException e) {
+ // Ignore exception because some indexes, e.g. the LSMBTree, don't
+ // support disk-order scan.
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Ignoring disk-order scan since it's not supported.");
+ }
+ }
+ }
+
+ private void rangeSearch(IBinaryComparator[] cmps, ITreeIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes,
+ ITupleReference lowKey, ITupleReference highKey) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ String lowKeyString = TupleUtils.printTuple(lowKey, fieldSerdes);
+ String highKeyString = TupleUtils.printTuple(highKey, fieldSerdes);
+ LOGGER.info("Range-Search in: [ " + lowKeyString + ", " + highKeyString + "]");
+ }
+ ITreeIndexCursor rangeCursor = indexAccessor.createSearchCursor();
+ MultiComparator lowKeySearchCmp = BTreeUtils.getSearchMultiComparator(cmps, lowKey);
+ MultiComparator highKeySearchCmp = BTreeUtils.getSearchMultiComparator(cmps, highKey);
+ RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, true, true, lowKeySearchCmp,
+ highKeySearchCmp);
+ indexAccessor.search(rangeCursor, rangePred);
+ try {
+ while (rangeCursor.hasNext()) {
+ rangeCursor.next();
+ ITupleReference frameTuple = rangeCursor.getTuple();
+ String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rec);
+ }
+ }
+ } finally {
+ rangeCursor.close();
+ }
+ }
+
+ public static String randomString(int length, Random random) {
+ String s = Long.toHexString(Double.doubleToLongBits(random.nextDouble()));
+ StringBuilder strBuilder = new StringBuilder();
+ for (int i = 0; i < s.length() && i < length; i++) {
+ strBuilder.append(s.charAt(Math.abs(random.nextInt()) % s.length()));
+ }
+ return strBuilder.toString();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexInsertTest.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexInsertTest.java
new file mode 100644
index 0000000..65bfc81
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexInsertTest.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.tests;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+
+/**
+ * Tests the BTree insert operation with strings and integer fields using
+ * various numbers of key and payload fields.
+ *
+ * Each tests first fills a BTree with randomly generated tuples. We compare the
+ * following operations against expected results: 1. Point searches for all
+ * tuples. 2. Ordered scan. 3. Disk-order scan. 4. Range search (and prefix
+ * search for composite keys).
+ *
+ */
+@SuppressWarnings("rawtypes")
+public abstract class OrderedIndexInsertTest extends OrderedIndexTestDriver {
+
+ public OrderedIndexInsertTest(BTreeLeafFrameType[] leafFrameTypesToTest) {
+ super(leafFrameTypesToTest);
+ }
+
+ @Override
+ protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType,
+ ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey)
+ throws Exception {
+ IOrderedIndexTestContext ctx = createTestContext(fieldSerdes, numKeys, leafType);
+ // We assume all fieldSerdes are of the same type. Check the first one
+ // to determine which field types to generate.
+ if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
+ OrderedIndexTestUtils.insertIntTuples(ctx, numTuplesToInsert, getRandom());
+ } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
+ OrderedIndexTestUtils.insertStringTuples(ctx, numTuplesToInsert, getRandom());
+ }
+
+ OrderedIndexTestUtils.checkPointSearches(ctx);
+ OrderedIndexTestUtils.checkOrderedScan(ctx);
+ OrderedIndexTestUtils.checkDiskOrderScan(ctx);
+
+ OrderedIndexTestUtils.checkRangeSearch(ctx, lowKey, highKey, true, true);
+ if (prefixLowKey != null && prefixHighKey != null) {
+ OrderedIndexTestUtils.checkRangeSearch(ctx, prefixLowKey, prefixHighKey, true, true);
+ }
+ ctx.getIndex().close();
+ }
+
+ @Override
+ protected String getTestOpName() {
+ return "Insert";
+ }
+}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexTestContext.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexTestContext.java
new file mode 100644
index 0000000..a56ed9c
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexTestContext.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.tests;
+
+import java.util.TreeSet;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+
+@SuppressWarnings("rawtypes")
+public abstract class OrderedIndexTestContext implements IOrderedIndexTestContext {
+ protected final ISerializerDeserializer[] fieldSerdes;
+ protected final ITreeIndex treeIndex;
+ protected final ArrayTupleBuilder tupleBuilder;
+ protected final ArrayTupleReference tuple = new ArrayTupleReference();
+ protected final TreeSet<CheckTuple> checkTuples = new TreeSet<CheckTuple>();
+ protected final ITreeIndexAccessor indexAccessor;
+
+ public OrderedIndexTestContext(ISerializerDeserializer[] fieldSerdes, ITreeIndex treeIndex) {
+ this.fieldSerdes = fieldSerdes;
+ this.treeIndex = treeIndex;
+ this.indexAccessor = treeIndex.createAccessor();
+ this.tupleBuilder = new ArrayTupleBuilder(fieldSerdes.length);
+ }
+
+ @Override
+ public int getFieldCount() {
+ return fieldSerdes.length;
+ }
+
+ @Override
+ public ITreeIndexAccessor getIndexAccessor() {
+ return indexAccessor;
+ }
+
+ @Override
+ public ArrayTupleReference getTuple() {
+ return tuple;
+ }
+
+ @Override
+ public ArrayTupleBuilder getTupleBuilder() {
+ return tupleBuilder;
+ }
+
+ @Override
+ public void insertIntCheckTuple(int[] fieldValues) {
+ CheckTuple<Integer> checkTuple = new CheckTuple<Integer>(getFieldCount(), getKeyFieldCount());
+ for(int v : fieldValues) {
+ checkTuple.add(v);
+ }
+ checkTuples.add(checkTuple);
+ }
+
+ @Override
+ public void insertStringCheckTuple(String[] fieldValues) {
+ CheckTuple<String> checkTuple = new CheckTuple<String>(getFieldCount(), getKeyFieldCount());
+ for(String s : fieldValues) {
+ checkTuple.add((String)s);
+ }
+ checkTuples.add(checkTuple);
+ }
+
+ @Override
+ public ISerializerDeserializer[] getFieldSerdes() {
+ return fieldSerdes;
+ }
+
+ @Override
+ public TreeSet<CheckTuple> getCheckTuples() {
+ return checkTuples;
+ }
+
+ @Override
+ public ITreeIndex getIndex() {
+ return treeIndex;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeTestDriver.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexTestDriver.java
similarity index 69%
rename from hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeTestDriver.java
rename to hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexTestDriver.java
index 1daa273..e89c38f 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeTestDriver.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexTestDriver.java
@@ -1,6 +1,23 @@
-package edu.uci.ics.hyracks.storage.am.btree;
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.btree.tests;
+
+import java.util.Random;
import java.util.logging.Level;
+import java.util.logging.Logger;
import org.junit.Test;
@@ -10,16 +27,24 @@
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
-import edu.uci.ics.hyracks.storage.am.btree.util.AbstractBTreeTest;
@SuppressWarnings("rawtypes")
-public abstract class BTreeTestDriver extends AbstractBTreeTest {
-
+public abstract class OrderedIndexTestDriver {
+ protected final Logger LOGGER = Logger.getLogger(OrderedIndexTestDriver.class.getName());
+
protected static final int numTuplesToInsert = 10000;
+ protected abstract IOrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType) throws Exception;
+ protected abstract Random getRandom();
protected abstract void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception;
protected abstract String getTestOpName();
+ protected final BTreeLeafFrameType[] leafFrameTypesToTest;
+
+ public OrderedIndexTestDriver(BTreeLeafFrameType[] leafFrameTypesToTest) {
+ this.leafFrameTypesToTest = leafFrameTypesToTest;
+ }
+
@Test
public void oneIntKeyAndValue() throws Exception {
if (LOGGER.isLoggable(Level.INFO)) {
@@ -31,8 +56,9 @@
ITupleReference lowKey = TupleUtils.createIntegerTuple(-1000);
ITupleReference highKey = TupleUtils.createIntegerTuple(1000);
- runTest(fieldSerdes, 1, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, null, null);
- runTest(fieldSerdes, 1, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, null, null);
+ for (BTreeLeafFrameType leafFrameType : leafFrameTypesToTest) {
+ runTest(fieldSerdes, 1, leafFrameType, lowKey, highKey, null, null);
+ }
}
@Test
@@ -51,8 +77,9 @@
ITupleReference prefixLowKey = TupleUtils.createIntegerTuple(50);
ITupleReference prefixHighKey = TupleUtils.createIntegerTuple(50);
- runTest(fieldSerdes, 2, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
- runTest(fieldSerdes, 2, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
+ for (BTreeLeafFrameType leafFrameType : leafFrameTypesToTest) {
+ runTest(fieldSerdes, 2, leafFrameType, lowKey, highKey, prefixLowKey, prefixHighKey);
+ }
}
@Test
@@ -71,8 +98,9 @@
ITupleReference prefixLowKey = TupleUtils.createIntegerTuple(50);
ITupleReference prefixHighKey = TupleUtils.createIntegerTuple(50);
- runTest(fieldSerdes, 2, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
- runTest(fieldSerdes, 2, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
+ for (BTreeLeafFrameType leafFrameType : leafFrameTypesToTest) {
+ runTest(fieldSerdes, 2, leafFrameType, lowKey, highKey, prefixLowKey, prefixHighKey);
+ }
}
@Test
@@ -87,8 +115,9 @@
ITupleReference lowKey = TupleUtils.createTuple(fieldSerdes, "cbf");
ITupleReference highKey = TupleUtils.createTuple(fieldSerdes, "cc7");
- runTest(fieldSerdes, 1, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, null, null);
- runTest(fieldSerdes, 1, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, null, null);
+ for (BTreeLeafFrameType leafFrameType : leafFrameTypesToTest) {
+ runTest(fieldSerdes, 1, leafFrameType, lowKey, highKey, null, null);
+ }
}
@Test
@@ -107,8 +136,9 @@
ITupleReference prefixLowKey = TupleUtils.createTuple(fieldSerdes, "cbf");
ITupleReference prefixHighKey = TupleUtils.createTuple(fieldSerdes, "cc7");
- runTest(fieldSerdes, 2, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
- runTest(fieldSerdes, 2, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
+ for (BTreeLeafFrameType leafFrameType : leafFrameTypesToTest) {
+ runTest(fieldSerdes, 2, leafFrameType, lowKey, highKey, prefixLowKey, prefixHighKey);
+ }
}
@Test
@@ -127,7 +157,8 @@
ITupleReference prefixLowKey = TupleUtils.createTuple(fieldSerdes, "cbf");
ITupleReference prefixHighKey = TupleUtils.createTuple(fieldSerdes, "cc7");
- runTest(fieldSerdes, 2, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
- runTest(fieldSerdes, 2, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
+ for (BTreeLeafFrameType leafFrameType : leafFrameTypesToTest) {
+ runTest(fieldSerdes, 2, leafFrameType, lowKey, highKey, prefixLowKey, prefixHighKey);
+ }
}
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexTestUtils.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexTestUtils.java
new file mode 100644
index 0000000..262a13e
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexTestUtils.java
@@ -0,0 +1,470 @@
+package edu.uci.ics.hyracks.storage.am.btree.tests;
+
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.util.Iterator;
+import java.util.NavigableSet;
+import java.util.Random;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeDuplicateKeyException;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+
+@SuppressWarnings("rawtypes")
+public class OrderedIndexTestUtils {
+ private static final Logger LOGGER = Logger.getLogger(OrderedIndexTestUtils.class.getName());
+
+ private static void compareActualAndExpected(ITupleReference actual, CheckTuple expected, ISerializerDeserializer[] fieldSerdes) throws HyracksDataException {
+ for (int i = 0; i < fieldSerdes.length; i++) {
+ ByteArrayInputStream inStream = new ByteArrayInputStream(
+ actual.getFieldData(i), actual.getFieldStart(i),
+ actual.getFieldLength(i));
+ DataInput dataIn = new DataInputStream(inStream);
+ Object actualObj = fieldSerdes[i].deserialize(dataIn);
+ if (!actualObj.equals(expected.get(i))) {
+ fail("Actual and expected fields do not match on field " + i + ".\nExpected: " + expected.get(i) + "\nActual : " + actualObj);
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static void createTupleFromCheckTuple(CheckTuple checkTuple, ArrayTupleBuilder tupleBuilder, ArrayTupleReference tuple, ISerializerDeserializer[] fieldSerdes) throws HyracksDataException {
+ int fieldCount = tupleBuilder.getFieldEndOffsets().length;
+ DataOutput dos = tupleBuilder.getDataOutput();
+ tupleBuilder.reset();
+ for (int i = 0; i < fieldCount; i++) {
+ fieldSerdes[i].serialize(checkTuple.get(i), dos);
+ tupleBuilder.addFieldEndOffset();
+ }
+ tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
+ }
+
+ @SuppressWarnings("unchecked")
+ private static CheckTuple createCheckTupleFromTuple(ITupleReference tuple, ISerializerDeserializer[] fieldSerdes, int numKeys) throws HyracksDataException {
+ CheckTuple checkTuple = new CheckTuple(fieldSerdes.length, numKeys);
+ int fieldCount = Math.min(fieldSerdes.length, tuple.getFieldCount());
+ for (int i = 0; i < fieldCount; i++) {
+ ByteArrayInputStream inStream = new ByteArrayInputStream(
+ tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i));
+ DataInput dataIn = new DataInputStream(inStream);
+ Comparable fieldObj = (Comparable)fieldSerdes[i].deserialize(dataIn);
+ checkTuple.add(fieldObj);
+ }
+ return checkTuple;
+ }
+
+ @SuppressWarnings("unchecked")
+ // Create a new TreeSet containing the elements satisfying the prefix search.
+ // Implementing prefix search by changing compareTo() in CheckTuple does not work.
+ public static TreeSet<CheckTuple> getPrefixExpectedSubset(TreeSet<CheckTuple> checkTuples, CheckTuple lowKey, CheckTuple highKey) {
+ TreeSet<CheckTuple> expectedSubset = new TreeSet<CheckTuple>();
+ Iterator<CheckTuple> iter = checkTuples.iterator();
+ while(iter.hasNext()) {
+ CheckTuple t = iter.next();
+ boolean geLowKey = true;
+ boolean leHighKey = true;
+ for (int i = 0; i < lowKey.getNumKeys(); i++) {
+ if (t.get(i).compareTo(lowKey.get(i)) < 0) {
+ geLowKey = false;
+ break;
+ }
+ }
+ for (int i = 0; i < highKey.getNumKeys(); i++) {
+ if (t.get(i).compareTo(highKey.get(i)) > 0) {
+ leHighKey = false;
+ break;
+ }
+ }
+ if (geLowKey && leHighKey) {
+ expectedSubset.add(t);
+ }
+ }
+ return expectedSubset;
+ }
+
+ public static void checkOrderedScan(IOrderedIndexTestContext ctx) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Testing Ordered Scan.");
+ }
+ ITreeIndexCursor scanCursor = ctx.getIndexAccessor().createSearchCursor();
+ RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
+ ctx.getIndexAccessor().search(scanCursor, nullPred);
+ Iterator<CheckTuple> checkIter = ctx.getCheckTuples().iterator();
+ int actualCount = 0;
+ try {
+ while (scanCursor.hasNext()) {
+ if (!checkIter.hasNext()) {
+ fail("Ordered scan returned more answers than expected.\nExpected: " + ctx.getCheckTuples().size());
+ }
+ scanCursor.next();
+ CheckTuple expectedTuple = checkIter.next();
+ ITupleReference tuple = scanCursor.getTuple();
+ compareActualAndExpected(tuple, expectedTuple, ctx.getFieldSerdes());
+ actualCount++;
+ }
+ if (actualCount < ctx.getCheckTuples().size()) {
+ fail("Ordered scan returned fewer answers than expected.\nExpected: " + ctx.getCheckTuples().size() + "\nActual : " + actualCount);
+ }
+ } finally {
+ scanCursor.close();
+ }
+ }
+
+ public static void checkDiskOrderScan(IOrderedIndexTestContext ctx) throws Exception {
+ try {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Testing Disk-Order Scan.");
+ }
+ ITreeIndexCursor diskOrderCursor = ctx.getIndexAccessor().createDiskOrderScanCursor();
+ ctx.getIndexAccessor().diskOrderScan(diskOrderCursor);
+ int actualCount = 0;
+ try {
+ while (diskOrderCursor.hasNext()) {
+ diskOrderCursor.next();
+ ITupleReference tuple = diskOrderCursor.getTuple();
+ CheckTuple checkTuple = createCheckTupleFromTuple(tuple, ctx.getFieldSerdes(), ctx.getKeyFieldCount());
+ if (!ctx.getCheckTuples().contains(checkTuple)) {
+ fail("Disk-order scan returned unexpected answer: " + checkTuple.toString());
+ }
+ actualCount++;
+ }
+ if (actualCount < ctx.getCheckTuples().size()) {
+ fail("Disk-order scan returned fewer answers than expected.\nExpected: "
+ + ctx.getCheckTuples().size() + "\nActual : " + actualCount);
+ }
+ if (actualCount > ctx.getCheckTuples().size()) {
+ fail("Disk-order scan returned more answers than expected.\nExpected: "
+ + ctx.getCheckTuples().size() + "\nActual : " + actualCount);
+ }
+ } finally {
+ diskOrderCursor.close();
+ }
+ } catch (UnsupportedOperationException e) {
+ // Ignore exception because some indexes, e.g. the LSMBTree, don't
+ // support disk-order scan.
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Ignoring disk-order scan since it's not supported.");
+ }
+ }
+ }
+
+ public static void checkRangeSearch(IOrderedIndexTestContext ctx, ITupleReference lowKey, ITupleReference highKey, boolean lowKeyInclusive, boolean highKeyInclusive) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Testing Range Search.");
+ }
+ MultiComparator lowKeyCmp = BTreeUtils.getSearchMultiComparator(ctx.getComparators(), lowKey);
+ MultiComparator highKeyCmp = BTreeUtils.getSearchMultiComparator(ctx.getComparators(), highKey);
+ ITreeIndexCursor searchCursor = ctx.getIndexAccessor().createSearchCursor();
+ RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, lowKeyInclusive, highKeyInclusive, lowKeyCmp, highKeyCmp);
+ ctx.getIndexAccessor().search(searchCursor, rangePred);
+ // Get the subset of elements from the expected set within given key range.
+ CheckTuple lowKeyCheck = createCheckTupleFromTuple(lowKey, ctx.getFieldSerdes(), lowKeyCmp.getKeyFieldCount());
+ CheckTuple highKeyCheck = createCheckTupleFromTuple(highKey, ctx.getFieldSerdes(), highKeyCmp.getKeyFieldCount());
+ NavigableSet<CheckTuple> expectedSubset = null;
+ if (lowKeyCmp.getKeyFieldCount() < ctx.getKeyFieldCount() ||
+ highKeyCmp.getKeyFieldCount() < ctx.getKeyFieldCount()) {
+ // Searching on a key prefix (low key or high key or both).
+ expectedSubset = getPrefixExpectedSubset(ctx.getCheckTuples(), lowKeyCheck, highKeyCheck);
+ } else {
+ // Searching on all key fields.
+ expectedSubset = ctx.getCheckTuples().subSet(lowKeyCheck, lowKeyInclusive, highKeyCheck, highKeyInclusive);
+ }
+ Iterator<CheckTuple> checkIter = expectedSubset.iterator();
+ int actualCount = 0;
+ try {
+ while (searchCursor.hasNext()) {
+ if (!checkIter.hasNext()) {
+ fail("Range search returned more answers than expected.\nExpected: " + expectedSubset.size());
+ }
+ searchCursor.next();
+ CheckTuple expectedTuple = checkIter.next();
+ ITupleReference tuple = searchCursor.getTuple();
+ compareActualAndExpected(tuple, expectedTuple, ctx.getFieldSerdes());
+ actualCount++;
+ }
+ if (actualCount < expectedSubset.size()) {
+ fail("Range search returned fewer answers than expected.\nExpected: " + expectedSubset.size() + "\nActual : " + actualCount);
+ }
+ } finally {
+ searchCursor.close();
+ }
+ }
+
+ public static void checkPointSearches(IOrderedIndexTestContext ctx) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Testing Point Searches On All Expected Keys.");
+ }
+
+ ITreeIndexCursor searchCursor = ctx.getIndexAccessor().createSearchCursor();
+
+ ArrayTupleBuilder lowKeyBuilder = new ArrayTupleBuilder(ctx.getKeyFieldCount());
+ ArrayTupleReference lowKey = new ArrayTupleReference();
+ ArrayTupleBuilder highKeyBuilder = new ArrayTupleBuilder(ctx.getKeyFieldCount());
+ ArrayTupleReference highKey = new ArrayTupleReference();
+ RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, true, true, null, null);
+
+ // Iterate through expected tuples, and perform a point search in the BTree to verify the tuple can be reached.
+ for (CheckTuple checkTuple : ctx.getCheckTuples()) {
+ createTupleFromCheckTuple(checkTuple, lowKeyBuilder, lowKey, ctx.getFieldSerdes());
+ createTupleFromCheckTuple(checkTuple, highKeyBuilder, highKey, ctx.getFieldSerdes());
+ MultiComparator lowKeyCmp = BTreeUtils.getSearchMultiComparator(ctx.getComparators(), lowKey);
+ MultiComparator highKeyCmp = BTreeUtils.getSearchMultiComparator(ctx.getComparators(), highKey);
+
+ rangePred.setLowKey(lowKey, true);
+ rangePred.setHighKey(highKey, true);
+ rangePred.setLowKeyComparator(lowKeyCmp);
+ rangePred.setHighKeyComparator(highKeyCmp);
+
+ ctx.getIndexAccessor().search(searchCursor, rangePred);
+
+ try {
+ // We expect exactly one answer.
+ if (searchCursor.hasNext()) {
+ searchCursor.next();
+ ITupleReference tuple = searchCursor.getTuple();
+ compareActualAndExpected(tuple, checkTuple, ctx.getFieldSerdes());
+ }
+ if (searchCursor.hasNext()) {
+ fail("Point search returned more than one answer.");
+ }
+ } finally {
+ searchCursor.close();
+ }
+ }
+ }
+
+ public static void insertIntTuples(IOrderedIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
+ int fieldCount = ctx.getFieldCount();
+ int numKeyFields = ctx.getKeyFieldCount();
+ int[] fieldValues = new int[ctx.getFieldCount()];
+ // Scale range of values according to number of keys.
+ // For example, for 2 keys we want the square root of numTuples, for 3 keys the cube root of numTuples, etc.
+ int maxValue = (int)Math.ceil(Math.pow(numTuples, 1.0/(double)numKeyFields));
+ for (int i = 0; i < numTuples; i++) {
+ // Set keys.
+ for (int j = 0; j < numKeyFields; j++) {
+ fieldValues[j] = rnd.nextInt() % maxValue;
+ }
+ // Set values.
+ for (int j = numKeyFields; j < fieldCount; j++) {
+ fieldValues[j] = j;
+ }
+ TupleUtils.createIntegerTuple(ctx.getTupleBuilder(), ctx.getTuple(), fieldValues);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
+ LOGGER.info("Inserting Tuple " + (i + 1) + "/" + numTuples);
+ }
+ }
+ try {
+ ctx.getIndexAccessor().insert(ctx.getTuple());
+ // Set expected values. Do this only after insertion succeeds because we ignore duplicate keys.
+ ctx.insertIntCheckTuple(fieldValues);
+ } catch (BTreeDuplicateKeyException e) {
+ // Ignore duplicate key insertions.
+ }
+ }
+ }
+
+ public static void insertStringTuples(IOrderedIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
+ int fieldCount = ctx.getFieldCount();
+ int numKeyFields = ctx.getKeyFieldCount();
+ String[] fieldValues = new String[fieldCount];
+ for (int i = 0; i < numTuples; i++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
+ LOGGER.info("Inserting Tuple " + (i + 1) + "/" + numTuples);
+ }
+ }
+ // Set keys.
+ for (int j = 0; j < numKeyFields; j++) {
+ int length = (Math.abs(rnd.nextInt()) % 10) + 1;
+ fieldValues[j] = getRandomString(length, rnd);
+ }
+ // Set values.
+ for (int j = numKeyFields; j < fieldCount; j++) {
+ fieldValues[j] = getRandomString(5, rnd);
+ }
+ TupleUtils.createTuple(ctx.getTupleBuilder(), ctx.getTuple(), ctx.getFieldSerdes(), (Object[])fieldValues);
+ try {
+ ctx.getIndexAccessor().insert(ctx.getTuple());
+ // Set expected values. Do this only after insertion succeeds because we ignore duplicate keys.
+ ctx.insertStringCheckTuple(fieldValues);
+ } catch (BTreeDuplicateKeyException e) {
+ // Ignore duplicate key insertions.
+ }
+ }
+ }
+
+ public static void bulkLoadIntTuples(IOrderedIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
+ int fieldCount = ctx.getFieldCount();
+ int numKeyFields = ctx.getKeyFieldCount();
+ int[] fieldValues = new int[ctx.getFieldCount()];
+ int maxValue = (int)Math.ceil(Math.pow(numTuples, 1.0/(double)numKeyFields));
+ for (int i = 0; i < numTuples; i++) {
+ // Set keys.
+ for (int j = 0; j < numKeyFields; j++) {
+ fieldValues[j] = rnd.nextInt() % maxValue;
+ }
+ // Set values.
+ for (int j = numKeyFields; j < fieldCount; j++) {
+ fieldValues[j] = j;
+ }
+
+ // Set expected values. We also use these as the pre-sorted stream for bulk loading.
+ ctx.insertIntCheckTuple(fieldValues);
+ }
+ bulkLoadCheckTuples(ctx, numTuples);
+ }
+
+ public static void bulkLoadStringTuples(IOrderedIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
+ int fieldCount = ctx.getFieldCount();
+ int numKeyFields = ctx.getKeyFieldCount();
+ String[] fieldValues = new String[fieldCount];
+ for (int i = 0; i < numTuples; i++) {
+ // Set keys.
+ for (int j = 0; j < numKeyFields; j++) {
+ int length = (Math.abs(rnd.nextInt()) % 10) + 1;
+ fieldValues[j] = getRandomString(length, rnd);
+ }
+ // Set values.
+ for (int j = numKeyFields; j < fieldCount; j++) {
+ fieldValues[j] = getRandomString(5, rnd);
+ }
+ // Set expected values. We also use these as the pre-sorted stream for bulk loading.
+ ctx.insertStringCheckTuple(fieldValues);
+ }
+ bulkLoadCheckTuples(ctx, numTuples);
+ }
+
+ private static void bulkLoadCheckTuples(IOrderedIndexTestContext ctx, int numTuples) throws HyracksDataException, TreeIndexException {
+ int fieldCount = ctx.getFieldCount();
+ ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ // Perform bulk load.
+ IIndexBulkLoadContext bulkLoadCtx = ctx.getIndex().beginBulkLoad(0.7f);
+ int c = 1;
+ for (CheckTuple checkTuple : ctx.getCheckTuples()) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (c % (numTuples / 10) == 0) {
+ LOGGER.info("Bulk Loading Tuple " + c + "/" + numTuples);
+ }
+ }
+ createTupleFromCheckTuple(checkTuple, tupleBuilder, tuple, ctx.getFieldSerdes());
+ ctx.getIndex().bulkLoadAddTuple(tuple, bulkLoadCtx);
+ c++;
+ }
+ ctx.getIndex().endBulkLoad(bulkLoadCtx);
+ }
+
+ public static void deleteTuples(IOrderedIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
+ ArrayTupleBuilder deleteTupleBuilder = new ArrayTupleBuilder(ctx.getKeyFieldCount());
+ ArrayTupleReference deleteTuple = new ArrayTupleReference();
+ int numCheckTuples = ctx.getCheckTuples().size();
+ // Copy CheckTuple references into array, so we can randomly pick from there.
+ CheckTuple[] checkTuples = new CheckTuple[numCheckTuples];
+ int idx = 0;
+ for (CheckTuple checkTuple : ctx.getCheckTuples()) {
+ checkTuples[idx++] = checkTuple;
+ }
+ for (int i = 0; i < numTuples && numCheckTuples > 0; i++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
+ LOGGER.info("Deleting Tuple " + (i + 1) + "/" + numTuples);
+ }
+ }
+ int checkTupleIdx = Math.abs(rnd.nextInt() % numCheckTuples);
+ CheckTuple checkTuple = checkTuples[checkTupleIdx];
+ createTupleFromCheckTuple(checkTuple, deleteTupleBuilder, deleteTuple, ctx.getFieldSerdes());
+ ctx.getIndexAccessor().delete(deleteTuple);
+
+ // Remove check tuple from expected results.
+ ctx.getCheckTuples().remove(checkTuple);
+
+ // Swap with last "valid" CheckTuple.
+ CheckTuple tmp = checkTuples[numCheckTuples - 1];
+ checkTuples[numCheckTuples - 1] = checkTuple;
+ checkTuples[checkTupleIdx] = tmp;
+ numCheckTuples--;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public static void updateTuples(IOrderedIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
+ int fieldCount = ctx.getFieldCount();
+ int keyFieldCount = ctx.getKeyFieldCount();
+ // This is a noop because we can only update non-key fields.
+ if (fieldCount == keyFieldCount) {
+ return;
+ }
+ ArrayTupleBuilder updateTupleBuilder = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference updateTuple = new ArrayTupleReference();
+ int numCheckTuples = ctx.getCheckTuples().size();
+ // Copy CheckTuple references into array, so we can randomly pick from there.
+ CheckTuple[] checkTuples = new CheckTuple[numCheckTuples];
+ int idx = 0;
+ for (CheckTuple checkTuple : ctx.getCheckTuples()) {
+ checkTuples[idx++] = checkTuple;
+ }
+ for (int i = 0; i < numTuples && numCheckTuples > 0; i++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
+ LOGGER.info("Updating Tuple " + (i + 1) + "/" + numTuples);
+ }
+ }
+ int checkTupleIdx = Math.abs(rnd.nextInt() % numCheckTuples);
+ CheckTuple checkTuple = checkTuples[checkTupleIdx];
+ // Update check tuple's non-key fields.
+ for (int j = keyFieldCount; j < fieldCount; j++) {
+ Comparable newValue = getRandomUpdateValue(ctx.getFieldSerdes()[j], rnd);
+ checkTuple.set(j, newValue);
+ }
+
+ createTupleFromCheckTuple(checkTuple, updateTupleBuilder, updateTuple, ctx.getFieldSerdes());
+ ctx.getIndexAccessor().update(updateTuple);
+
+ // Swap with last "valid" CheckTuple.
+ CheckTuple tmp = checkTuples[numCheckTuples - 1];
+ checkTuples[numCheckTuples - 1] = checkTuple;
+ checkTuples[checkTupleIdx] = tmp;
+ numCheckTuples--;
+ }
+ }
+
+ private static Comparable getRandomUpdateValue(ISerializerDeserializer serde, Random rnd) {
+ if (serde instanceof IntegerSerializerDeserializer) {
+ return Integer.valueOf(rnd.nextInt());
+ } else if (serde instanceof UTF8StringSerializerDeserializer) {
+ return getRandomString(10, rnd);
+ }
+ return null;
+ }
+
+ public static String getRandomString(int length, Random rnd) {
+ String s = Long.toHexString(Double.doubleToLongBits(rnd.nextDouble()));
+ StringBuilder strBuilder = new StringBuilder();
+ for (int i = 0; i < s.length() && i < length; i++) {
+ strBuilder.append(s.charAt(Math.abs(rnd.nextInt()) % s.length()));
+ }
+ return strBuilder.toString();
+ }
+}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexUpdateTest.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexUpdateTest.java
new file mode 100644
index 0000000..6022058
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexUpdateTest.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.tests;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+
+@SuppressWarnings("rawtypes")
+public abstract class OrderedIndexUpdateTest extends OrderedIndexTestDriver {
+
+ public OrderedIndexUpdateTest(BTreeLeafFrameType[] leafFrameTypesToTest) {
+ super(leafFrameTypesToTest);
+ }
+
+ private static final int numUpdateRounds = 3;
+
+ @Override
+ protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType,
+ ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey)
+ throws Exception {
+ // This is a noop because we can only update non-key fields.
+ if (fieldSerdes.length == numKeys) {
+ return;
+ }
+ IOrderedIndexTestContext ctx = createTestContext(fieldSerdes, numKeys, leafType);
+ // We assume all fieldSerdes are of the same type. Check the first one
+ // to determine which field types to generate.
+ if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
+ OrderedIndexTestUtils.insertIntTuples(ctx, numTuplesToInsert, getRandom());
+ } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
+ OrderedIndexTestUtils.insertStringTuples(ctx, numTuplesToInsert, getRandom());
+ }
+ int numTuplesPerDeleteRound = (int) Math.ceil((float) ctx.getCheckTuples().size() / (float) numUpdateRounds);
+ for (int j = 0; j < numUpdateRounds; j++) {
+ OrderedIndexTestUtils.updateTuples(ctx, numTuplesPerDeleteRound, getRandom());
+ OrderedIndexTestUtils.checkPointSearches(ctx);
+ OrderedIndexTestUtils.checkOrderedScan(ctx);
+ OrderedIndexTestUtils.checkDiskOrderScan(ctx);
+ OrderedIndexTestUtils.checkRangeSearch(ctx, lowKey, highKey, true, true);
+ if (prefixLowKey != null && prefixHighKey != null) {
+ OrderedIndexTestUtils.checkRangeSearch(ctx, prefixLowKey, prefixHighKey, true, true);
+ }
+ }
+ }
+
+ @Override
+ protected String getTestOpName() {
+ return "Update";
+ }
+}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java
index f6310af..a9adfbf 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java
@@ -31,18 +31,18 @@
return btree;
}
- public static MultiComparator getSearchMultiComparator(MultiComparator btreeCmp, ITupleReference searchKey) {
+ public static MultiComparator getSearchMultiComparator(IBinaryComparator[] cmps, ITupleReference searchKey) {
if (searchKey == null) {
- return btreeCmp;
+ return new MultiComparator(cmps);
}
- if (btreeCmp.getKeyFieldCount() == searchKey.getFieldCount()) {
- return btreeCmp;
+ if (cmps.length == searchKey.getFieldCount()) {
+ return new MultiComparator(cmps);
}
- IBinaryComparator[] cmps = new IBinaryComparator[searchKey.getFieldCount()];
+ IBinaryComparator[] newCmps = new IBinaryComparator[searchKey.getFieldCount()];
for (int i = 0; i < searchKey.getFieldCount(); i++) {
- cmps[i] = btreeCmp.getComparators()[i];
+ newCmps[i] = cmps[i];
}
- return new MultiComparator(cmps);
+ return new MultiComparator(newCmps);
}
public static ITreeIndexFrameFactory getLeafFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory, BTreeLeafFrameType leafType) throws BTreeException {
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IFreePageManager.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IFreePageManager.java
index a7901c8..045ff9d 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IFreePageManager.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IFreePageManager.java
@@ -4,7 +4,7 @@
public interface IFreePageManager {
public int getFreePage(ITreeIndexMetaDataFrame metaFrame)
- throws HyracksDataException, PageAllocationException;
+ throws HyracksDataException;
public void addFreePage(ITreeIndexMetaDataFrame metaFrame, int freePage)
throws HyracksDataException;
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndex.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndex.java
index 46d22d1..1c9dca3 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndex.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndex.java
@@ -47,12 +47,10 @@
* If the BufferCache throws while un/pinning or un/latching.
* @throws TreeIndexException
* If the tree is not empty.
- * @throws PageAllocationException
* @returns A new context for bulk loading, required for appending tuples.
*/
public IIndexBulkLoadContext beginBulkLoad(float fillFactor)
- throws TreeIndexException, HyracksDataException,
- PageAllocationException;
+ throws TreeIndexException, HyracksDataException;
/**
* Append a tuple to the index in the context of a bulk load.
@@ -63,11 +61,9 @@
* Existing bulk load context.
* @throws HyracksDataException
* If the BufferCache throws while un/pinning or un/latching.
- * @throws PageAllocationException
*/
public void bulkLoadAddTuple(ITupleReference tuple,
- IIndexBulkLoadContext ictx) throws HyracksDataException,
- PageAllocationException;
+ IIndexBulkLoadContext ictx) throws HyracksDataException;
/**
* Finalize the bulk loading operation in the given context.
@@ -76,10 +72,9 @@
* Existing bulk load context to be finalized.
* @throws HyracksDataException
* If the BufferCache throws while un/pinning or un/latching.
- * @throws PageAllocationException
*/
public void endBulkLoad(IIndexBulkLoadContext ictx)
- throws HyracksDataException, PageAllocationException;
+ throws HyracksDataException;
/**
* @return The index's leaf frame factory.
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexAccessor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexAccessor.java
index 1e679b2..e0271f3 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexAccessor.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexAccessor.java
@@ -36,10 +36,9 @@
* @throws TreeIndexException
* If an index-specific constraint is violated, e.g., the key
* already exists.
- * @throws PageAllocationException
*/
public void insert(ITupleReference tuple) throws HyracksDataException,
- TreeIndexException, PageAllocationException;
+ TreeIndexException;
/**
* Updates the tuple in the index matching the given tuple with the new
@@ -52,10 +51,9 @@
* If the BufferCache throws while un/pinning or un/latching.
* @throws TreeIndexException
* If there is no matching tuple in the index.
- * @throws PageAllocationException
*/
public void update(ITupleReference tuple) throws HyracksDataException,
- TreeIndexException, PageAllocationException;
+ TreeIndexException;
/**
* Deletes the tuple in the index matching the given tuple.
@@ -66,12 +64,17 @@
* If the BufferCache throws while un/pinning or un/latching.
* @throws TreeIndexException
* If there is no matching tuple in the index.
- * @throws PageAllocationException
*/
public void delete(ITupleReference tuple) throws HyracksDataException,
- TreeIndexException, PageAllocationException;
+ TreeIndexException;
/**
+ * Creates a cursor appropriate for passing into search().
+ *
+ */
+ public ITreeIndexCursor createSearchCursor();
+
+ /**
* Open the given cursor for an index search using the given predicate as
* search condition.
*
@@ -82,12 +85,17 @@
* @throws HyracksDataException
* If the BufferCache throws while un/pinning or un/latching.
* @throws TreeIndexException
- * @throws PageAllocationException
*/
public void search(ITreeIndexCursor cursor, ISearchPredicate searchPred)
- throws HyracksDataException, TreeIndexException, PageAllocationException;
+ throws HyracksDataException, TreeIndexException;
/**
+ * Creates a cursor appropriate for passing into diskOrderScan().
+ *
+ */
+ public ITreeIndexCursor createDiskOrderScanCursor();
+
+ /**
* Open the given cursor for a disk-order scan, positioning the cursor to
* the first leaf tuple.
*
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexCursor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexCursor.java
index d3ce3867..24e552d 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexCursor.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexCursor.java
@@ -23,16 +23,16 @@
public interface ITreeIndexCursor {
public void reset();
- public boolean hasNext() throws Exception;
+ public boolean hasNext() throws HyracksDataException;
- public void next() throws Exception;
+ public void next() throws HyracksDataException;
public void open(ICursorInitialState initialState,
ISearchPredicate searchPred) throws HyracksDataException;
public ICachedPage getPage();
- public void close() throws Exception;
+ public void close() throws HyracksDataException;
public void setBufferCache(IBufferCache bufferCache);
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexTupleWriter.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexTupleWriter.java
index f0bb7aa..30e8f39 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexTupleWriter.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexTupleWriter.java
@@ -26,8 +26,7 @@
public int bytesRequired(ITupleReference tuple);
- // TODO: change to byte[] as well.
- public int writeTupleFields(ITupleReference tuple, int startField, int numFields, ByteBuffer targetBuf,
+ public int writeTupleFields(ITupleReference tuple, int startField, int numFields, byte[] targetBuf,
int targetOff);
public int bytesRequired(ITupleReference tuple, int startField, int numFields);
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndex.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndex.java
index 38b275b..3543564 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndex.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndex.java
@@ -38,10 +38,10 @@
* @param indexFileId
* The file id backing this index.
*/
- public void open(int indexFileId);
+ public void open(int indexFileId) throws HyracksDataException;
/**
* Closes the index.
*/
- public void close();
+ public void close() throws HyracksDataException;
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorNodePushable.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorNodePushable.java
index bc2d67e..fe3b10f 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorNodePushable.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorNodePushable.java
@@ -24,7 +24,6 @@
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
public class TreeIndexBulkLoadOperatorNodePushable extends AbstractUnaryInputSinkOperatorNodePushable {
private float fillFactor;
@@ -70,11 +69,7 @@
int tupleCount = accessor.getTupleCount();
for (int i = 0; i < tupleCount; i++) {
tuple.reset(accessor, i);
- try {
- treeIndex.bulkLoadAddTuple(tuple, bulkLoadCtx);
- } catch (PageAllocationException e) {
- throw new HyracksDataException(e);
- }
+ treeIndex.bulkLoadAddTuple(tuple, bulkLoadCtx);
}
}
@@ -82,7 +77,7 @@
public void close() throws HyracksDataException {
try {
treeIndex.endBulkLoad(bulkLoadCtx);
- } catch (PageAllocationException e) {
+ } catch (Exception e) {
throw new HyracksDataException(e);
} finally {
treeIndexHelper.deinit();
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DataGenThread.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DataGenThread.java
new file mode 100644
index 0000000..4bccba4
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DataGenThread.java
@@ -0,0 +1,86 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+
+/**
+ * Quick & dirty data generator for performance testing.
+ *
+ */
+public class DataGenThread extends Thread {
+ public final BlockingQueue<TupleBatch> tupleBatchQueue;
+ private final int maxNumBatches;
+ private final int maxOutstandingBatches;
+ private int numBatches;
+ private final boolean sorted;
+ private final Random rnd;
+
+ // maxOutstandingBatches pre-created tuple-batches for populating the queue.
+ private TupleBatch[] tupleBatches;
+ private int ringPos;
+
+ public DataGenThread(int maxNumBatches, int batchSize, int maxOutstandingBatches, int numConsumers, ISerializerDeserializer[] fieldSerdes, int payloadSize, int rndSeed, boolean sorted) {
+ this.maxNumBatches = maxNumBatches;
+ this.maxOutstandingBatches = maxOutstandingBatches;
+ this.sorted = sorted;
+ rnd = new Random(rndSeed);
+ tupleBatches = new TupleBatch[maxOutstandingBatches];
+ IFieldValueGenerator[] fieldGens = new IFieldValueGenerator[fieldSerdes.length];
+ for (int i = 0; i < fieldSerdes.length; i++) {
+ fieldGens[i] = getFieldGenFromSerde(fieldSerdes[i]);
+ }
+ for (int i = 0; i < maxOutstandingBatches; i++) {
+ tupleBatches[i] = new TupleBatch(batchSize, fieldGens, fieldSerdes, payloadSize);
+ }
+ // make sure we don't overwrite tuples that are in use by consumers.
+ // -1 because we first generate a new tuple, and then try to put it into the queue.
+ int capacity = Math.max(maxOutstandingBatches - numConsumers - 1, 1);
+ tupleBatchQueue = new LinkedBlockingQueue<TupleBatch>(capacity);
+ ringPos = 0;
+ }
+
+ @Override
+ public void run() {
+ while(numBatches < maxNumBatches) {
+ try {
+ tupleBatches[ringPos].generate();
+ tupleBatchQueue.put(tupleBatches[ringPos]);
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ numBatches++;
+ ringPos++;
+ if (ringPos >= maxOutstandingBatches) {
+ ringPos = 0;
+ }
+ }
+ }
+
+ public IFieldValueGenerator getFieldGenFromSerde(ISerializerDeserializer serde) {
+ if (serde instanceof IntegerSerializerDeserializer) {
+ if (sorted) {
+ return new SortedIntegerFieldValueGenerator();
+ } else {
+ return new IntegerFieldValueGenerator(rnd);
+ }
+ }
+ System.out.println("NULL");
+ //if (serde instanceof Integer64SerializerDeserializer) {
+ // throw new UnsupportedOperationException("Binary comparator factory for Integer64 not implemented.");
+ //}
+ //if (serde instanceof FloatSerializerDeserializer) {
+ // return FloatBinaryComparatorFactory.INSTANCE;
+ //}
+ //if (serde instanceof DoubleSerializerDeserializer) {
+ // return DoubleBinaryComparatorFactory.INSTANCE;
+ //}
+ return null;
+ }
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/IFieldValueGenerator.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/IFieldValueGenerator.java
new file mode 100644
index 0000000..ee0d30b
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/IFieldValueGenerator.java
@@ -0,0 +1,5 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+public interface IFieldValueGenerator<T> {
+ public T next();
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java
new file mode 100644
index 0000000..134b1f7
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java
@@ -0,0 +1,16 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+import java.util.Random;
+
+public class IntegerFieldValueGenerator implements IFieldValueGenerator<Integer> {
+ protected final Random rnd;
+
+ public IntegerFieldValueGenerator(Random rnd) {
+ this.rnd = rnd;
+ }
+
+ @Override
+ public Integer next() {
+ return rnd.nextInt();
+ }
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java
new file mode 100644
index 0000000..8f7fdcf
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java
@@ -0,0 +1,17 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+public class SortedIntegerFieldValueGenerator implements IFieldValueGenerator<Integer> {
+ private int val = 0;
+
+ public SortedIntegerFieldValueGenerator() {
+ }
+
+ public SortedIntegerFieldValueGenerator(int startVal) {
+ val = startVal;
+ }
+
+ @Override
+ public Integer next() {
+ return val++;
+ }
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/TupleBatch.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/TupleBatch.java
new file mode 100644
index 0000000..4c202c0
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/TupleBatch.java
@@ -0,0 +1,33 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+public class TupleBatch {
+ private final int size;
+ private final TupleGenerator[] tupleGens;
+
+ public TupleBatch(int size, IFieldValueGenerator[] fieldGens, ISerializerDeserializer[] fieldSerdes, int payloadSize) {
+ this.size = size;
+ tupleGens = new TupleGenerator[size];
+ for (int i = 0; i < size; i++) {
+ tupleGens[i] = new TupleGenerator(fieldGens, fieldSerdes, payloadSize);
+ }
+ }
+
+ public void generate() throws IOException {
+ for(TupleGenerator tupleGen : tupleGens) {
+ tupleGen.next();
+ }
+ }
+
+ public int size() {
+ return size;
+ }
+
+ public ITupleReference get(int ix) {
+ return tupleGens[ix].get();
+ }
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/TupleGenerator.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/TupleGenerator.java
new file mode 100644
index 0000000..2801205
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/TupleGenerator.java
@@ -0,0 +1,51 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+@SuppressWarnings({"rawtypes", "unchecked" })
+public class TupleGenerator {
+ protected final ISerializerDeserializer[] fieldSerdes;
+ protected final IFieldValueGenerator[] fieldGens;
+ protected final ArrayTupleBuilder tb;
+ protected final ArrayTupleReference tuple;
+ protected final byte[] payload;
+ protected final DataOutput tbDos;
+
+ public TupleGenerator(IFieldValueGenerator[] fieldGens, ISerializerDeserializer[] fieldSerdes, int payloadSize) {
+ this.fieldSerdes = fieldSerdes;
+ this.fieldGens = fieldGens;
+ tuple = new ArrayTupleReference();
+ if (payloadSize > 0) {
+ tb = new ArrayTupleBuilder(fieldSerdes.length + 1);
+ payload = new byte[payloadSize];
+ } else {
+ tb = new ArrayTupleBuilder(fieldSerdes.length);
+ payload = null;
+ }
+ tbDos = tb.getDataOutput();
+ }
+
+ public ITupleReference next() throws IOException {
+ tb.reset();
+ for (int i = 0; i < fieldSerdes.length; i++) {
+ fieldSerdes[i].serialize(fieldGens[i].next(), tbDos);
+ tb.addFieldEndOffset();
+ }
+ if (payload != null) {
+ tbDos.write(payload);
+ tb.addFieldEndOffset();
+ }
+ tuple.reset(tb.getFieldEndOffsets(), tb.getByteArray());
+ return tuple;
+ }
+
+ public ITupleReference get() {
+ return tuple;
+ }
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java
new file mode 100644
index 0000000..bcf7e70
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.freepage;
+
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+public class LinkedListFreePageManagerFactory {
+
+ private final ITreeIndexMetaDataFrameFactory metaDataFrameFactory;
+ private final IBufferCache bufferCache;
+
+ public LinkedListFreePageManagerFactory(IBufferCache bufferCache, ITreeIndexMetaDataFrameFactory metaDataFrameFactory) {
+ this.metaDataFrameFactory = metaDataFrameFactory;
+ this.bufferCache = bufferCache;
+ }
+
+ public IFreePageManager createFreePageManager(int fileId) {
+ return new LinkedListFreePageManager(bufferCache, fileId, 0, metaDataFrameFactory);
+ }
+
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/TreeDiskOrderScanCursor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/TreeDiskOrderScanCursor.java
index 1a81c3f..15b63ec 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/TreeDiskOrderScanCursor.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/TreeDiskOrderScanCursor.java
@@ -43,7 +43,7 @@
}
@Override
- public void close() throws Exception {
+ public void close() throws HyracksDataException {
page.releaseReadLatch();
bufferCache.unpin(page);
page = null;
@@ -84,7 +84,7 @@
}
@Override
- public boolean hasNext() throws Exception {
+ public boolean hasNext() throws HyracksDataException {
if (currentPageId > maxPageId) {
return false;
}
@@ -102,7 +102,7 @@
}
@Override
- public void next() throws Exception {
+ public void next() throws HyracksDataException {
tupleIndex++;
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/MultiComparator.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/MultiComparator.java
index df24484..d911d96 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/MultiComparator.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/MultiComparator.java
@@ -59,7 +59,7 @@
return cmps;
}
- public int getKeyFieldCount() {
+ public int getKeyFieldCount() {
return cmps.length;
}
}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/SimpleTupleWriter.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/SimpleTupleWriter.java
index 831247e..f5ec5f3 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/SimpleTupleWriter.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/SimpleTupleWriter.java
@@ -29,12 +29,6 @@
buf[targetOff + 1] = (byte)(s >> 0);
}
- // Write short in big endian to target byte array at given offset.
- private static void writeShortB(short s, byte[] buf, int targetOff) {
- buf[targetOff] = (byte) (s >> 0);
- buf[targetOff + 1] = (byte) (s >> 8);
- }
-
@Override
public int bytesRequired(ITupleReference tuple) {
int bytes = getNullFlagsBytes(tuple) + getFieldSlotsBytes(tuple);
@@ -84,23 +78,23 @@
}
@Override
- public int writeTupleFields(ITupleReference tuple, int startField, int numFields, ByteBuffer targetBuf,
+ public int writeTupleFields(ITupleReference tuple, int startField, int numFields, byte[] targetBuf,
int targetOff) {
int runner = targetOff;
int nullFlagsBytes = getNullFlagsBytes(tuple, startField, numFields);
for (int i = 0; i < nullFlagsBytes; i++) {
- targetBuf.put(runner++, (byte) 0);
+ targetBuf[runner++] = (byte) 0;
}
runner += getFieldSlotsBytes(tuple, startField, numFields);
int fieldEndOff = 0;
int fieldCounter = 0;
for (int i = startField; i < startField + numFields; i++) {
- System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf.array(), runner,
+ System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf, runner,
tuple.getFieldLength(i));
fieldEndOff += tuple.getFieldLength(i);
- runner += tuple.getFieldLength(i);
- targetBuf.putShort(targetOff + nullFlagsBytes + fieldCounter * 2, (short) fieldEndOff);
+ runner += tuple.getFieldLength(i);
+ writeShortL((short) fieldEndOff, targetBuf, targetOff + nullFlagsBytes + fieldCounter * 2);
fieldCounter++;
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriter.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriter.java
index fe52608..9730346 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriter.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriter.java
@@ -79,8 +79,6 @@
// write data fields
for (int i = 0; i < tuple.getFieldCount(); i++) {
- int s = tuple.getFieldStart(i);
- int l = tuple.getFieldLength(i);
System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf, runner, tuple.getFieldLength(i));
runner += tuple.getFieldLength(i);
}
@@ -89,17 +87,17 @@
}
@Override
- public int writeTupleFields(ITupleReference tuple, int startField, int numFields, ByteBuffer targetBuf,
+ public int writeTupleFields(ITupleReference tuple, int startField, int numFields, byte[] targetBuf,
int targetOff) {
int runner = targetOff;
int nullFlagsBytes = getNullFlagsBytes(numFields);
// write null indicator bits
for (int i = 0; i < nullFlagsBytes; i++) {
- targetBuf.put(runner++, (byte) 0);
+ targetBuf[runner++] = (byte) 0;
}
// write field slots for variable length fields
- encDec.reset(targetBuf.array(), runner);
+ encDec.reset(targetBuf, runner);
for (int i = startField; i < startField + numFields; i++) {
if (!typeTraits[i].isFixedLength()) {
encDec.encode(tuple.getFieldLength(i));
@@ -108,7 +106,7 @@
runner = encDec.getPos();
for (int i = startField; i < startField + numFields; i++) {
- System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf.array(), runner,
+ System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf, runner,
tuple.getFieldLength(i));
runner += tuple.getFieldLength(i);
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java
index 9dcabb4..f6cd4cd 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java
@@ -23,7 +23,6 @@
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
import edu.uci.ics.hyracks.storage.am.common.dataflow.PermutingFrameTupleReference;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
@@ -97,11 +96,7 @@
int tupleCount = accessor.getTupleCount();
for (int i = 0; i < tupleCount; i++) {
tuple.reset(accessor, i);
- try {
- invIndex.bulkLoadAddTuple(bulkLoadCtx, tuple);
- } catch (PageAllocationException e) {
- throw new HyracksDataException(e);
- }
+ invIndex.bulkLoadAddTuple(bulkLoadCtx, tuple);
}
}
@@ -109,7 +104,7 @@
public void close() throws HyracksDataException {
try {
invIndex.endBulkLoad(bulkLoadCtx);
- } catch (PageAllocationException e) {
+ } catch (Exception e) {
throw new HyracksDataException(e);
} finally {
try {
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
index 986e57b..d4b5827 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
@@ -33,7 +33,6 @@
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
-import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
@@ -85,7 +84,7 @@
}
public BulkLoadContext beginBulkLoad(IInvertedListBuilder invListBuilder, int hyracksFrameSize,
- float btreeFillFactor) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ float btreeFillFactor) throws HyracksDataException, TreeIndexException {
BulkLoadContext ctx = new BulkLoadContext(invListBuilder, hyracksFrameSize, btreeFillFactor);
ctx.init(rootPageId, fileId);
return ctx;
@@ -97,7 +96,7 @@
// the next invListCmp.getKeyFieldCount() fields in tuple are keys of the
// inverted list (e.g., primary key)
// key fields of inverted list are fixed size
- public void bulkLoadAddTuple(BulkLoadContext ctx, ITupleReference tuple) throws HyracksDataException, PageAllocationException {
+ public void bulkLoadAddTuple(BulkLoadContext ctx, ITupleReference tuple) throws HyracksDataException {
// first inverted list, copy token to baaos and start new list
if (ctx.currentInvListTokenBaaos.size() == 0) {
@@ -190,7 +189,7 @@
return ret;
}
- public void createAndInsertBTreeTuple(BulkLoadContext ctx) throws HyracksDataException, PageAllocationException {
+ public void createAndInsertBTreeTuple(BulkLoadContext ctx) throws HyracksDataException {
// build tuple
ctx.btreeTupleBuilder.reset();
ctx.btreeTupleBuilder.addField(ctx.currentInvListTokenBaaos.getByteArray(), 0,
@@ -211,7 +210,7 @@
btree.bulkLoadAddTuple(ctx.btreeFrameTupleReference, ctx.btreeBulkLoadCtx);
}
- public void endBulkLoad(BulkLoadContext ctx) throws HyracksDataException, PageAllocationException {
+ public void endBulkLoad(BulkLoadContext ctx) throws HyracksDataException {
// create entry in btree for last inverted list
createAndInsertBTreeTuple(ctx);
btree.endBulkLoad(ctx.btreeBulkLoadCtx);
@@ -274,7 +273,7 @@
this.btreeFillFactor = btreeFillFactor;
}
- public void init(int startPageId, int fileId) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ public void init(int startPageId, int fileId) throws HyracksDataException, TreeIndexException {
btreeBulkLoadCtx = btree.beginBulkLoad(BTree.DEFAULT_FILL_FACTOR);
currentPageId = startPageId;
currentPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), true);
diff --git a/hyracks-storage-am-lsm-btree/pom.xml b/hyracks-storage-am-lsm-btree/pom.xml
new file mode 100644
index 0000000..4f65bd5
--- /dev/null
+++ b/hyracks-storage-am-lsm-btree/pom.xml
@@ -0,0 +1,56 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-btree</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-test-support</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-btree</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-common</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <type>jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/BTreeFactory.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/BTreeFactory.java
new file mode 100644
index 0000000..b9a1c2a
--- /dev/null
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/BTreeFactory.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.impls;
+
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+public class BTreeFactory {
+
+ private IBufferCache bufferCache;
+ private int fieldCount;
+ private MultiComparator cmp;
+ private ITreeIndexFrameFactory interiorFrameFactory;
+ private ITreeIndexFrameFactory leafFrameFactory;
+ private LinkedListFreePageManagerFactory freePageManagerFactory;
+
+ public BTreeFactory(IBufferCache bufferCache, LinkedListFreePageManagerFactory freePageManagerFactory, MultiComparator cmp,
+ int fieldCount, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
+ this.bufferCache = bufferCache;
+ this.fieldCount = fieldCount;
+ this.cmp = cmp;
+ this.interiorFrameFactory = interiorFrameFactory;
+ this.leafFrameFactory = leafFrameFactory;
+ this.freePageManagerFactory = freePageManagerFactory;
+ }
+
+ public BTree createBTreeInstance(int fileId) {
+ return new BTree(bufferCache, fieldCount, cmp, freePageManagerFactory.createFreePageManager(fileId),
+ interiorFrameFactory, leafFrameFactory);
+ }
+
+ public IBufferCache getBufferCache() {
+ return bufferCache;
+ }
+}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/LSMBTree.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/LSMBTree.java
new file mode 100644
index 0000000..2563cda
--- /dev/null
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/LSMBTree.java
@@ -0,0 +1,564 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.impls;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.ListIterator;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeDuplicateKeyException;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeNonExistentKeyException;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFileNameManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMTree;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+
+public class LSMBTree implements ILSMTree {
+ protected final Logger LOGGER = Logger.getLogger(LSMBTree.class.getName());
+
+ // In-memory components.
+ private final BTree memBTree;
+ private final InMemoryFreePageManager memFreePageManager;
+
+ // On-disk components.
+ private final ILSMFileNameManager fileNameManager;
+ // For creating BTree's used in flush and merge.
+ private final BTreeFactory diskBTreeFactory;
+ // For creating BTree's used in bulk load. Different from diskBTreeFactory
+ // because it should have a different tuple writer in it's leaf frames.
+ private final BTreeFactory bulkLoadBTreeFactory;
+ private final IBufferCache diskBufferCache;
+ private final IFileMapProvider diskFileMapProvider;
+ private LinkedList<BTree> diskBTrees = new LinkedList<BTree>();
+
+ // Common for in-memory and on-disk components.
+ private final ITreeIndexFrameFactory insertLeafFrameFactory;
+ private final ITreeIndexFrameFactory deleteLeafFrameFactory;
+ private final MultiComparator cmp;
+
+ // For dealing with concurrent accesses.
+ private int threadRefCount;
+ private boolean flushFlag;
+
+ public LSMBTree(IBufferCache memBufferCache, InMemoryFreePageManager memFreePageManager,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory,
+ ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMFileNameManager fileNameManager, BTreeFactory diskBTreeFactory,
+ BTreeFactory bulkLoadBTreeFactory, IFileMapProvider diskFileMapProvider, int fieldCount, MultiComparator cmp) {
+ memBTree = new BTree(memBufferCache, fieldCount, cmp, memFreePageManager, interiorFrameFactory,
+ insertLeafFrameFactory);
+ this.memFreePageManager = memFreePageManager;
+ this.insertLeafFrameFactory = insertLeafFrameFactory;
+ this.deleteLeafFrameFactory = deleteLeafFrameFactory;
+ this.diskBufferCache = diskBTreeFactory.getBufferCache();
+ this.diskFileMapProvider = diskFileMapProvider;
+ this.diskBTreeFactory = diskBTreeFactory;
+ this.bulkLoadBTreeFactory = bulkLoadBTreeFactory;
+ this.cmp = cmp;
+ this.diskBTrees = new LinkedList<BTree>();
+ this.threadRefCount = 0;
+ this.flushFlag = false;
+ this.fileNameManager = fileNameManager;
+ }
+
+ @Override
+ public void create(int indexFileId) throws HyracksDataException {
+ memBTree.create(indexFileId);
+ }
+
+ /**
+ * Opens LSMBTree, assuming a consistent state of the disk-resident
+ * components. In particular, registers all files in in base dir of
+ * fileNameManager as on-disk BTrees.
+ *
+ * Example pathological scenario to explain "consistent state assumption":
+ * Suppose a merge finished, but before the original files were deleted the
+ * system crashes. We are left in a state where we have the original BTrees
+ * in addition to the merged one. We assume that prior to calling this
+ * method a separate recovery process has ensured the consistent of the
+ * disk-resident components.
+ *
+ * @param indexFileId
+ * Dummy file id used for in-memory BTree.
+ * @throws HyracksDataException
+ */
+ @Override
+ public void open(int indexFileId) throws HyracksDataException {
+ memBTree.open(indexFileId);
+ File dir = new File(fileNameManager.getBaseDir());
+ FilenameFilter filter = new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return !name.startsWith(".");
+ }
+ };
+ String[] files = dir.list(filter);
+ if (files == null) {
+ return;
+ }
+ Comparator<String> fileNameCmp = fileNameManager.getFileNameComparator();
+ Arrays.sort(files, fileNameCmp);
+ for (String fileName : files) {
+ BTree btree = createDiskBTree(diskBTreeFactory, fileName, false);
+ diskBTrees.add(btree);
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ for (BTree btree : diskBTrees) {
+ diskBufferCache.closeFile(btree.getFileId());
+ btree.close();
+ }
+ diskBTrees.clear();
+ memBTree.close();
+ }
+
+ private void lsmPerformOp(ITupleReference tuple, LSMBTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
+ boolean waitForFlush = true;
+ do {
+ // Wait for ongoing flush to complete.
+ synchronized (this) {
+ if (!flushFlag) {
+ // Increments threadRefCount, to force a flush to wait for this operation to finish.
+ // (a flush can only begin once threadRefCount == 0).
+ threadEnter();
+ // Proceed with operation.
+ waitForFlush = false;
+ }
+ }
+ } while (waitForFlush);
+ // TODO: This will become much simpler once the BTree supports a true upsert operation.
+ try {
+ ctx.memBTreeAccessor.insert(tuple);
+ } catch (BTreeDuplicateKeyException e) {
+ // Notice that a flush must wait for the current operation to
+ // finish (threadRefCount must reach zero).
+ // TODO: This methods below are very inefficient, we'd rather like
+ // to flip the antimatter bit one single BTree traversal.
+ if (ctx.getIndexOp() == IndexOp.DELETE) {
+ deleteExistingKey(tuple, ctx);
+ } else {
+ insertOrUpdateExistingKey(tuple, ctx);
+ }
+ }
+ threadExit();
+ }
+
+ private void deleteExistingKey(ITupleReference tuple, LSMBTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
+ // We assume that tuple given by the user for deletion only contains the
+ // key fields, but not any non-key fields.
+ // Therefore, to set the delete bit in the tuple that already exist in
+ // the BTree, we must retrieve the original tuple first. This is to
+ // ensure that we have the proper value field.
+ if (cmp.getKeyFieldCount() != memBTree.getFieldCount()) {
+ ctx.reset(IndexOp.SEARCH);
+ RangePredicate rangePredicate = new RangePredicate(true, tuple, tuple, true, true, cmp, cmp);
+ ITreeIndexCursor cursor = ctx.memBTreeAccessor.createSearchCursor();
+ ctx.memBTreeAccessor.search(cursor, rangePredicate);
+ ITupleReference tupleCopy = null;
+ try {
+ if (cursor.hasNext()) {
+ cursor.next();
+ tupleCopy = TupleUtils.copyTuple(cursor.getTuple());
+ }
+ } finally {
+ cursor.close();
+ }
+ // This means the tuple we are looking for must have been truly deleted by another thread.
+ // Simply restart the original operation to insert the antimatter tuple.
+ // There is a remote chance of livelocks due to this behavior.
+ if (tupleCopy == null) {
+ ctx.reset(IndexOp.DELETE);
+ lsmPerformOp(tuple, ctx);
+ return;
+ }
+ memBTreeUpdate(tupleCopy, ctx);
+ } else {
+ // Since the existing tuple could be a matter tuple, we must delete it and re-insert.
+ memBTreeDeleteAndReinsert(tuple, ctx);
+ }
+ }
+
+ private void insertOrUpdateExistingKey(ITupleReference tuple, LSMBTreeOpContext ctx) throws HyracksDataException,
+ TreeIndexException {
+ // If all fields are keys, and the key we are trying to insert/update
+ // already exists, then we are already done.
+ // Otherwise, we must update the non-key fields.
+ if (cmp.getKeyFieldCount() != memBTree.getFieldCount()) {
+ memBTreeUpdate(tuple, ctx);
+ } else {
+ // Since the existing tuple could be an antimatter tuple, we must delete it and re-insert.
+ memBTreeDeleteAndReinsert(tuple, ctx);
+ }
+ }
+
+ private void memBTreeDeleteAndReinsert(ITupleReference tuple, LSMBTreeOpContext ctx) throws HyracksDataException,
+ TreeIndexException {
+ // All fields are key fields, therefore a true BTree update is not
+ // allowed.
+ // In order to set/unset the antimatter bit, we
+ // must truly delete the existing tuple from the BTree, and then
+ // re-insert it (with the antimatter bit set/unset).
+ // Since the tuple given by the user already has all fields, we
+ // don't need to retrieve the already existing tuple.
+ IndexOp originalOp = ctx.getIndexOp();
+ try {
+ ctx.memBTreeAccessor.delete(tuple);
+ } catch (BTreeNonExistentKeyException e) {
+ // Tuple has been deleted in the meantime. We will restart
+ // our operation anyway.
+ }
+ // Restart performOp to insert the tuple.
+ ctx.reset(originalOp);
+ lsmPerformOp(tuple, ctx);
+ }
+
+ private void memBTreeUpdate(ITupleReference tuple, LSMBTreeOpContext ctx) throws HyracksDataException,
+ TreeIndexException {
+ IndexOp originalOp = ctx.getIndexOp();
+ try {
+ ctx.reset(IndexOp.UPDATE);
+ ctx.memBTreeAccessor.update(tuple);
+ } catch (BTreeNonExistentKeyException e) {
+ // It is possible that the key has truly been deleted.
+ // Simply restart the operation. There is a remote chance of
+ // livelocks due to this behavior.
+ ctx.reset(originalOp);
+ lsmPerformOp(tuple, ctx);
+ }
+ }
+
+ public void threadEnter() {
+ threadRefCount++;
+ }
+
+ public void threadExit() throws HyracksDataException, TreeIndexException {
+ synchronized (this) {
+ threadRefCount--;
+ // Check if we've reached or exceeded the maximum number of pages.
+ if (!flushFlag && memFreePageManager.isFull()) {
+ flushFlag = true;
+ }
+ // Flush will only be handled by last exiting thread.
+ if (flushFlag && threadRefCount == 0) {
+ flush();
+ flushFlag = false;
+ }
+ }
+ }
+
+ @Override
+ public void flush() throws HyracksDataException, TreeIndexException {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Flushing LSM-BTree.");
+ }
+ // Bulk load a new on-disk BTree from the in-memory BTree.
+ RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
+ ITreeIndexAccessor memBTreeAccessor = memBTree.createAccessor();
+ ITreeIndexCursor scanCursor = memBTreeAccessor.createSearchCursor();
+ memBTreeAccessor.search(scanCursor, nullPred);
+ BTree diskBTree = createFlushTargetBTree();
+ // Bulk load the tuples from the in-memory BTree into the new disk BTree.
+ IIndexBulkLoadContext bulkLoadCtx = diskBTree.beginBulkLoad(1.0f);
+ try {
+ while (scanCursor.hasNext()) {
+ scanCursor.next();
+ diskBTree.bulkLoadAddTuple(scanCursor.getTuple(), bulkLoadCtx);
+ }
+ } finally {
+ scanCursor.close();
+ }
+ diskBTree.endBulkLoad(bulkLoadCtx);
+ resetMemBTree();
+ diskBTrees.addFirst(diskBTree);
+ }
+
+ private void resetMemBTree() throws HyracksDataException {
+ memFreePageManager.reset();
+ memBTree.create(memBTree.getFileId());
+ }
+
+ private BTree bulkLoadTargetBTree() throws HyracksDataException {
+ // Note that by using a flush target file name, we state that the new
+ // bulk loaded tree is "newer" than any other merged tree.
+ String fileName = fileNameManager.getFlushFileName();
+ return createDiskBTree(bulkLoadBTreeFactory, fileName, true);
+ }
+
+ private BTree createFlushTargetBTree() throws HyracksDataException {
+ String fileName = fileNameManager.getFlushFileName();
+ return createDiskBTree(diskBTreeFactory, fileName, true);
+ }
+
+ private BTree createMergeTargetBTree() throws HyracksDataException {
+ String fileName = fileNameManager.getMergeFileName();
+ return createDiskBTree(diskBTreeFactory, fileName, true);
+ }
+
+ private BTree createDiskBTree(BTreeFactory factory, String fileName, boolean createBTree) throws HyracksDataException {
+ // Register the new BTree file.
+ FileReference file = new FileReference(new File(fileName));
+ // TODO: Delete the file during cleanup.
+ diskBufferCache.createFile(file);
+ int diskBTreeFileId = diskFileMapProvider.lookupFileId(file);
+ // TODO: Close the file during cleanup.
+ diskBufferCache.openFile(diskBTreeFileId);
+ // Create new BTree instance.
+ BTree diskBTree = factory.createBTreeInstance(diskBTreeFileId);
+ if (createBTree) {
+ diskBTree.create(diskBTreeFileId);
+ }
+ // TODO: Close the BTree during cleanup.
+ diskBTree.open(diskBTreeFileId);
+ return diskBTree;
+ }
+
+ private void search(ITreeIndexCursor cursor, RangePredicate pred, LSMBTreeOpContext ctx, boolean includeMemBTree) throws HyracksDataException, TreeIndexException {
+ boolean waitForFlush = true;
+ do {
+ synchronized (this) {
+ if (!flushFlag) {
+ // The corresponding threadExit() is in LSMTreeRangeSearchCursor.close().
+ threadEnter();
+ waitForFlush = false;
+ }
+ }
+ } while (waitForFlush);
+
+ // TODO: Think about what happens with possibly concurrent merges.
+ LSMBTreeRangeSearchCursor lsmTreeCursor = (LSMBTreeRangeSearchCursor) cursor;
+ int numDiskBTrees = diskBTrees.size();
+ int numBTrees = (includeMemBTree) ? numDiskBTrees + 1 : numDiskBTrees;
+ ListIterator<BTree> diskBTreesIter = diskBTrees.listIterator();
+ LSMBTreeCursorInitialState initialState = new LSMBTreeCursorInitialState(numBTrees,
+ insertLeafFrameFactory, cmp, this);
+ lsmTreeCursor.open(initialState, pred);
+
+ int cursorIx;
+ if (includeMemBTree) {
+ // Open cursor of in-memory BTree at index 0.
+ ctx.memBTreeAccessor.search(lsmTreeCursor.getCursor(0), pred);
+ // Skip 0 because it is the in-memory BTree.
+ cursorIx = 1;
+ } else {
+ cursorIx = 0;
+ }
+
+ // Open cursors of on-disk BTrees.
+ ITreeIndexAccessor[] diskBTreeAccessors = new ITreeIndexAccessor[numDiskBTrees];
+ int diskBTreeIx = 0;
+ while(diskBTreesIter.hasNext()) {
+ BTree diskBTree = diskBTreesIter.next();
+ diskBTreeAccessors[diskBTreeIx] = diskBTree.createAccessor();
+ diskBTreeAccessors[diskBTreeIx].search(lsmTreeCursor.getCursor(cursorIx), pred);
+ cursorIx++;
+ diskBTreeIx++;
+ }
+ lsmTreeCursor.initPriorityQueue();
+ }
+
+ private void insert(ITupleReference tuple, LSMBTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
+ lsmPerformOp(tuple, ctx);
+ }
+
+ private void delete(ITupleReference tuple, LSMBTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
+ lsmPerformOp(tuple, ctx);
+ }
+
+ public void merge() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Merging LSM-BTree.");
+ }
+ LSMBTreeOpContext ctx = createOpContext();
+ ITreeIndexCursor cursor = new LSMBTreeRangeSearchCursor();
+ RangePredicate rangePred = new RangePredicate(true, null, null, true, true, null, null);
+ // Ordered scan, ignoring the in-memory BTree.
+ search(cursor, (RangePredicate) rangePred, ctx, false);
+
+ // Bulk load the tuples from all on-disk BTrees into the new BTree.
+ BTree mergedBTree = createMergeTargetBTree();
+ IIndexBulkLoadContext bulkLoadCtx = mergedBTree.beginBulkLoad(1.0f);
+ try {
+ while (cursor.hasNext()) {
+ cursor.next();
+ ITupleReference frameTuple = cursor.getTuple();
+ mergedBTree.bulkLoadAddTuple(frameTuple, bulkLoadCtx);
+ }
+ } finally {
+ cursor.close();
+ }
+ mergedBTree.endBulkLoad(bulkLoadCtx);
+ }
+
+ public class LSMTreeBulkLoadContext implements IIndexBulkLoadContext {
+ private final BTree btree;
+ private IIndexBulkLoadContext bulkLoadCtx;
+
+ public LSMTreeBulkLoadContext(BTree btree) {
+ this.btree = btree;
+ }
+
+ public void beginBulkLoad(float fillFactor) throws HyracksDataException, TreeIndexException {
+ bulkLoadCtx = btree.beginBulkLoad(fillFactor);
+ }
+
+ public BTree getBTree() {
+ return btree;
+ }
+
+ public IIndexBulkLoadContext getBulkLoadCtx() {
+ return bulkLoadCtx;
+ }
+ }
+
+ @Override
+ public IIndexBulkLoadContext beginBulkLoad(float fillFactor) throws TreeIndexException, HyracksDataException {
+ BTree diskBTree = bulkLoadTargetBTree();
+ LSMTreeBulkLoadContext bulkLoadCtx = new LSMTreeBulkLoadContext(diskBTree);
+ bulkLoadCtx.beginBulkLoad(fillFactor);
+ return bulkLoadCtx;
+ }
+
+ @Override
+ public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException {
+ LSMTreeBulkLoadContext bulkLoadCtx = (LSMTreeBulkLoadContext) ictx;
+ bulkLoadCtx.getBTree().bulkLoadAddTuple(tuple, bulkLoadCtx.getBulkLoadCtx());
+ }
+
+ @Override
+ public void endBulkLoad(IIndexBulkLoadContext ictx) throws HyracksDataException {
+ LSMTreeBulkLoadContext bulkLoadCtx = (LSMTreeBulkLoadContext) ictx;
+ bulkLoadCtx.getBTree().endBulkLoad(bulkLoadCtx.getBulkLoadCtx());
+ diskBTrees.addFirst(bulkLoadCtx.getBTree());
+ }
+
+ @Override
+ public ITreeIndexFrameFactory getLeafFrameFactory() {
+ return memBTree.getLeafFrameFactory();
+ }
+
+ @Override
+ public ITreeIndexFrameFactory getInteriorFrameFactory() {
+ return memBTree.getInteriorFrameFactory();
+ }
+
+ @Override
+ public IFreePageManager getFreePageManager() {
+ return memBTree.getFreePageManager();
+ }
+
+ @Override
+ public int getFieldCount() {
+ return memBTree.getFieldCount();
+ }
+
+ @Override
+ public int getRootPageId() {
+ return memBTree.getRootPageId();
+ }
+
+ @Override
+ public IndexType getIndexType() {
+ return memBTree.getIndexType();
+ }
+
+ public MultiComparator getMultiComparator() {
+ return cmp;
+ }
+
+ public LSMBTreeOpContext createOpContext() {
+ return new LSMBTreeOpContext(memBTree, insertLeafFrameFactory, deleteLeafFrameFactory);
+ }
+
+ @Override
+ public ITreeIndexAccessor createAccessor() {
+ return new LSMTreeIndexAccessor(this);
+ }
+
+ private class LSMTreeIndexAccessor implements ITreeIndexAccessor {
+ private LSMBTree lsmTree;
+ private LSMBTreeOpContext ctx;
+
+ public LSMTreeIndexAccessor(LSMBTree lsmTree) {
+ this.lsmTree = lsmTree;
+ this.ctx = lsmTree.createOpContext();
+ }
+
+ @Override
+ public void insert(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
+ ctx.reset(IndexOp.INSERT);
+ lsmTree.insert(tuple, ctx);
+ }
+
+ @Override
+ public void update(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
+ // Update is the same as insert.
+ ctx.reset(IndexOp.INSERT);
+ insert(tuple);
+ }
+
+ @Override
+ public void delete(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
+ ctx.reset(IndexOp.DELETE);
+ lsmTree.delete(tuple, ctx);
+ }
+
+ @Override
+ public ITreeIndexCursor createSearchCursor() {
+ return new LSMBTreeRangeSearchCursor();
+ }
+
+ @Override
+ public void search(ITreeIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException,
+ TreeIndexException {
+ ctx.reset(IndexOp.SEARCH);
+ lsmTree.search(cursor, (RangePredicate) searchPred, ctx, true);
+ }
+
+ @Override
+ public ITreeIndexCursor createDiskOrderScanCursor() {
+ // Disk-order scan doesn't make sense for the LSMBTree because it cannot correctly resolve deleted tuples.
+ throw new UnsupportedOperationException("DiskOrderScan not supported by LSMTree.");
+ }
+
+ @Override
+ public void diskOrderScan(ITreeIndexCursor cursor) throws HyracksDataException {
+ // Disk-order scan doesn't make sense for the LSMBTree because it cannot correctly resolve deleted tuples.
+ throw new UnsupportedOperationException("DiskOrderScan not supported by LSMTree.");
+ }
+ }
+}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/LSMBTreeCursorInitialState.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/LSMBTreeCursorInitialState.java
new file mode 100644
index 0000000..672ff4a
--- /dev/null
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/LSMBTreeCursorInitialState.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.impls;
+
+import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+
+public class LSMBTreeCursorInitialState implements ICursorInitialState {
+
+ private int numBTrees;
+ private ITreeIndexFrameFactory leafFrameFactory;
+ private MultiComparator cmp;
+ private LSMBTree lsm;
+
+ public LSMBTreeCursorInitialState(int numBTrees, ITreeIndexFrameFactory leafFrameFactory, MultiComparator cmp, LSMBTree lsm) {
+ this.numBTrees = numBTrees;
+ this.leafFrameFactory = leafFrameFactory;
+ this.cmp = cmp;
+ this.lsm = lsm;
+ }
+
+ public int getNumBTrees() {
+ return numBTrees;
+ }
+
+ public ITreeIndexFrameFactory getLeafFrameFactory() {
+ return leafFrameFactory;
+ }
+
+ public MultiComparator getCmp() {
+ return cmp;
+ }
+
+ @Override
+ public ICachedPage getPage() {
+ return null;
+ }
+
+ @Override
+ public void setPage(ICachedPage page) {
+ }
+
+ public LSMBTree getLsm() {
+ return lsm;
+ }
+
+}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/LSMBTreeFileNameManager.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/LSMBTreeFileNameManager.java
new file mode 100644
index 0000000..8e0cbe2
--- /dev/null
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/LSMBTreeFileNameManager.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.impls;
+
+import java.text.Format;
+import java.text.SimpleDateFormat;
+import java.util.Comparator;
+import java.util.Date;
+
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFileNameManager;
+
+public class LSMBTreeFileNameManager implements ILSMFileNameManager {
+
+ private final String baseDir;
+ private final Format formatter = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss_SSS");
+ private final Comparator<String> cmp = new FileNameComparator();
+
+ public LSMBTreeFileNameManager(String baseDir) {
+ if (!baseDir.endsWith(System.getProperty("file.separator"))) {
+ baseDir += System.getProperty("file.separator");
+ }
+ this.baseDir = baseDir;
+ }
+
+ @Override
+ public String getFlushFileName() {
+ Date date = new Date();
+ // "Z" prefix to indicate flush. Relies on "Z" sorting higher than "A".
+ return baseDir + "Z" + formatter.format(date);
+ }
+
+ @Override
+ public String getMergeFileName() {
+ Date date = new Date();
+ // "A" prefix to indicate merge. Relies on "A" sorting lower than "Z".
+ return baseDir + "A" + formatter.format(date);
+ }
+
+ @Override
+ public Comparator<String> getFileNameComparator() {
+ return cmp;
+ }
+
+ /**
+ * Sorts strings in reverse lexicographical order. The way we construct the
+ * file names above guarantees that:
+ *
+ * 1. Flushed files ("Z" prefix) sort lower than merged files ("A" prefix)
+ *
+ * 2. Flushed files are sorted from newest to oldest (based on the timestamp
+ * string)
+ *
+ */
+ private class FileNameComparator implements Comparator<String> {
+ @Override
+ public int compare(String a, String b) {
+ // Consciously ignoring locale.
+ return -a.compareTo(b);
+ }
+ }
+
+ @Override
+ public String getBaseDir() {
+ return baseDir;
+ }
+}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/LSMBTreeOpContext.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/LSMBTreeOpContext.java
new file mode 100644
index 0000000..daf27ad
--- /dev/null
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/LSMBTreeOpContext.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.impls;
+
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+
+public final class LSMBTreeOpContext implements IIndexOpContext {
+
+ public ITreeIndexFrameFactory insertLeafFrameFactory;
+ public ITreeIndexFrameFactory deleteLeafFrameFactory;
+ public IBTreeLeafFrame insertLeafFrame;
+ public IBTreeLeafFrame deleteLeafFrame;
+ public final BTree memBTree;
+ public BTree.BTreeAccessor memBTreeAccessor;
+ public BTreeOpContext memBTreeOpCtx;
+ public IndexOp op;
+
+ public LSMBTreeOpContext(BTree memBTree, ITreeIndexFrameFactory insertLeafFrameFactory,
+ ITreeIndexFrameFactory deleteLeafFrameFactory) {
+ this.memBTree = memBTree;
+ this.insertLeafFrameFactory = insertLeafFrameFactory;
+ this.deleteLeafFrameFactory = deleteLeafFrameFactory;
+ this.insertLeafFrame = (IBTreeLeafFrame) insertLeafFrameFactory.createFrame();
+ this.deleteLeafFrame = (IBTreeLeafFrame) deleteLeafFrameFactory.createFrame();
+ if (insertLeafFrame != null) {
+ insertLeafFrame.setMultiComparator(memBTree.getMultiComparator());
+ }
+ if (deleteLeafFrame != null) {
+ deleteLeafFrame.setMultiComparator(memBTree.getMultiComparator());
+ }
+ }
+
+ @Override
+ public void reset(IndexOp newOp) {
+ this.op = newOp;
+ switch (newOp) {
+ case SEARCH:
+ setMemBTreeAccessor();
+ break;
+
+ case DISKORDERSCAN:
+ case UPDATE:
+ // Attention: It is important to leave the leafFrame and
+ // leafFrameFactory of the memBTree as is when doing an update.
+ // Update will only be set if a previous attempt to delete or
+ // insert failed, so we must preserve the semantics of the
+ // previously requested operation.
+ return;
+
+ case INSERT:
+ setInsertMode();
+ break;
+
+ case DELETE:
+ setDeleteMode();
+ break;
+ }
+ }
+
+ private void setMemBTreeAccessor() {
+ if (memBTreeAccessor == null) {
+ memBTreeAccessor = (BTree.BTreeAccessor) memBTree.createAccessor();
+ memBTreeOpCtx = memBTreeAccessor.getOpContext();
+ }
+ }
+
+ public void setInsertMode() {
+ setMemBTreeAccessor();
+ memBTreeOpCtx.leafFrame = insertLeafFrame;
+ memBTreeOpCtx.leafFrameFactory = insertLeafFrameFactory;
+ }
+
+ public void setDeleteMode() {
+ setMemBTreeAccessor();
+ memBTreeOpCtx.leafFrame = deleteLeafFrame;
+ memBTreeOpCtx.leafFrameFactory = deleteLeafFrameFactory;
+ }
+
+ @Override
+ public void reset() {
+ }
+
+ public IndexOp getIndexOp() {
+ return op;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/LSMBTreeRangeSearchCursor.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/LSMBTreeRangeSearchCursor.java
new file mode 100644
index 0000000..cf2a9e3
--- /dev/null
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/impls/LSMBTreeRangeSearchCursor.java
@@ -0,0 +1,255 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.impls;
+
+import java.util.Comparator;
+import java.util.PriorityQueue;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.tuples.LSMBTreeTupleReference;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+
+public class LSMBTreeRangeSearchCursor implements ITreeIndexCursor {
+ private BTreeRangeSearchCursor[] rangeCursors;
+ private PriorityQueue<PriorityQueueElement> outputPriorityQueue;
+ private MultiComparator cmp;
+ private PriorityQueueComparator pqCmp;
+ private PriorityQueueElement outputElement;
+ private PriorityQueueElement reusedElement;
+ private boolean needPush;
+ private LSMBTree lsmTree;
+
+ public LSMBTreeRangeSearchCursor() {
+ outputElement = null;
+ needPush = false;
+ }
+
+ public void initPriorityQueue()
+ throws HyracksDataException {
+ outputPriorityQueue = new PriorityQueue<PriorityQueueElement>(rangeCursors.length, pqCmp);
+ for (int i = 0; i < rangeCursors.length; i++) {
+ PriorityQueueElement element;
+ if (rangeCursors[i].hasNext()) {
+ rangeCursors[i].next();
+ element = new PriorityQueueElement(rangeCursors[i].getTuple(), i);
+ outputPriorityQueue.offer(element);
+ }
+ }
+ checkPriorityQueue();
+ }
+
+ public BTreeRangeSearchCursor getCursor(int cursorIndex) {
+ return rangeCursors[cursorIndex];
+ }
+
+ @Override
+ public void reset() {
+ outputElement = null;
+ needPush = false;
+ }
+
+ @Override
+ public boolean hasNext() throws HyracksDataException {
+ checkPriorityQueue();
+ return !outputPriorityQueue.isEmpty();
+ }
+
+ @Override
+ public void next() throws HyracksDataException {
+ outputElement = outputPriorityQueue.poll();
+ needPush = true;
+ if (outputElement == null) {
+ throw new HyracksDataException("The outputPriorityQueue is empty");
+ }
+ }
+
+ @Override
+ public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ LSMBTreeCursorInitialState lsmInitialState = (LSMBTreeCursorInitialState) initialState;
+ lsmTree = lsmInitialState.getLsm();
+ cmp = lsmInitialState.getCmp();
+ int numBTrees = lsmInitialState.getNumBTrees();
+ rangeCursors = new BTreeRangeSearchCursor[numBTrees];
+ for (int i = 0; i < numBTrees; i++) {
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) lsmInitialState.getLeafFrameFactory().createFrame();
+ rangeCursors[i] = new BTreeRangeSearchCursor(leafFrame, false);
+ }
+ setPriorityQueueComparator();
+ }
+
+ private void setPriorityQueueComparator() {
+ if (pqCmp == null || cmp != pqCmp.getMultiComparator()) {
+ pqCmp = new PriorityQueueComparator(cmp);
+ }
+ }
+
+ @Override
+ public ICachedPage getPage() {
+ // do nothing
+ return null;
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ outputPriorityQueue.clear();
+ for (int i = 0; i < rangeCursors.length; i++) {
+ rangeCursors[i].close();
+ }
+ rangeCursors = null;
+ try {
+ lsmTree.threadExit();
+ } catch (TreeIndexException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void setBufferCache(IBufferCache bufferCache) {
+ // do nothing
+ }
+
+ @Override
+ public void setFileId(int fileId) {
+ // do nothing
+ }
+
+ @Override
+ public ITupleReference getTuple() {
+ return (ITupleReference) outputElement.getTuple();
+ }
+
+ private void pushIntoPriorityQueue(int cursorIndex) throws HyracksDataException {
+ if (rangeCursors[cursorIndex].hasNext()) {
+ rangeCursors[cursorIndex].next();
+ reusedElement.reset(rangeCursors[cursorIndex].getTuple(), cursorIndex);
+ outputPriorityQueue.offer(reusedElement);
+ }
+ }
+
+ private void checkPriorityQueue() throws HyracksDataException {
+ while (!outputPriorityQueue.isEmpty() || needPush == true) {
+ if (!outputPriorityQueue.isEmpty()) {
+ PriorityQueueElement checkElement = outputPriorityQueue.peek();
+ // If there is no previous tuple or the previous tuple can be
+ // ignored
+ if (outputElement == null) {
+ // Test the tuple is a delete tuple or not
+ if (((LSMBTreeTupleReference) checkElement.getTuple()).isAntimatter() == true) {
+ // If the tuple is a delete tuple then pop it and mark
+ // it "needPush"
+ // Cannot push at this time because the tuple may be
+ // modified if "hasNext" is called
+ outputElement = outputPriorityQueue.poll();
+ needPush = true;
+ } else {
+ break;
+ }
+ } else {
+ // Compare the previous tuple and the head tuple in the PQ
+ if (cmp.compare(outputElement.getTuple(), checkElement.getTuple()) == 0) {
+ // If the previous tuple and the head tuple are
+ // identical
+ // then pop the head tuple and push the next tuple from
+ // the tree of head tuple
+
+ // the head element of PQ is useless now
+ reusedElement = outputPriorityQueue.poll();
+ // int treeNum = reusedElement.getTreeNum();
+ pushIntoPriorityQueue(reusedElement.getCursorIndex());
+ } else {
+ // If the previous tuple and the head tuple are
+ // different
+ // the info of previous tuple is useless
+ if (needPush == true) {
+ reusedElement = outputElement;
+ pushIntoPriorityQueue(outputElement.getCursorIndex());
+ needPush = false;
+ }
+ outputElement = null;
+ }
+ }
+ } else {
+ // the priority queue is empty and needPush
+ reusedElement = outputElement;
+ pushIntoPriorityQueue(outputElement.getCursorIndex());
+ needPush = false;
+ outputElement = null;
+ }
+ }
+ }
+
+ @Override
+ public boolean exclusiveLatchNodes() {
+ return false;
+ }
+
+ public class PriorityQueueComparator implements Comparator<PriorityQueueElement> {
+
+ private final MultiComparator cmp;
+
+ public PriorityQueueComparator(MultiComparator cmp) {
+ this.cmp = cmp;
+ }
+
+ @Override
+ public int compare(PriorityQueueElement elementA, PriorityQueueElement elementB) {
+ int result = cmp.compare(elementA.getTuple(), elementB.getTuple());
+ if (result != 0) {
+ return result;
+ }
+ if (elementA.getCursorIndex() > elementB.getCursorIndex()) {
+ return 1;
+ } else {
+ return -1;
+ }
+ }
+
+ public MultiComparator getMultiComparator() {
+ return cmp;
+ }
+ }
+
+ public class PriorityQueueElement {
+ private ITupleReference tuple;
+ private int cursorIndex;
+
+ public PriorityQueueElement(ITupleReference tuple, int cursorIndex) {
+ reset(tuple, cursorIndex);
+ }
+
+ public ITupleReference getTuple() {
+ return tuple;
+ }
+
+ public int getCursorIndex() {
+ return cursorIndex;
+ }
+
+ public void reset(ITupleReference tuple, int cursorIndex) {
+ this.tuple = tuple;
+ this.cursorIndex = cursorIndex;
+ }
+ }
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/PageAllocationException.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/ILSMBTreeTupleReference.java
similarity index 65%
rename from hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/PageAllocationException.java
rename to hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/ILSMBTreeTupleReference.java
index e6eec66..a4d2f92 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/PageAllocationException.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/ILSMBTreeTupleReference.java
@@ -13,17 +13,10 @@
* limitations under the License.
*/
-package edu.uci.ics.hyracks.storage.am.common.api;
+package edu.uci.ics.hyracks.storage.am.lsm.tuples;
-public class PageAllocationException extends Exception {
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
- private static final long serialVersionUID = 1L;
-
- public PageAllocationException(Throwable cause) {
- super(cause);
- }
-
- public PageAllocationException(String message) {
- super(message);
- }
+public interface ILSMBTreeTupleReference extends ITreeIndexTupleReference {
+ public boolean isAntimatter();
}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/LSMBTreeCopyTupleWriter.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/LSMBTreeCopyTupleWriter.java
new file mode 100644
index 0000000..db75995
--- /dev/null
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/LSMBTreeCopyTupleWriter.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.tuples;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+public class LSMBTreeCopyTupleWriter extends LSMBTreeTupleWriter {
+ public LSMBTreeCopyTupleWriter(ITypeTraits[] typeTraits, int numKeyFields){
+ // Third parameter is never used locally, just give false.
+ super(typeTraits, numKeyFields, false);
+ }
+
+ @Override
+ public int writeTuple(ITupleReference tuple, byte[] targetBuf, int targetOff) {
+ int tupleSize = bytesRequired(tuple);
+ byte[] buf = tuple.getFieldData(0);
+ int tupleStartOff = ((LSMBTreeTupleReference)tuple).getTupleStart();
+ System.arraycopy(buf, tupleStartOff, targetBuf, targetOff, tupleSize);
+ return tupleSize;
+ }
+}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/LSMBTreeCopyTupleWriterFactory.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/LSMBTreeCopyTupleWriterFactory.java
new file mode 100644
index 0000000..78de794
--- /dev/null
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/LSMBTreeCopyTupleWriterFactory.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.tuples;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+
+public class LSMBTreeCopyTupleWriterFactory extends TypeAwareTupleWriterFactory {
+ private static final long serialVersionUID = 1L;
+ private final ITypeTraits[] typeTraits;
+ private final int numKeyFields;
+
+ public LSMBTreeCopyTupleWriterFactory(ITypeTraits[] typeTraits, int numKeyFields) {
+ super(typeTraits);
+ this.typeTraits = typeTraits;
+ this.numKeyFields = numKeyFields;
+ }
+
+ @Override
+ public ITreeIndexTupleWriter createTupleWriter() {
+ return new LSMBTreeCopyTupleWriter(typeTraits, numKeyFields);
+ }
+}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/LSMBTreeTupleReference.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/LSMBTreeTupleReference.java
new file mode 100644
index 0000000..d5d44bd
--- /dev/null
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/LSMBTreeTupleReference.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.tuples;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleReference;
+
+public class LSMBTreeTupleReference extends TypeAwareTupleReference implements ILSMBTreeTupleReference {
+
+ // Indicates whether the last call to setFieldCount() was initiated by
+ // by the outside or whether it was called internally to set up an
+ // antimatter tuple.
+ private boolean resetFieldCount = false;
+ private final int numKeyFields;
+
+ public LSMBTreeTupleReference(ITypeTraits[] typeTraits, int numKeyFields) {
+ super(typeTraits);
+ this.numKeyFields = numKeyFields;
+ }
+
+ public void setFieldCount(int fieldCount) {
+ super.setFieldCount(fieldCount);
+ // Don't change the fieldCount in reset calls.
+ resetFieldCount = false;
+ }
+
+ @Override
+ public void setFieldCount(int fieldStartIndex, int fieldCount) {
+ super.setFieldCount(fieldStartIndex, fieldCount);
+ // Don't change the fieldCount in reset calls.
+ resetFieldCount = false;
+ }
+
+ @Override
+ public void resetByTupleOffset(ByteBuffer buf, int tupleStartOff) {
+ this.buf = buf;
+ this.tupleStartOff = tupleStartOff;
+ if (numKeyFields != typeTraits.length) {
+ if (isAntimatter()) {
+ setFieldCount(numKeyFields);
+ // Reset the original field count for matter tuples.
+ resetFieldCount = true;
+ } else {
+ if (resetFieldCount) {
+ setFieldCount(typeTraits.length);
+ }
+ }
+ }
+ super.resetByTupleOffset(buf, tupleStartOff);
+ }
+
+ @Override
+ public void resetByTupleIndex(ITreeIndexFrame frame, int tupleIndex) {
+ resetByTupleOffset(frame.getBuffer(), frame.getTupleOffset(tupleIndex));
+ }
+
+ @Override
+ protected int getNullFlagsBytes() {
+ // +1.0 is for matter/antimatter bit.
+ return (int) Math.ceil((fieldCount + 1.0) / 8.0);
+ }
+
+ @Override
+ public boolean isAntimatter() {
+ // Check if the leftmost bit is 0 or 1.
+ final byte mask = (byte) (1 << 7);
+ if ((buf.array()[tupleStartOff] & mask) != 0) {
+ return true;
+ }
+ return false;
+ }
+
+ public int getTupleStart() {
+ return tupleStartOff;
+ }
+}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/LSMBTreeTupleWriter.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/LSMBTreeTupleWriter.java
new file mode 100644
index 0000000..ea58282
--- /dev/null
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/LSMBTreeTupleWriter.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.tuples;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
+
+public class LSMBTreeTupleWriter extends TypeAwareTupleWriter {
+ private final boolean isDelete;
+ private final int numKeyFields;
+
+ public LSMBTreeTupleWriter(ITypeTraits[] typeTraits, int numKeyFields, boolean isDelete) {
+ super(typeTraits);
+ this.numKeyFields = numKeyFields;
+ this.isDelete = isDelete;
+ }
+
+ @Override
+ public ITreeIndexTupleReference createTupleReference() {
+ return new LSMBTreeTupleReference(typeTraits, numKeyFields);
+ }
+
+ @Override
+ protected int getNullFlagsBytes(int numFields) {
+ // +1.0 is for matter/antimatter bit.
+ return (int) Math.ceil(((double) numFields + 1.0) / 8.0);
+ }
+
+ @Override
+ protected int getNullFlagsBytes(ITupleReference tuple) {
+ // +1.0 is for matter/antimatter bit.
+ return (int) Math.ceil(((double) tuple.getFieldCount() + 1.0) / 8.0);
+ }
+
+ @Override
+ public int writeTuple(ITupleReference tuple, byte[] targetBuf, int targetOff) {
+ int bytesWritten = -1;
+ if (isDelete) {
+ bytesWritten = super.writeTupleFields(tuple, 0, numKeyFields, targetBuf, targetOff);
+ setAntimatterBit(targetBuf, targetOff);
+ } else {
+ bytesWritten = super.writeTuple(tuple, targetBuf, targetOff);
+ }
+ return bytesWritten;
+ }
+
+ private void setAntimatterBit(byte[] targetBuf, int targetOff) {
+ // Set leftmost bit to 1.
+ targetBuf[targetOff] = (byte) (targetBuf[targetOff] | (1 << 7));
+ }
+}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/LSMBTreeTupleWriterFactory.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/LSMBTreeTupleWriterFactory.java
new file mode 100644
index 0000000..cd016ea
--- /dev/null
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/tuples/LSMBTreeTupleWriterFactory.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.tuples;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+
+public class LSMBTreeTupleWriterFactory extends TypeAwareTupleWriterFactory {
+
+ private static final long serialVersionUID = 1L;
+ private final ITypeTraits[] typeTraits;
+ private final int numKeyFields;
+ private final boolean isDelete;
+
+ public LSMBTreeTupleWriterFactory(ITypeTraits[] typeTraits, int numKeyFields, boolean isDelete) {
+ super(typeTraits);
+ this.typeTraits = typeTraits;
+ this.numKeyFields = numKeyFields;
+ this.isDelete = isDelete;
+ }
+
+ @Override
+ public ITreeIndexTupleWriter createTupleWriter() {
+ return new LSMBTreeTupleWriter(typeTraits, numKeyFields, isDelete);
+ }
+}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/util/LSMBTreeUtils.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/util/LSMBTreeUtils.java
new file mode 100644
index 0000000..1262a14
--- /dev/null
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/util/LSMBTreeUtils.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.util;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFileNameManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsm.impls.BTreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.impls.LSMBTreeFileNameManager;
+import edu.uci.ics.hyracks.storage.am.lsm.impls.LSMBTree;
+import edu.uci.ics.hyracks.storage.am.lsm.tuples.LSMBTreeCopyTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.tuples.LSMBTreeTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+
+public class LSMBTreeUtils {
+ public static LSMBTree createLSMTree(InMemoryBufferCache memBufferCache, InMemoryFreePageManager memFreePageManager,
+ String onDiskDir, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider,
+ ITypeTraits[] typeTraits, IBinaryComparator[] cmps) {
+ MultiComparator cmp = new MultiComparator(cmps);
+ LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
+ cmps.length, false);
+ LSMBTreeTupleWriterFactory deleteTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
+ cmps.length, true);
+ LSMBTreeCopyTupleWriterFactory copyTupleWriterFactory = new LSMBTreeCopyTupleWriterFactory(typeTraits, cmps.length);
+ ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
+ ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory);
+ ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+ LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
+ metaFrameFactory);
+ BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, cmp,
+ typeTraits.length, interiorFrameFactory, copyTupleLeafFrameFactory);
+ BTreeFactory bulkLoadBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, cmp,
+ typeTraits.length, interiorFrameFactory, insertLeafFrameFactory);
+ ILSMFileNameManager fileNameManager = new LSMBTreeFileNameManager(onDiskDir);
+ LSMBTree lsmTree = new LSMBTree(memBufferCache, memFreePageManager, interiorFrameFactory, insertLeafFrameFactory,
+ deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, diskFileMapProvider,
+ typeTraits.length, cmp);
+ return lsmTree;
+ }
+}
diff --git a/hyracks-storage-am-lsm-common/pom.xml b/hyracks-storage-am-lsm-common/pom.xml
new file mode 100644
index 0000000..836b4a2
--- /dev/null
+++ b/hyracks-storage-am-lsm-common/pom.xml
@@ -0,0 +1,42 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-common</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-common</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <type>jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFileNameManager.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFileNameManager.java
new file mode 100644
index 0000000..c48c48e
--- /dev/null
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFileNameManager.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.common.api;
+
+import java.util.Comparator;
+
+/**
+ * Provides file names for LSM on-disk components.
+ *
+ * There are separate methods to get file names for merge and flush, because we
+ * need to guarantee the correct order of on-disk components (i.e., the
+ * components produced by flush are always newer than those produced by a
+ * merge).
+ *
+ *
+ */
+public interface ILSMFileNameManager {
+ public String getFlushFileName();
+
+ public String getMergeFileName();
+
+ public String getBaseDir();
+
+ public Comparator<String> getFileNameComparator();
+}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMTree.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMTree.java
new file mode 100644
index 0000000..9775484
--- /dev/null
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMTree.java
@@ -0,0 +1,9 @@
+package edu.uci.ics.hyracks.storage.am.lsm.common.api;
+
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+
+public interface ILSMTree extends ITreeIndex {
+ public void merge() throws Exception;
+
+ public void flush() throws Exception;
+}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryBufferCache.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryBufferCache.java
new file mode 100644
index 0000000..8fcb4a0
--- /dev/null
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryBufferCache.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.common.freepage;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCacheInternal;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPageInternal;
+import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+
+public class InMemoryBufferCache implements IBufferCacheInternal {
+ protected final ICacheMemoryAllocator allocator;
+ protected final int pageSize;
+ protected final CachedPage[] pages;
+ protected final List<CachedPage> overflowPages = new ArrayList<CachedPage>();
+
+ public InMemoryBufferCache(ICacheMemoryAllocator allocator, int pageSize, int numPages){
+ this.allocator = allocator;
+ this.pageSize = pageSize;
+ ByteBuffer[] buffers = allocator.allocate(pageSize, numPages);
+ pages = new CachedPage[buffers.length];
+ for (int i = 0; i < buffers.length; ++i) {
+ pages[i] = new CachedPage(i, buffers[i]);
+ }
+ }
+
+ @Override
+ public ICachedPage pin(long dpid, boolean newPage) {
+ int pageId = BufferedFileHandle.getPageId(dpid);
+ if (pageId < pages.length) {
+ // Common case: Return regular page.
+ return pages[pageId];
+ } else {
+ // Rare case: Return overflow page, possibly expanding overflow array.
+ synchronized(overflowPages) {
+ int numNewPages = pageId - pages.length - overflowPages.size() + 1;
+ if (numNewPages > 0) {
+ ByteBuffer[] buffers = allocator.allocate(pageSize, numNewPages);
+ for (int i = 0; i < numNewPages; i++) {
+ CachedPage overflowPage = new CachedPage(pages.length + overflowPages.size(), buffers[i]);
+ overflowPages.add(overflowPage);
+ }
+ }
+ return overflowPages.get(pageId - pages.length);
+ }
+ }
+ }
+
+ @Override
+ public ICachedPage tryPin(long dpid) throws HyracksDataException {
+ return pin(dpid, false);
+ }
+
+ @Override
+ public int getPageSize() {
+ return pageSize;
+ }
+
+ @Override
+ public int getNumPages() {
+ return pages.length;
+ }
+
+ @Override
+ public ICachedPageInternal getPage(int cpid) {
+ return pages[cpid];
+ }
+
+ public int getNumOverflowPages() {
+ return overflowPages.size();
+ }
+
+ @Override
+ public void createFile(FileReference fileRef) throws HyracksDataException {
+ // Do nothing.
+ }
+
+ @Override
+ public void openFile(int fileId) throws HyracksDataException {
+ // Do nothing.
+ }
+
+ @Override
+ public void closeFile(int fileId) throws HyracksDataException {
+ // Do nothing.
+ }
+
+ @Override
+ public void deleteFile(int fileId) throws HyracksDataException {
+ // Do nothing.
+ }
+
+ @Override
+ public void unpin(ICachedPage page) throws HyracksDataException {
+ // Do Nothing.
+ }
+
+ @Override
+ public void close() {
+ // Do nothing.
+ }
+
+ public class CachedPage implements ICachedPageInternal {
+ private final int cpid;
+ private final ByteBuffer buffer;
+ private final ReadWriteLock latch;
+
+ public CachedPage(int cpid, ByteBuffer buffer) {
+ this.cpid = cpid;
+ this.buffer = buffer;
+ latch = new ReentrantReadWriteLock(true);
+ }
+
+ @Override
+ public ByteBuffer getBuffer() {
+ return buffer;
+ }
+
+ @Override
+ public Object getReplacementStrategyObject() {
+ // Do nothing.
+ return null;
+ }
+
+ @Override
+ public boolean pinIfGoodVictim() {
+ // Do nothing.
+ return false;
+ }
+
+ @Override
+ public int getCachedPageId() {
+ return cpid;
+ }
+
+ @Override
+ public void acquireReadLatch() {
+ latch.readLock().lock();
+ }
+
+ @Override
+ public void acquireWriteLatch() {
+ latch.writeLock().lock();
+ }
+
+ @Override
+ public void releaseReadLatch() {
+ latch.readLock().unlock();
+ }
+
+ @Override
+ public void releaseWriteLatch() {
+ latch.writeLock().unlock();
+ }
+ }
+}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryFreePageManager.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryFreePageManager.java
new file mode 100644
index 0000000..304aa43
--- /dev/null
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryFreePageManager.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.common.freepage;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+
+public class InMemoryFreePageManager implements IFreePageManager {
+ protected final int capacity;
+ protected final AtomicInteger currentPageId = new AtomicInteger();
+ protected final ITreeIndexMetaDataFrameFactory metaDataFrameFactory;
+
+ public InMemoryFreePageManager(int capacity, ITreeIndexMetaDataFrameFactory metaDataFrameFactory) {
+ // We start the currentPageId from 1, because the BTree uses
+ // the first page as metadata page, and the second page as root page.
+ // (when returning free pages we first increment, then get)
+ currentPageId.set(1);
+ this.capacity = capacity;
+ this.metaDataFrameFactory = metaDataFrameFactory;
+ }
+
+ @Override
+ public int getFreePage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
+ // The very call returns page id 2 because the BTree uses
+ // the first page as metadata page, and the second page as root page.
+ return currentPageId.incrementAndGet();
+ }
+
+ @Override
+ public int getMaxPage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
+ return currentPageId.get();
+ }
+
+ @Override
+ public void init(ITreeIndexMetaDataFrame metaFrame, int currentMaxPage) throws HyracksDataException {
+ currentPageId.set(1);
+ }
+
+ @Override
+ public ITreeIndexMetaDataFrameFactory getMetaDataFrameFactory() {
+ return metaDataFrameFactory;
+ }
+
+ public int getCapacity() {
+ return capacity - 2;
+ }
+
+ public void reset() {
+ currentPageId.set(1);
+ }
+
+ public boolean isFull() {
+ return currentPageId.get() >= capacity;
+ }
+
+ @Override
+ public void addFreePage(ITreeIndexMetaDataFrame metaFrame, int freePage) throws HyracksDataException {
+ }
+
+ @Override
+ public byte getMetaPageLevelIndicator() {
+ return 0;
+ }
+
+ @Override
+ public byte getFreePageLevelIndicator() {
+ return 0;
+ }
+
+ @Override
+ public boolean isMetaPage(ITreeIndexMetaDataFrame metaFrame) {
+ return false;
+ }
+
+ @Override
+ public boolean isFreePage(ITreeIndexMetaDataFrame metaFrame) {
+ return false;
+ }
+}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeBulkLoadContext.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeBulkLoadContext.java
new file mode 100644
index 0000000..727c211
--- /dev/null
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeBulkLoadContext.java
@@ -0,0 +1,27 @@
+package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+
+public class LSMTreeBulkLoadContext implements IIndexBulkLoadContext {
+ private final ITreeIndex tree;
+ private IIndexBulkLoadContext bulkLoadCtx;
+
+ public LSMTreeBulkLoadContext(ITreeIndex tree) {
+ this.tree = tree;
+ }
+
+ public void beginBulkLoad(float fillFactor) throws HyracksDataException, TreeIndexException {
+ bulkLoadCtx = tree.beginBulkLoad(fillFactor);
+ }
+
+ public ITreeIndex getTree() {
+ return tree;
+ }
+
+ public IIndexBulkLoadContext getBulkLoadCtx() {
+ return bulkLoadCtx;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileNameManager.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileNameManager.java
new file mode 100644
index 0000000..628fbf3
--- /dev/null
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileNameManager.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
+
+import java.text.Format;
+import java.text.SimpleDateFormat;
+import java.util.Comparator;
+import java.util.Date;
+
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFileNameManager;
+
+public class LSMTreeFileNameManager implements ILSMFileNameManager {
+
+ private final String baseDir;
+ private final Format formatter = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss_SSS");
+ private final Comparator<String> cmp = new FileNameComparator();
+
+ public LSMTreeFileNameManager(String baseDir) {
+ if (!baseDir.endsWith(System.getProperty("file.separator"))) {
+ baseDir += System.getProperty("file.separator");
+ }
+ this.baseDir = baseDir;
+ }
+
+ @Override
+ public String getFlushFileName() {
+ Date date = new Date();
+ // "Z" prefix to indicate flush. Relies on "Z" sorting higher than "A".
+ return baseDir + "Z" + formatter.format(date);
+ }
+
+ @Override
+ public String getMergeFileName() {
+ Date date = new Date();
+ // "A" prefix to indicate merge. Relies on "A" sorting lower than "Z".
+ return baseDir + "A" + formatter.format(date);
+ }
+
+ @Override
+ public Comparator<String> getFileNameComparator() {
+ return cmp;
+ }
+
+ /**
+ * Sorts strings in reverse lexicographical order. The way we construct the
+ * file names above guarantees that:
+ *
+ * 1. Flushed files ("Z" prefix) sort lower than merged files ("A" prefix)
+ *
+ * 2. Flushed files are sorted from newest to oldest (based on the timestamp
+ * string)
+ *
+ */
+ private class FileNameComparator implements Comparator<String> {
+ @Override
+ public int compare(String a, String b) {
+ // Consciously ignoring locale.
+ return -a.compareTo(b);
+ }
+ }
+
+ @Override
+ public String getBaseDir() {
+ return baseDir;
+ }
+}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java
new file mode 100644
index 0000000..8eec377
--- /dev/null
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java
@@ -0,0 +1,33 @@
+package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
+
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+public abstract class TreeFactory {
+
+ protected IBufferCache bufferCache;
+ protected int fieldCount;
+ protected MultiComparator cmp;
+ protected ITreeIndexFrameFactory interiorFrameFactory;
+ protected ITreeIndexFrameFactory leafFrameFactory;
+ protected LinkedListFreePageManagerFactory freePageManagerFactory;
+
+ public TreeFactory(IBufferCache bufferCache, LinkedListFreePageManagerFactory freePageManagerFactory, MultiComparator cmp,
+ int fieldCount, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
+ this.bufferCache = bufferCache;
+ this.fieldCount = fieldCount;
+ this.cmp = cmp;
+ this.interiorFrameFactory = interiorFrameFactory;
+ this.leafFrameFactory = leafFrameFactory;
+ this.freePageManagerFactory = freePageManagerFactory;
+ }
+
+ public abstract ITreeIndex createIndexInstance(int fileId);
+
+ public IBufferCache getBufferCache() {
+ return bufferCache;
+ }
+}
diff --git a/hyracks-storage-am-lsm-rtree/pom.xml b/hyracks-storage-am-lsm-rtree/pom.xml
new file mode 100644
index 0000000..06a3719
--- /dev/null
+++ b/hyracks-storage-am-lsm-rtree/pom.xml
@@ -0,0 +1,56 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-rtree</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-common</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-btree</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-rtree</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <type>jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/BTreeFactory.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/BTreeFactory.java
new file mode 100644
index 0000000..98824ab
--- /dev/null
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/BTreeFactory.java
@@ -0,0 +1,24 @@
+package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
+
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeFactory;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+public class BTreeFactory extends TreeFactory {
+
+ public BTreeFactory(IBufferCache bufferCache, LinkedListFreePageManagerFactory freePageManagerFactory, MultiComparator cmp,
+ int fieldCount, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
+ super(bufferCache, freePageManagerFactory, cmp, fieldCount, interiorFrameFactory, leafFrameFactory);
+ }
+
+ @Override
+ public ITreeIndex createIndexInstance(int fileId) {
+ return new BTree(bufferCache, fieldCount, cmp, freePageManagerFactory.createFreePageManager(fileId),
+ interiorFrameFactory, leafFrameFactory);
+ }
+
+}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
new file mode 100644
index 0000000..7b59f1b
--- /dev/null
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -0,0 +1,592 @@
+package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.ListIterator;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFileNameManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMTree;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
+import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.SearchPredicate;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+
+public class LSMRTree implements ILSMTree {
+
+ // In-memory components.
+ private RTree memRTree;
+ private final BTree memBTree;
+ private final InMemoryFreePageManager memFreePageManager;
+ private int rtreeFileId;
+ private int btreeFileId;
+
+ // On-disk components.
+ private final ILSMFileNameManager fileNameManager;
+ private final RTreeFactory diskRTreeFactory;
+ private final BTreeFactory diskBTreeFactory;
+ private final IBufferCache diskBufferCache;
+ private final IFileMapProvider diskFileMapProvider;
+ private LinkedList<RTree> onDiskRTrees = new LinkedList<RTree>();
+ private LinkedList<RTree> mergedRTrees = new LinkedList<RTree>();
+ private LinkedList<BTree> onDiskBTrees = new LinkedList<BTree>();
+ private LinkedList<BTree> mergedBTrees = new LinkedList<BTree>();
+ private int onDiskRTreeCount;
+ private int onDiskBTreeCount;
+
+ // Common for in-memory and on-disk components.
+ private final ITreeIndexFrameFactory rtreeInteriorFrameFactory;
+ private final ITreeIndexFrameFactory btreeInteriorFrameFactory;
+ private final ITreeIndexFrameFactory rtreeLeafFrameFactory;
+ private final ITreeIndexFrameFactory btreeLeafFrameFactory;
+ private final MultiComparator btreeCmp;
+
+ // For dealing with concurrent accesses.
+ private int threadRefCount;
+ private boolean flushFlag;
+
+ public LSMRTree(IBufferCache memBufferCache, InMemoryFreePageManager memFreePageManager,
+ ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
+ ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
+ ILSMFileNameManager fileNameManager, RTreeFactory diskRTreeFactory, BTreeFactory diskBTreeFactory,
+ IFileMapProvider diskFileMapProvider, int fieldCount, MultiComparator rtreeCmp, MultiComparator btreeCmp) {
+
+ memRTree = new RTree(memBufferCache, fieldCount, rtreeCmp, memFreePageManager, rtreeInteriorFrameFactory,
+ rtreeLeafFrameFactory);
+ memBTree = new BTree(memBufferCache, fieldCount, btreeCmp, memFreePageManager, btreeInteriorFrameFactory,
+ btreeLeafFrameFactory);
+
+ this.memFreePageManager = memFreePageManager;
+ this.rtreeInteriorFrameFactory = rtreeInteriorFrameFactory;
+ this.rtreeLeafFrameFactory = rtreeLeafFrameFactory;
+ this.btreeInteriorFrameFactory = btreeInteriorFrameFactory;
+ this.btreeLeafFrameFactory = btreeLeafFrameFactory;
+
+ this.diskBufferCache = diskRTreeFactory.getBufferCache();
+ this.diskFileMapProvider = diskFileMapProvider;
+ this.diskRTreeFactory = diskRTreeFactory;
+ this.diskBTreeFactory = diskBTreeFactory;
+ this.btreeCmp = btreeCmp;
+ this.onDiskRTreeCount = 0;
+ this.onDiskBTreeCount = 0;
+ this.threadRefCount = 0;
+ this.flushFlag = false;
+ this.fileNameManager = fileNameManager;
+ this.rtreeFileId = 0;
+ this.btreeFileId = 1;
+ }
+
+ @Override
+ public ITreeIndexAccessor createAccessor() {
+ return new LSMRTreeAccessor(this);
+ }
+
+ @Override
+ public IIndexBulkLoadContext beginBulkLoad(float fillFactor) throws TreeIndexException, HyracksDataException {
+ // Note that by using a flush target file name, we state that the new
+ // bulk loaded tree is "newer" than any other merged tree.
+
+ String fileName = fileNameManager.getFlushFileName();
+ RTree diskRTree = (RTree) createDiskTree(fileName + "-rtree", diskRTreeFactory, true);
+ // For each RTree, we require to have a buddy BTree. thus, we create an
+ // empty BTree. This can be optimized later.
+ createDiskTree(fileName + "-btree", diskBTreeFactory, true);
+ LSMTreeBulkLoadContext bulkLoadCtx = new LSMTreeBulkLoadContext(diskRTree);
+ bulkLoadCtx.beginBulkLoad(fillFactor);
+ return bulkLoadCtx;
+ }
+
+ @Override
+ public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException {
+ LSMTreeBulkLoadContext bulkLoadCtx = (LSMTreeBulkLoadContext) ictx;
+ bulkLoadCtx.getTree().bulkLoadAddTuple(tuple, bulkLoadCtx.getBulkLoadCtx());
+
+ }
+
+ @Override
+ public void endBulkLoad(IIndexBulkLoadContext ictx) throws HyracksDataException {
+ LSMTreeBulkLoadContext bulkLoadCtx = (LSMTreeBulkLoadContext) ictx;
+ bulkLoadCtx.getTree().endBulkLoad(bulkLoadCtx.getBulkLoadCtx());
+ onDiskRTrees.addFirst((RTree) bulkLoadCtx.getTree());
+
+ }
+
+ @Override
+ public ITreeIndexFrameFactory getLeafFrameFactory() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public ITreeIndexFrameFactory getInteriorFrameFactory() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public IFreePageManager getFreePageManager() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public int getFieldCount() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public int getRootPageId() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public IndexType getIndexType() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public void create(int indexFileId) throws HyracksDataException {
+ memRTree.create(rtreeFileId);
+ memBTree.create(btreeFileId);
+ }
+
+ /**
+ * Opens LSMRTree, assuming a consistent state of the disk-resident
+ * components. In particular, registers all files in in base dir of
+ * fileNameManager as on-disk RTrees and BTrees.
+ *
+ * Example pathological scenario to explain "consistent state assumption":
+ * Suppose a merge finished, but before the original files were deleted the
+ * system crashes. We are left in a state where we have the original RTrees
+ * and BTrees in addition to the merged ones. We assume that prior to
+ * calling this method a separate recovery process has ensured the
+ * consistent of the disk-resident components.
+ *
+ * @param indexFileId
+ * Dummy file id.
+ * @throws HyracksDataException
+ */
+ @Override
+ public void open(int indexFileId) throws HyracksDataException {
+ memRTree.open(rtreeFileId);
+ memBTree.open(btreeFileId);
+ File dir = new File(fileNameManager.getBaseDir());
+ FilenameFilter rtreeFilter = new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return !name.startsWith(".") && !name.endsWith("btree");
+ }
+ };
+ String[] rtreeFiles = dir.list(rtreeFilter);
+
+ FilenameFilter btreeFilter = new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return !name.startsWith(".") && !name.endsWith("rtree");
+ }
+ };
+ String[] btreeFiles = dir.list(btreeFilter);
+
+ if (rtreeFiles == null || btreeFiles == null) {
+ return;
+ }
+
+ Comparator<String> fileNameCmp = fileNameManager.getFileNameComparator();
+ Arrays.sort(rtreeFiles, fileNameCmp);
+ for (String fileName : rtreeFiles) {
+ RTree rtree = (RTree) createDiskTree(fileName, diskRTreeFactory, false);
+ onDiskRTrees.add(rtree);
+ }
+
+ Arrays.sort(btreeFiles, fileNameCmp);
+ for (String fileName : btreeFiles) {
+ BTree btree = (BTree) createDiskTree(fileName, diskBTreeFactory, false);
+ onDiskBTrees.add(btree);
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ for (RTree rtree : onDiskRTrees) {
+ diskBufferCache.closeFile(rtree.getFileId());
+ rtree.close();
+ }
+ for (BTree btree : onDiskBTrees) {
+ diskBufferCache.closeFile(btree.getFileId());
+ btree.close();
+ }
+ onDiskRTrees.clear();
+ onDiskBTrees.clear();
+ onDiskRTreeCount = 0;
+ onDiskBTreeCount = 0;
+ memRTree.close();
+ memBTree.close();
+ }
+
+ private ITreeIndex createDiskTree(String fileName, TreeFactory diskTreeFactory, boolean createTree)
+ throws HyracksDataException {
+ // Register the new tree file.
+ FileReference file = new FileReference(new File(fileName));
+ // TODO: Delete the file during cleanup.
+ diskBufferCache.createFile(file);
+ int diskTreeFileId = diskFileMapProvider.lookupFileId(file);
+ // TODO: Close the file during cleanup.
+ diskBufferCache.openFile(diskTreeFileId);
+ // Create new tree instance.
+ ITreeIndex diskTree = diskTreeFactory.createIndexInstance(diskTreeFileId);
+ if (createTree) {
+ diskTree.create(diskTreeFileId);
+ }
+ // TODO: Close the tree during cleanup.
+ diskTree.open(diskTreeFileId);
+ return diskTree;
+ }
+
+ @Override
+ public void merge() throws Exception {
+
+ // Cursor setting -- almost the same as search, only difference is
+ // "no cursor for in-memory tree"
+ int numberOfInDiskTrees = onDiskRTrees.size();
+
+ RTreeSearchCursor[] rtreeCursors = new RTreeSearchCursor[numberOfInDiskTrees];
+ BTreeRangeSearchCursor[] btreeCursors = new BTreeRangeSearchCursor[numberOfInDiskTrees];
+
+ for (int i = 0; i < numberOfInDiskTrees; i++) {
+ rtreeCursors[i] = new RTreeSearchCursor((IRTreeInteriorFrame) rtreeInteriorFrameFactory.createFrame(),
+ (IRTreeLeafFrame) rtreeLeafFrameFactory.createFrame());
+
+ btreeCursors[i] = new BTreeRangeSearchCursor((IBTreeLeafFrame) btreeLeafFrameFactory.createFrame(), false);
+ }
+
+ String fileName = fileNameManager.getMergeFileName();
+ RTree mergedRTree = (RTree) createDiskTree(fileName + "-rtree", diskRTreeFactory, true);
+ BTree mergedBTree = (BTree) createDiskTree(fileName + "-btree", diskBTreeFactory, true);
+
+ // BulkLoad the tuples from the trees into the new merged trees.
+ IIndexBulkLoadContext rtreeBulkLoadCtx = mergedRTree.beginBulkLoad(1.0f);
+ IIndexBulkLoadContext btreeBulkLoadCtx = mergedBTree.beginBulkLoad(1.0f);
+
+ for (int i = 0; i < numberOfInDiskTrees; i++) {
+
+ // scan the RTrees
+ ITreeIndexCursor rtreeScanCursor = new RTreeSearchCursor(
+ (IRTreeInteriorFrame) rtreeInteriorFrameFactory.createFrame(),
+ (IRTreeLeafFrame) rtreeLeafFrameFactory.createFrame());
+ SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
+
+ ITreeIndexAccessor onDiskRTreeAccessor = onDiskRTrees.get(i).createAccessor();
+ onDiskRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
+
+ try {
+ while (rtreeScanCursor.hasNext()) {
+ rtreeScanCursor.next();
+ ITupleReference frameTuple = rtreeScanCursor.getTuple();
+ mergedRTree.bulkLoadAddTuple(frameTuple, rtreeBulkLoadCtx);
+ }
+ } finally {
+ rtreeScanCursor.close();
+ }
+
+ // scan the BTrees
+ ITreeIndexCursor btreeScanCursor = new BTreeRangeSearchCursor(
+ (IBTreeLeafFrame) btreeLeafFrameFactory.createFrame(), false);
+ RangePredicate btreeNullPredicate = new RangePredicate(true, null, null, true, true, null, null);
+ ITreeIndexAccessor onDiskBTreeAccessor = onDiskBTrees.get(i).createAccessor();
+ onDiskBTreeAccessor.search(btreeScanCursor, btreeNullPredicate);
+
+ try {
+ while (btreeScanCursor.hasNext()) {
+ btreeScanCursor.next();
+ ITupleReference frameTuple = btreeScanCursor.getTuple();
+ mergedBTree.bulkLoadAddTuple(frameTuple, btreeBulkLoadCtx);
+ }
+ } finally {
+ btreeScanCursor.close();
+ }
+
+ }
+ mergedRTree.endBulkLoad(rtreeBulkLoadCtx);
+ mergedBTree.endBulkLoad(btreeBulkLoadCtx);
+
+ // TODO: complete the merge code
+
+ }
+
+ @Override
+ public void flush() throws HyracksDataException, TreeIndexException {
+
+ // scan the RTree
+ ITreeIndexCursor rtreeScanCursor = new RTreeSearchCursor(
+ (IRTreeInteriorFrame) rtreeInteriorFrameFactory.createFrame(),
+ (IRTreeLeafFrame) rtreeLeafFrameFactory.createFrame());
+ SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
+
+ ITreeIndexAccessor memRTreeAccessor = memRTree.createAccessor();
+ memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
+
+ String fileName = fileNameManager.getFlushFileName();
+ RTree diskRTree = (RTree) createDiskTree(fileName + "-rtree", diskRTreeFactory, true);
+
+ // BulkLoad the tuples from the in-memory tree into the new disk RTree.
+ IIndexBulkLoadContext rtreeBulkLoadCtx = diskRTree.beginBulkLoad(1.0f);
+
+ try {
+ while (rtreeScanCursor.hasNext()) {
+ rtreeScanCursor.next();
+ ITupleReference frameTuple = rtreeScanCursor.getTuple();
+ diskRTree.bulkLoadAddTuple(frameTuple, rtreeBulkLoadCtx);
+ }
+ } finally {
+ rtreeScanCursor.close();
+ }
+ diskRTree.endBulkLoad(rtreeBulkLoadCtx);
+
+ // scan the BTree
+ ITreeIndexCursor btreeScanCursor = new BTreeRangeSearchCursor(
+ (IBTreeLeafFrame) btreeLeafFrameFactory.createFrame(), false);
+ RangePredicate btreeNullPredicate = new RangePredicate(true, null, null, true, true, null, null);
+ ITreeIndexAccessor memBTreeAccessor = memBTree.createAccessor();
+ memBTreeAccessor.search(btreeScanCursor, btreeNullPredicate);
+
+ BTree diskBTree = (BTree) createDiskTree(fileName + "-btree", diskBTreeFactory, true);
+
+ // BulkLoad the tuples from the in-memory tree into the new disk BTree.
+ IIndexBulkLoadContext btreeBulkLoadCtx = diskBTree.beginBulkLoad(1.0f);
+ try {
+ while (btreeScanCursor.hasNext()) {
+ btreeScanCursor.next();
+ ITupleReference frameTuple = btreeScanCursor.getTuple();
+ diskBTree.bulkLoadAddTuple(frameTuple, btreeBulkLoadCtx);
+ }
+ } finally {
+ btreeScanCursor.close();
+ }
+ diskBTree.endBulkLoad(btreeBulkLoadCtx);
+
+ resetInMemoryTrees();
+
+ onDiskRTrees.addFirst(diskRTree);
+ onDiskBTrees.addFirst(diskBTree);
+
+ }
+
+ public void resetInMemoryTrees() throws HyracksDataException {
+ memFreePageManager.reset();
+ memRTree.create(rtreeFileId);
+ memBTree.create(btreeFileId);
+ }
+
+ public void threadEnter() {
+ threadRefCount++;
+ }
+
+ public void threadExit() throws HyracksDataException, TreeIndexException {
+ synchronized (this) {
+ threadRefCount--;
+ // Check if we've reached or exceeded the maximum number of pages.
+ if (!flushFlag && memFreePageManager.isFull()) {
+ flushFlag = true;
+ }
+ // Flush will only be handled by last exiting thread.
+ if (flushFlag && threadRefCount == 0) {
+ flush();
+ flushFlag = false;
+ }
+ }
+ }
+
+ private void insert(ITupleReference tuple, LSMRTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
+ boolean waitForFlush = false;
+ do {
+ synchronized (this) {
+ if (!flushFlag) {
+ threadEnter();
+ waitForFlush = false;
+ }
+ }
+ } while (waitForFlush == true);
+ ctx.memRTreeAccessor.insert(tuple);
+ try {
+ threadExit();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ private void delete(ITupleReference tuple, LSMRTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
+ boolean waitForFlush = false;
+ do {
+ synchronized (this) {
+ if (!flushFlag) {
+ threadEnter();
+ waitForFlush = false;
+ }
+ }
+ } while (waitForFlush == true);
+ ctx.memBTreeAccessor.insert(tuple);
+ try {
+ threadExit();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ private void search(ITreeIndexCursor cursor, ISearchPredicate rtreeSearchPred, LSMRTreeOpContext ctx,
+ boolean includeMemTree) throws Exception {
+
+ boolean continuePerformOp = false;
+ ctx.reset(IndexOp.SEARCH);
+ while (continuePerformOp == false) {
+ synchronized (this) {
+ if (!flushFlag) {
+ threadRefCount++;
+ continuePerformOp = true;
+ }
+ }
+ }
+
+ int numDiskTrees = onDiskRTrees.size();
+
+ ITreeIndexAccessor[] bTreeAccessors;
+ int diskBTreeIx = 0;
+ if (includeMemTree) {
+ bTreeAccessors = new ITreeIndexAccessor[numDiskTrees + 1];
+ bTreeAccessors[0] = ctx.memBTreeAccessor;
+ diskBTreeIx++;
+ } else {
+ bTreeAccessors = new ITreeIndexAccessor[numDiskTrees];
+ }
+
+ ListIterator<BTree> diskBTreesIter = onDiskBTrees.listIterator();
+ while (diskBTreesIter.hasNext()) {
+ BTree diskBTree = diskBTreesIter.next();
+ bTreeAccessors[diskBTreeIx] = diskBTree.createAccessor();
+ diskBTreeIx++;
+ }
+
+ LSMRTreeSearchCursor lsmRTreeCursor = (LSMRTreeSearchCursor) cursor;
+ LSMRTreeCursorInitialState initialState = new LSMRTreeCursorInitialState(numDiskTrees + 1,
+ rtreeLeafFrameFactory, rtreeInteriorFrameFactory, btreeLeafFrameFactory, btreeCmp, bTreeAccessors, this);
+ lsmRTreeCursor.open(initialState, rtreeSearchPred);
+
+ int cursorIx = 1;
+ if (includeMemTree) {
+ ctx.memRTreeAccessor.search(((LSMRTreeSearchCursor) lsmRTreeCursor).getCursor(0), rtreeSearchPred);
+ cursorIx = 1;
+ } else {
+ cursorIx = 0;
+ }
+
+ // Open cursors of on-disk RTrees
+ ITreeIndexAccessor[] diskRTreeAccessors = new ITreeIndexAccessor[numDiskTrees];
+ ListIterator<RTree> diskRTreesIter = onDiskRTrees.listIterator();
+
+ int diskRTreeIx = 0;
+ while (diskRTreesIter.hasNext()) {
+ RTree diskRTree = diskRTreesIter.next();
+ diskRTreeAccessors[diskRTreeIx] = diskRTree.createAccessor();
+ diskRTreeAccessors[diskRTreeIx].search(lsmRTreeCursor.getCursor(cursorIx), rtreeSearchPred);
+ cursorIx++;
+ diskRTreeIx++;
+ }
+
+ }
+
+ public LinkedList<BTree> getOnDiskBTrees() {
+ return onDiskBTrees;
+ }
+
+ private LSMRTreeOpContext createOpContext() {
+
+ return new LSMRTreeOpContext((RTree.RTreeAccessor) memRTree.createAccessor(),
+ (IRTreeLeafFrame) rtreeLeafFrameFactory.createFrame(),
+ (IRTreeInteriorFrame) rtreeInteriorFrameFactory.createFrame(), memFreePageManager
+ .getMetaDataFrameFactory().createFrame(), 8, (BTree.BTreeAccessor) memBTree.createAccessor(),
+ btreeLeafFrameFactory, btreeInteriorFrameFactory, memFreePageManager.getMetaDataFrameFactory()
+ .createFrame(), btreeCmp);
+ }
+
+ private class LSMRTreeAccessor implements ITreeIndexAccessor {
+ private LSMRTree lsmRTree;
+ private LSMRTreeOpContext ctx;
+
+ public LSMRTreeAccessor(LSMRTree lsmRTree) {
+ this.lsmRTree = lsmRTree;
+ this.ctx = lsmRTree.createOpContext();
+
+ }
+
+ @Override
+ public void insert(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
+ ctx.reset(IndexOp.INSERT);
+ lsmRTree.insert(tuple, ctx);
+ }
+
+ @Override
+ public void update(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
+ throw new UnsupportedOperationException("Update not supported by LSMRTree");
+ }
+
+ @Override
+ public void delete(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
+ ctx.reset(IndexOp.DELETE);
+ lsmRTree.delete(tuple, ctx);
+ }
+
+ @Override
+ public ITreeIndexCursor createSearchCursor() {
+ return new LSMRTreeSearchCursor();
+ }
+
+ @Override
+ public void search(ITreeIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException,
+ TreeIndexException {
+ ctx.reset(IndexOp.SEARCH);
+ // TODO: fix exception handling throughout LSM tree.
+ try {
+ lsmRTree.search(cursor, searchPred, ctx, true);
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public ITreeIndexCursor createDiskOrderScanCursor() {
+ throw new UnsupportedOperationException("DiskOrderScan not supported by LSM-RTree.");
+ }
+
+ @Override
+ public void diskOrderScan(ITreeIndexCursor cursor) throws HyracksDataException {
+ throw new UnsupportedOperationException("DiskOrderScan not supported by LSM-RTree.");
+ }
+ }
+
+}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeCursorInitialState.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeCursorInitialState.java
new file mode 100644
index 0000000..7239f3a
--- /dev/null
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeCursorInitialState.java
@@ -0,0 +1,68 @@
+package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
+
+import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+
+public class LSMRTreeCursorInitialState implements ICursorInitialState {
+
+ private int numberOfTrees;
+ private ITreeIndexFrameFactory rtreeInteriorFrameFactory;
+ private ITreeIndexFrameFactory rtreeLeafFrameFactory;
+ private ITreeIndexFrameFactory btreeLeafFrameFactory;
+ private MultiComparator btreeCmp;
+ private LSMRTree lsmRTree;
+ private ITreeIndexAccessor[] bTreeAccessors;
+
+ public LSMRTreeCursorInitialState(int numberOfTrees, ITreeIndexFrameFactory rtreeLeafFrameFactory,
+ ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
+ MultiComparator btreeCmp, ITreeIndexAccessor[] bTreeAccessors, LSMRTree lsmRTree) {
+ this.numberOfTrees = numberOfTrees;
+ this.rtreeLeafFrameFactory = rtreeLeafFrameFactory;
+ this.rtreeInteriorFrameFactory = rtreeInteriorFrameFactory;
+ this.btreeLeafFrameFactory = btreeLeafFrameFactory;
+ this.btreeCmp = btreeCmp;
+ this.lsmRTree = lsmRTree;
+ this.bTreeAccessors = bTreeAccessors;
+ }
+
+ public int getNumberOfTrees() {
+ return numberOfTrees;
+ }
+
+ public ITreeIndexFrameFactory getRTreeInteriorFrameFactory() {
+ return rtreeInteriorFrameFactory;
+ }
+
+ public ITreeIndexFrameFactory getRTreeLeafFrameFactory() {
+ return rtreeLeafFrameFactory;
+ }
+
+ public ITreeIndexFrameFactory getBTreeLeafFrameFactory() {
+ return btreeLeafFrameFactory;
+ }
+
+ public MultiComparator getBTreeCmp() {
+ return btreeCmp;
+ }
+
+ @Override
+ public ICachedPage getPage() {
+ return null;
+ }
+
+ @Override
+ public void setPage(ICachedPage page) {
+ }
+
+ public ITreeIndexAccessor[] getBTreeAccessors() {
+ return bTreeAccessors;
+ }
+
+ public LSMRTree getLsmRTree() {
+ return lsmRTree;
+ }
+
+}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeInMemoryBufferCache.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeInMemoryBufferCache.java
new file mode 100644
index 0000000..bf13ca0
--- /dev/null
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeInMemoryBufferCache.java
@@ -0,0 +1,47 @@
+package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache.CachedPage;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+
+public class LSMRTreeInMemoryBufferCache extends InMemoryBufferCache {
+
+ public LSMRTreeInMemoryBufferCache(ICacheMemoryAllocator allocator, int pageSize, int numPages) {
+ super(allocator, pageSize, numPages);
+ }
+
+ @Override
+ public ICachedPage pin(long dpid, boolean newPage) {
+ int pageId = BufferedFileHandle.getPageId(dpid);
+ int fileId = BufferedFileHandle.getFileId(dpid);
+
+ if (pageId < pages.length) {
+ // Common case: Return regular page.
+
+ if (pageId == 0 || pageId == 1) {
+ return pages[pageId + 2 * fileId];
+ } else {
+ return pages[pageId];
+ }
+ } else {
+ // Rare case: Return overflow page, possibly expanding overflow
+ // array.
+ synchronized (overflowPages) {
+ int numNewPages = pageId - pages.length - overflowPages.size() + 1;
+ if (numNewPages > 0) {
+ ByteBuffer[] buffers = allocator.allocate(pageSize, numNewPages);
+ for (int i = 0; i < numNewPages; i++) {
+ CachedPage overflowPage = new CachedPage(pages.length + overflowPages.size(), buffers[i]);
+ overflowPages.add(overflowPage);
+ }
+ }
+ return overflowPages.get(pageId - pages.length);
+ }
+ }
+ }
+
+}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeInMemoryFreePageManager.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeInMemoryFreePageManager.java
new file mode 100644
index 0000000..adec7e7
--- /dev/null
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeInMemoryFreePageManager.java
@@ -0,0 +1,31 @@
+package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
+
+public class LSMRTreeInMemoryFreePageManager extends InMemoryFreePageManager {
+
+ public LSMRTreeInMemoryFreePageManager(int capacity, ITreeIndexMetaDataFrameFactory metaDataFrameFactory) {
+ super(capacity, metaDataFrameFactory);
+ // We start the currentPageId from 3, because the RTree uses
+ // the first page as metadata page, and the second page as root page.
+ // And the BTree uses the third page as metadata, and the third page as root page
+ // (when returning free pages we first increment, then get)
+ currentPageId.set(3);
+ }
+
+ @Override
+ public void init(ITreeIndexMetaDataFrame metaFrame, int currentMaxPage) throws HyracksDataException {
+ currentPageId.set(3);
+ }
+
+ public int getCapacity() {
+ return capacity - 4;
+ }
+
+ public void reset() {
+ currentPageId.set(3);
+ }
+}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
new file mode 100644
index 0000000..4cf18c4
--- /dev/null
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
@@ -0,0 +1,42 @@
+package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
+
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
+import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeOpContext;
+
+public final class LSMRTreeOpContext {
+
+ private RTreeOpContext rtreeOpContext;
+ private BTreeOpContext btreeOpContext;
+ public final RTree.RTreeAccessor memRTreeAccessor;
+ public final BTree.BTreeAccessor memBTreeAccessor;
+
+ public LSMRTreeOpContext(RTree.RTreeAccessor memRtreeAccessor, IRTreeLeafFrame rtreeLeafFrame,
+ IRTreeInteriorFrame rtreeInteriorFrame, ITreeIndexMetaDataFrame rtreeMetaFrame, int rTreeHeightHint,
+ BTree.BTreeAccessor memBtreeAccessor, ITreeIndexFrameFactory btreeLeafFrameFactory,
+ ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexMetaDataFrame btreeMetaFrame,
+ MultiComparator btreeCmp) {
+
+ this.memRTreeAccessor = memRtreeAccessor;
+ this.memBTreeAccessor = memBtreeAccessor;
+ this.rtreeOpContext = new RTreeOpContext(rtreeLeafFrame, rtreeInteriorFrame, rtreeMetaFrame, rTreeHeightHint);
+ this.btreeOpContext = new BTreeOpContext(btreeLeafFrameFactory, btreeInteriorFrameFactory, btreeMetaFrame,
+ btreeCmp);
+ }
+
+ public void reset(IndexOp newOp) {
+ if (newOp == IndexOp.INSERT) {
+ rtreeOpContext.reset(newOp);
+ } else if (newOp == IndexOp.DELETE) {
+ btreeOpContext.reset(IndexOp.INSERT);
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java
new file mode 100644
index 0000000..98de361
--- /dev/null
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java
@@ -0,0 +1,153 @@
+package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
+import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeSearchCursor;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+
+public class LSMRTreeSearchCursor implements ITreeIndexCursor {
+
+ private RTreeSearchCursor[] rtreeCursors;
+ private BTreeRangeSearchCursor[] btreeCursors;
+ private ITreeIndexAccessor[] diskBTreeAccessors;
+ private int currentCursror;
+ private MultiComparator btreeCmp;
+ private int numberOfTrees;
+ private LSMRTree lsmRTree;
+ private RangePredicate btreeRangePredicate;
+ private ITupleReference frameTuple;
+
+ public LSMRTreeSearchCursor() {
+ currentCursror = 0;
+ }
+
+ public RTreeSearchCursor getCursor(int cursorIndex) {
+ return rtreeCursors[cursorIndex];
+ }
+
+ @Override
+ public void reset() {
+ // do nothing
+ }
+
+ @Override
+ public boolean hasNext() throws HyracksDataException {
+ for (int i = currentCursror; i < numberOfTrees; i++) {
+ while (rtreeCursors[i].hasNext()) {
+ rtreeCursors[i].next();
+ ITupleReference currentTuple = rtreeCursors[i].getTuple();
+ boolean killerTupleFound = false;
+ for (int j = 0; j <= i; j++) {
+ btreeRangePredicate.setHighKey(currentTuple, true);
+ btreeRangePredicate.setLowKey(currentTuple, true);
+
+ try {
+ diskBTreeAccessors[j].search(btreeCursors[j], btreeRangePredicate);
+ } catch (TreeIndexException e) {
+ throw new HyracksDataException(e);
+ }
+
+ if (btreeCursors[j].hasNext()) {
+ killerTupleFound = true;
+ break;
+ }
+ }
+ if (!killerTupleFound) {
+ frameTuple = currentTuple;
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public void next() throws HyracksDataException {
+ while (true) {
+ if (currentCursror < numberOfTrees || rtreeCursors[currentCursror].hasNext()) {
+ break;
+ } else {
+ currentCursror++;
+ }
+ }
+ }
+
+ @Override
+ public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+
+ lsmRTree = ((LSMRTreeCursorInitialState) initialState).getLsmRTree();
+ btreeCmp = ((LSMRTreeCursorInitialState) initialState).getBTreeCmp();
+
+ numberOfTrees = ((LSMRTreeCursorInitialState) initialState).getNumberOfTrees();
+
+ diskBTreeAccessors = ((LSMRTreeCursorInitialState) initialState).getBTreeAccessors();
+
+ rtreeCursors = new RTreeSearchCursor[numberOfTrees];
+ btreeCursors = new BTreeRangeSearchCursor[numberOfTrees];
+
+ for (int i = 0; i < numberOfTrees; i++) {
+ rtreeCursors[i] = new RTreeSearchCursor((IRTreeInteriorFrame) ((LSMRTreeCursorInitialState) initialState)
+ .getRTreeInteriorFrameFactory().createFrame(),
+ (IRTreeLeafFrame) ((LSMRTreeCursorInitialState) initialState).getRTreeLeafFrameFactory()
+ .createFrame());
+
+ btreeCursors[i] = new BTreeRangeSearchCursor((IBTreeLeafFrame) ((LSMRTreeCursorInitialState) initialState)
+ .getBTreeLeafFrameFactory().createFrame(), false);
+ }
+ btreeRangePredicate = new RangePredicate(true, null, null, true, true, btreeCmp, btreeCmp);
+ }
+
+ @Override
+ public ICachedPage getPage() {
+ // do nothing
+ return null;
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ for (int i = 0; i < numberOfTrees; i++) {
+ rtreeCursors[i].close();
+ btreeCursors[i].close();
+ }
+ rtreeCursors = null;
+ btreeCursors = null;
+
+ try {
+ lsmRTree.threadExit();
+ } catch (TreeIndexException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void setBufferCache(IBufferCache bufferCache) {
+ // do nothing
+ }
+
+ @Override
+ public void setFileId(int fileId) {
+ // do nothing
+ }
+
+ @Override
+ public ITupleReference getTuple() {
+ return frameTuple;
+ }
+
+ @Override
+ public boolean exclusiveLatchNodes() {
+ return false;
+ }
+}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
new file mode 100644
index 0000000..56780e7
--- /dev/null
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
@@ -0,0 +1,24 @@
+package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
+
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+public class RTreeFactory extends TreeFactory {
+
+ public RTreeFactory(IBufferCache bufferCache, LinkedListFreePageManagerFactory freePageManagerFactory, MultiComparator cmp,
+ int fieldCount, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
+ super(bufferCache, freePageManagerFactory, cmp, fieldCount, interiorFrameFactory, leafFrameFactory);
+ }
+
+ @Override
+ public ITreeIndex createIndexInstance(int fileId) {
+ return new RTree(bufferCache, fieldCount, cmp, freePageManagerFactory.createFreePageManager(fileId),
+ interiorFrameFactory, leafFrameFactory);
+ }
+
+}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/tuples/LSMTypeAwareTupleWriterFactory.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/tuples/LSMTypeAwareTupleWriterFactory.java
new file mode 100644
index 0000000..876df56
--- /dev/null
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/tuples/LSMTypeAwareTupleWriterFactory.java
@@ -0,0 +1,30 @@
+package edu.uci.ics.hyracks.storage.am.lsm.rtree.tuples;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriter;
+
+public class LSMTypeAwareTupleWriterFactory extends TypeAwareTupleWriterFactory {
+
+ private static final long serialVersionUID = 1L;
+ private ITypeTraits[] typeTraits;
+ private final boolean isDelete;
+
+ public LSMTypeAwareTupleWriterFactory(ITypeTraits[] typeTraits, boolean isDelete) {
+ super(typeTraits);
+ this.typeTraits = typeTraits;
+ this.isDelete = isDelete;
+ }
+
+ @Override
+ public ITreeIndexTupleWriter createTupleWriter() {
+ if (isDelete) {
+ return new TypeAwareTupleWriter(typeTraits);
+ } else {
+ return new RTreeTypeAwareTupleWriter(typeTraits);
+ }
+ }
+
+}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java
index 2b3065d..242adad 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java
@@ -25,6 +25,8 @@
public int getBestChildPageId();
+ public int getChildPageId(int tupleIndex);
+
public int getChildPageIdIfIntersect(ITupleReference tuple, int tupleIndex,
MultiComparator cmp);
@@ -40,4 +42,6 @@
MultiComparator cmp);
public void enlarge(ITupleReference tuple, MultiComparator cmp);
+
+ boolean checkEnlargement(ITupleReference tuple, MultiComparator cmp);
}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
index f3e6be2..696443b 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
@@ -169,6 +169,12 @@
}
@Override
+ public int getChildPageId(int tupleIndex) {
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ return buf.getInt(getChildPointerOff(frameTuple));
+ }
+
+ @Override
public int getChildPageIdIfIntersect(ITupleReference tuple, int tupleIndex, MultiComparator cmp) {
frameTuple.setFieldCount(cmp.getKeyFieldCount());
frameTuple.resetByTupleIndex(this, tupleIndex);
@@ -437,7 +443,7 @@
frameTuple.setFieldCount(tuple.getFieldCount());
slotManager.insertSlot(-1, buf.getInt(freeSpaceOff));
int freeSpace = buf.getInt(freeSpaceOff);
- int bytesWritten = tupleWriter.writeTupleFields(tuple, 0, tuple.getFieldCount(), buf, freeSpace);
+ int bytesWritten = tupleWriter.writeTupleFields(tuple, 0, tuple.getFieldCount(), buf.array(), freeSpace);
System.arraycopy(tuple.getFieldData(tuple.getFieldCount() - 1), getChildPointerOff(tuple), buf.array(),
freeSpace + bytesWritten, childPtrSize);
int tupleSize = bytesWritten + childPtrSize;
@@ -585,6 +591,27 @@
}
@Override
+ public boolean checkEnlargement(ITupleReference tuple, MultiComparator cmp) {
+ int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ int c = cmp.getComparators()[i].compare(frameTuple.getFieldData(i), frameTuple.getFieldStart(i),
+ frameTuple.getFieldLength(i), tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i));
+ if (c > 0) {
+ return true;
+ }
+ c = cmp.getComparators()[j].compare(frameTuple.getFieldData(j), frameTuple.getFieldStart(j),
+ frameTuple.getFieldLength(j), tuple.getFieldData(j), tuple.getFieldStart(j),
+ tuple.getFieldLength(j));
+ if (c < 0) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
public void enlarge(ITupleReference tuple, MultiComparator cmp) {
int maxFieldPos = cmp.getKeyFieldCount() / 2;
for (int i = 0; i < maxFieldPos; i++) {
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
index 5f78dcc..86cad34e 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
@@ -34,7 +34,6 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
-import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
@@ -51,7 +50,6 @@
public class RTree implements ITreeIndex {
- private boolean created = false;
private boolean loaded = false;
private final int rootPage = 1; // the root page never changes
@@ -206,10 +204,6 @@
public void create(int fileId) throws HyracksDataException {
treeLatch.writeLock().lock();
try {
- if (created) {
- return;
- }
-
ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
freePageManager.init(metaFrame, rootPage);
@@ -230,8 +224,6 @@
incrementUnpins();
}
currentLevel = 0;
-
- created = true;
} finally {
treeLatch.writeLock().unlock();
}
@@ -245,13 +237,17 @@
fileId = -1;
}
+ public int getFileId() {
+ return fileId;
+ }
+
private RTreeOpContext createOpContext() {
return new RTreeOpContext((IRTreeLeafFrame) leafFrameFactory.createFrame(),
(IRTreeInteriorFrame) interiorFrameFactory.createFrame(), freePageManager.getMetaDataFrameFactory()
.createFrame(), 8);
}
- private void insert(ITupleReference tuple, IIndexOpContext ictx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void insert(ITupleReference tuple, IIndexOpContext ictx) throws HyracksDataException, TreeIndexException {
RTreeOpContext ctx = (RTreeOpContext) ictx;
ctx.reset();
ctx.setTuple(tuple);
@@ -289,7 +285,7 @@
incrementUnpins();
}
- public ICachedPage findLeaf(RTreeOpContext ctx) throws HyracksDataException {
+ private ICachedPage findLeaf(RTreeOpContext ctx) throws HyracksDataException {
int pageId = rootPage;
boolean writeLatched = false;
boolean readLatched = false;
@@ -329,7 +325,7 @@
incrementReadLatchesAcquired();
}
}
-
+
if (pageId != rootPage && parentLsn < ctx.interiorFrame.getPageNsn()) {
// Concurrent split detected, go back to parent and
// re-choose
@@ -364,40 +360,58 @@
ctx.interiorFrame.findBestChild(ctx.getTuple(), cmp);
int childPageId = ctx.interiorFrame.getBestChildPageId();
- if (!writeLatched) {
- node.releaseReadLatch();
- readLatched = false;
- incrementReadLatchesReleased();
- // TODO: do we need to un-pin and pin again?
+ // check if enlargement is needed
+ boolean enlarementIsNeeded = ctx.interiorFrame.checkEnlargement(ctx.getTuple(), cmp);
+ if (enlarementIsNeeded) {
+ if (!writeLatched) {
+ node.releaseReadLatch();
+ readLatched = false;
+ incrementReadLatchesReleased();
+ // TODO: do we need to un-pin and pin again?
+ bufferCache.unpin(node);
+ incrementUnpins();
+
+ node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ incrementPins();
+ node.acquireWriteLatch();
+ writeLatched = true;
+ incrementWriteLatchesAcquired();
+ ctx.interiorFrame.setPage(node);
+
+ if (ctx.interiorFrame.getPageLsn() != pageLsn) {
+ // The page was changed while we unlocked it;
+ // thus,
+ // retry (re-choose best child)
+
+ ctx.pathList.moveLast();
+ continue;
+ }
+ }
+ // We don't need to reset the frameTuple because it is
+ // already pointing to the best child
+ ctx.interiorFrame.enlarge(ctx.getTuple(), cmp);
+
+ node.releaseWriteLatch();
+ writeLatched = false;
+ incrementWriteLatchesReleased();
bufferCache.unpin(node);
incrementUnpins();
-
- node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- incrementPins();
- node.acquireWriteLatch();
- writeLatched = true;
- incrementWriteLatchesAcquired();
- ctx.interiorFrame.setPage(node);
-
- if (ctx.interiorFrame.getPageLsn() != pageLsn) {
- // The page was changed while we unlocked it; thus,
- // retry (re-choose best child)
-
- ctx.pathList.moveLast();
- continue;
+ } else {
+ if (readLatched) {
+ node.releaseReadLatch();
+ readLatched = false;
+ incrementReadLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
+ } else if (writeLatched) {
+ node.releaseWriteLatch();
+ writeLatched = false;
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
}
}
- // We don't need to reset the frameTuple because it is
- // already pointing to the best child
- ctx.interiorFrame.enlarge(ctx.getTuple(), cmp);
-
- node.releaseWriteLatch();
- writeLatched = false;
- incrementWriteLatchesReleased();
- bufferCache.unpin(node);
- incrementUnpins();
-
pageId = childPageId;
parentLsn = pageLsn;
} else {
@@ -426,7 +440,7 @@
}
private void insertTuple(ICachedPage node, int pageId, ITupleReference tuple, RTreeOpContext ctx, boolean isLeaf)
- throws HyracksDataException, TreeIndexException, PageAllocationException {
+ throws HyracksDataException, TreeIndexException {
FrameOpSpaceStatus spaceStatus;
if (!isLeaf) {
spaceStatus = ctx.interiorFrame.hasSpaceInsert(tuple);
@@ -552,7 +566,7 @@
}
}
- public void updateParentForInsert(RTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void updateParentForInsert(RTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
boolean writeLatched = false;
int parentId = ctx.pathList.getLastPageId();
ICachedPage parentNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, parentId), false);
@@ -625,7 +639,7 @@
updateParentForInsert(ctx);
}
- public void findPath(RTreeOpContext ctx) throws HyracksDataException {
+ private void findPath(RTreeOpContext ctx) throws HyracksDataException {
boolean readLatched = false;
int pageId = rootPage;
int parentIndex = -1;
@@ -680,14 +694,14 @@
}
}
- public void fillPath(RTreeOpContext ctx, int pageIndex) {
+ private void fillPath(RTreeOpContext ctx, int pageIndex) {
if (pageIndex != -1) {
fillPath(ctx, ctx.traverseList.getPageIndex(pageIndex));
ctx.pathList.add(ctx.traverseList.getPageId(pageIndex), ctx.traverseList.getPageLsn(pageIndex), -1);
}
}
- public void delete(ITupleReference tuple, RTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
+ private void delete(ITupleReference tuple, RTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
ctx.reset();
ctx.setTuple(tuple);
ctx.splitKey.reset();
@@ -715,7 +729,7 @@
}
}
- public void updateParentForDelete(RTreeOpContext ctx) throws HyracksDataException {
+ private void updateParentForDelete(RTreeOpContext ctx) throws HyracksDataException {
boolean writeLatched = false;
int parentId = ctx.pathList.getLastPageId();
ICachedPage parentNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, parentId), false);
@@ -806,7 +820,7 @@
updateParentForDelete(ctx);
}
- public int findTupleToDelete(RTreeOpContext ctx) throws HyracksDataException {
+ private int findTupleToDelete(RTreeOpContext ctx) throws HyracksDataException {
boolean writeLatched = false;
boolean readLatched = false;
boolean succeed = false;
@@ -921,7 +935,7 @@
return -1;
}
- public void deleteTuple(int pageId, int tupleIndex, RTreeOpContext ctx) throws HyracksDataException {
+ private void deleteTuple(int pageId, int tupleIndex, RTreeOpContext ctx) throws HyracksDataException {
ctx.leafFrame.delete(tupleIndex, cmp);
incrementGlobalNsn();
ctx.leafFrame.setPageLsn(getGlobalNsn());
@@ -933,14 +947,15 @@
}
}
- private void search(ITreeIndexCursor cursor, ISearchPredicate searchPred, RTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
- ctx.reset();
+ private void search(ITreeIndexCursor cursor, ISearchPredicate searchPred, RTreeOpContext ctx)
+ throws HyracksDataException, TreeIndexException {
+ ctx.reset();
ctx.cursor = cursor;
cursor.setBufferCache(bufferCache);
cursor.setFileId(fileId);
ctx.cursorInitialState.setRootPage(rootPage);
- ctx.cursor.open(ctx.cursorInitialState, (SearchPredicate)searchPred);
+ ctx.cursor.open(ctx.cursorInitialState, (SearchPredicate) searchPred);
}
public ITreeIndexFrameFactory getInteriorFrameFactory() {
@@ -969,7 +984,7 @@
public BulkLoadContext(float fillFactor, IRTreeFrame leafFrame, IRTreeFrame interiorFrame,
ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
- indexAccessor = createAccessor();
+ indexAccessor = createAccessor();
}
}
@@ -988,7 +1003,7 @@
@Override
public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException {
try {
- ((BulkLoadContext) ictx).indexAccessor.insert(tuple);
+ ((BulkLoadContext) ictx).indexAccessor.insert(tuple);
} catch (Exception e) {
throw new HyracksDataException("BulkLoad Error");
}
@@ -1036,23 +1051,23 @@
public IndexType getIndexType() {
return IndexType.RTREE;
}
-
+
@Override
- public ITreeIndexAccessor createAccessor() {
- return new RTreeAccessor(this);
- }
-
- private class RTreeAccessor implements ITreeIndexAccessor {
+ public ITreeIndexAccessor createAccessor() {
+ return new RTreeAccessor(this);
+ }
+
+ public class RTreeAccessor implements ITreeIndexAccessor {
private RTree rtree;
private RTreeOpContext ctx;
-
+
public RTreeAccessor(RTree rtree) {
this.rtree = rtree;
this.ctx = rtree.createOpContext();
}
-
+
@Override
- public void insert(ITupleReference tuple) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ public void insert(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
ctx.reset(IndexOp.INSERT);
rtree.insert(tuple, ctx);
}
@@ -1069,6 +1084,13 @@
rtree.delete(tuple, ctx);
}
+ @Override
+ public ITreeIndexCursor createSearchCursor() {
+ return new RTreeSearchCursor(
+ (IRTreeInteriorFrame) interiorFrameFactory.createFrame(),
+ (IRTreeLeafFrame) leafFrameFactory.createFrame());
+ }
+
@Override
public void search(ITreeIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException,
TreeIndexException {
@@ -1077,6 +1099,11 @@
}
@Override
+ public ITreeIndexCursor createDiskOrderScanCursor() {
+ return new TreeDiskOrderScanCursor(leafFrameFactory.createFrame());
+ }
+
+ @Override
public void diskOrderScan(ITreeIndexCursor cursor) throws HyracksDataException {
ctx.reset(IndexOp.DISKORDERSCAN);
rtree.diskOrderScan(cursor, ctx);
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java
index c258377..9c60492 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java
@@ -23,7 +23,7 @@
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
-public final class RTreeOpContext implements IIndexOpContext {
+public class RTreeOpContext implements IIndexOpContext {
public final IRTreeInteriorFrame interiorFrame;
public final IRTreeLeafFrame leafFrame;
public IndexOp op;
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
index e65f1d4..32138db 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
@@ -49,9 +49,6 @@
private ITreeIndexTupleReference frameTuple;
private boolean readLatched = false;
- private int pin = 0;
- private int unpin = 0;
-
public RTreeSearchCursor(IRTreeInteriorFrame interiorFrame, IRTreeLeafFrame leafFrame) {
this.interiorFrame = interiorFrame;
this.leafFrame = leafFrame;
@@ -59,7 +56,7 @@
}
@Override
- public void close() throws Exception {
+ public void close() throws HyracksDataException {
if (readLatched) {
page.releaseReadLatch();
bufferCache.unpin(page);
@@ -80,12 +77,11 @@
return page;
}
- public boolean fetchNextLeafPage() throws HyracksDataException {
+ private boolean fetchNextLeafPage() throws HyracksDataException {
boolean succeed = false;
if (readLatched) {
page.releaseReadLatch();
bufferCache.unpin(page);
- unpin++;
readLatched = false;
}
@@ -94,7 +90,6 @@
long parentLsn = pathList.getLastPageLsn();
pathList.moveLast();
ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- pin++;
node.acquireReadLatch();
readLatched = true;
try {
@@ -112,12 +107,20 @@
}
if (!isLeaf) {
- for (int i = 0; i < interiorFrame.getTupleCount(); i++) {
- int childPageId = interiorFrame.getChildPageIdIfIntersect(searchKey, i, cmp);
- if (childPageId != -1) {
+ if (searchKey != null) {
+ for (int i = 0; i < interiorFrame.getTupleCount(); i++) {
+ int childPageId = interiorFrame.getChildPageIdIfIntersect(searchKey, i, cmp);
+ if (childPageId != -1) {
+ pathList.add(childPageId, pageLsn, -1);
+ }
+ }
+ } else {
+ for (int i = 0; i < interiorFrame.getTupleCount(); i++) {
+ int childPageId = interiorFrame.getChildPageId(i);
pathList.add(childPageId, pageLsn, -1);
}
}
+
} else {
page = node;
leafFrame.setPage(page);
@@ -131,7 +134,6 @@
node.releaseReadLatch();
readLatched = false;
bufferCache.unpin(node);
- unpin++;
}
}
}
@@ -140,7 +142,7 @@
}
@Override
- public boolean hasNext() throws Exception {
+ public boolean hasNext() throws HyracksDataException {
if (page == null) {
return false;
}
@@ -153,7 +155,13 @@
do {
for (int i = tupleIndex; i < leafFrame.getTupleCount(); i++) {
- if (leafFrame.intersect(searchKey, i, cmp)) {
+ if (searchKey != null) {
+ if (leafFrame.intersect(searchKey, i, cmp)) {
+ frameTuple.resetByTupleIndex(leafFrame, i);
+ tupleIndexInc = i + 1;
+ return true;
+ }
+ } else {
frameTuple.resetByTupleIndex(leafFrame, i);
tupleIndexInc = i + 1;
return true;
@@ -164,7 +172,7 @@
}
@Override
- public void next() throws Exception {
+ public void next() throws HyracksDataException {
tupleIndex = tupleIndexInc;
}
@@ -185,17 +193,19 @@
cmp = pred.getLowKeyComparator();
searchKey = pred.getSearchKey();
- int maxFieldPos = cmp.getKeyFieldCount() / 2;
- for (int i = 0; i < maxFieldPos; i++) {
- int j = maxFieldPos + i;
- int c = cmp.getComparators()[i].compare(searchKey.getFieldData(i), searchKey.getFieldStart(i),
- searchKey.getFieldLength(i), searchKey.getFieldData(j), searchKey.getFieldStart(j),
- searchKey.getFieldLength(j));
- if (c > 0) {
- throw new IllegalArgumentException("The low key point has larger coordinates than the high key point.");
+ if (searchKey != null) {
+ int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ int c = cmp.getComparators()[i].compare(searchKey.getFieldData(i), searchKey.getFieldStart(i),
+ searchKey.getFieldLength(i), searchKey.getFieldData(j), searchKey.getFieldStart(j),
+ searchKey.getFieldLength(j));
+ if (c > 0) {
+ throw new IllegalArgumentException("The low key point has larger coordinates than the high key point.");
+ }
}
}
-
+
pathList.add(this.rootPage, -1, -1);
tupleIndex = 0;
fetchNextLeafPage();
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeBulkLoadTest.java
new file mode 100644
index 0000000..f509284
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeBulkLoadTest.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.tests.IOrderedIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexBulkLoadTest;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
+
+@SuppressWarnings("rawtypes")
+public class BTreeBulkLoadTest extends OrderedIndexBulkLoadTest {
+
+ public BTreeBulkLoadTest() {
+ super(BTreeTestHarness.LEAF_FRAMES_TO_TEST);
+ }
+
+ private final BTreeTestHarness harness = new BTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected IOrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType) throws Exception {
+ return BTreeTestContext.create(harness.getBufferCache(),
+ harness.getBTreeFileId(), fieldSerdes, numKeys, leafType);
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeDeleteTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeDeleteTest.java
new file mode 100644
index 0000000..e4475fe
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeDeleteTest.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.tests.IOrderedIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexDeleteTest;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
+
+@SuppressWarnings("rawtypes")
+public class BTreeDeleteTest extends OrderedIndexDeleteTest {
+
+ public BTreeDeleteTest() {
+ super(BTreeTestHarness.LEAF_FRAMES_TO_TEST);
+ }
+
+ private final BTreeTestHarness harness = new BTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected IOrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType) throws Exception {
+ return BTreeTestContext.create(harness.getBufferCache(),
+ harness.getBTreeFileId(), fieldSerdes, numKeys, leafType);
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java
index 5bb86b9..0a22ecc 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java
@@ -15,610 +15,38 @@
package edu.uci.ics.hyracks.storage.am.btree;
-import java.util.Random;
-import java.util.logging.Level;
-
-import org.junit.Test;
+import org.junit.After;
+import org.junit.Before;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
-import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
-import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
-import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
-import edu.uci.ics.hyracks.storage.am.btree.util.AbstractBTreeTest;
+import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexExamplesTest;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
-import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-@SuppressWarnings("rawtypes")
-public class BTreeExamplesTest extends AbstractBTreeTest {
+public class BTreeExamplesTest extends OrderedIndexExamplesTest {
+ private final BTreeTestHarness harness = new BTreeTestHarness();
- /**
- * Fixed-Length Key,Value Example.
- *
- * Create a BTree with one fixed-length key field and one fixed-length value
- * field. Fill BTree with random values using insertions (not bulk load).
- * Perform scans and range search.
- */
- @Test
- public void fixedLengthKeyValueExample() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Fixed-Length Key,Value Example.");
- }
-
- // Declare fields.
- int fieldCount = 2;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = IntegerPointable.TYPE_TRAITS;
- typeTraits[1] = IntegerPointable.TYPE_TRAITS;
- // Declare field serdes.
- ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
-
- // Declare keys.
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
-
- BTree btree = BTreeUtils
- .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
-
- long start = System.currentTimeMillis();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Inserting into tree...");
- }
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- ArrayTupleReference tuple = new ArrayTupleReference();
- ITreeIndexAccessor indexAccessor = btree.createAccessor();
- int numInserts = 10000;
- for (int i = 0; i < numInserts; i++) {
- int f0 = rnd.nextInt() % numInserts;
- int f1 = 5;
- TupleUtils.createIntegerTuple(tb, tuple, f0, f1);
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("Inserting " + i + " : " + f0 + " " + f1);
- }
- }
- try {
- indexAccessor.insert(tuple);
- } catch (TreeIndexException e) {
- }
- }
- long end = System.currentTimeMillis();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
- }
-
- orderedScan(btree, indexAccessor, fieldSerdes);
- diskOrderScan(btree, indexAccessor, fieldSerdes);
-
- // Build low key.
- ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(keyFieldCount);
- ArrayTupleReference lowKey = new ArrayTupleReference();
- TupleUtils.createIntegerTuple(lowKeyTb, lowKey, -1000);
-
- // Build high key.
- ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(keyFieldCount);
- ArrayTupleReference highKey = new ArrayTupleReference();
- TupleUtils.createIntegerTuple(highKeyTb, highKey, 1000);
-
- rangeSearch(btree, indexAccessor, fieldSerdes, lowKey, highKey);
-
- btree.close();
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
}
- /**
- * Composite Key Example (Non-Unique B-Tree).
- *
- * Create a BTree with two fixed-length key fields and one fixed-length
- * value field. Fill BTree with random values using insertions (not bulk
- * load) Perform scans and range search.
- */
- @Test
- public void twoFixedLengthKeysOneFixedLengthValueExample() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Composite Key Test");
- }
-
- // Declare fields.
- int fieldCount = 3;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = IntegerPointable.TYPE_TRAITS;
- typeTraits[1] = IntegerPointable.TYPE_TRAITS;
- typeTraits[2] = IntegerPointable.TYPE_TRAITS;
- // Declare field serdes.
- ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
-
- // declare keys
- int keyFieldCount = 2;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
- cmps[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
-
- BTree btree = BTreeUtils
- .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
-
- long start = System.currentTimeMillis();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Inserting into tree...");
- }
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- ArrayTupleReference tuple = new ArrayTupleReference();
- ITreeIndexAccessor indexAccessor = btree.createAccessor();
- int numInserts = 10000;
- for (int i = 0; i < 10000; i++) {
- int f0 = rnd.nextInt() % 2000;
- int f1 = rnd.nextInt() % 1000;
- int f2 = 5;
- TupleUtils.createIntegerTuple(tb, tuple, f0, f1, f2);
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("Inserting " + i + " : " + f0 + " " + f1 + " " + f2);
- }
- }
- try {
- indexAccessor.insert(tuple);
- } catch (TreeIndexException e) {
- }
- }
- long end = System.currentTimeMillis();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
- }
-
- orderedScan(btree, indexAccessor, fieldSerdes);
- diskOrderScan(btree, indexAccessor, fieldSerdes);
-
- // Build low key.
- ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
- ArrayTupleReference lowKey = new ArrayTupleReference();
- TupleUtils.createIntegerTuple(lowKeyTb, lowKey, -3);
-
- // Build high key.
- ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
- ArrayTupleReference highKey = new ArrayTupleReference();
- TupleUtils.createIntegerTuple(highKeyTb, highKey, 3);
-
- // Prefix-Range search in [-3, 3]
- rangeSearch(btree, indexAccessor, fieldSerdes, lowKey, highKey);
-
- btree.close();
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
}
-
- /**
- * Variable-Length Example. Create a BTree with one variable-length key
- * field and one variable-length value field. Fill BTree with random values
- * using insertions (not bulk load) Perform ordered scans and range search.
- */
- @Test
- public void varLenKeyValueExample() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Variable-Length Key,Value Example");
- }
-
- // Declare fields.
- int fieldCount = 2;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
- typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
- // Declare field serdes.
- ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
-
- // Declare keys.
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY).createBinaryComparator();
-
- BTree btree = BTreeUtils
- .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
-
- long start = System.currentTimeMillis();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Inserting into tree...");
- }
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- ArrayTupleReference tuple = new ArrayTupleReference();
- ITreeIndexAccessor indexAccessor = btree.createAccessor();
- // Max string length to be generated.
- int maxLength = 10;
- int numInserts = 10000;
- for (int i = 0; i < 10000; i++) {
- String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
- String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
- TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("Inserting " + f0 + " " + f1);
- }
- }
- try {
- indexAccessor.insert(tuple);
- } catch (TreeIndexException e) {
- }
- }
- long end = System.currentTimeMillis();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
- }
-
- orderedScan(btree, indexAccessor, fieldSerdes);
- diskOrderScan(btree, indexAccessor, fieldSerdes);
-
- // Build low key.
- ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
- ArrayTupleReference lowKey = new ArrayTupleReference();
- TupleUtils.createTuple(lowKeyTb, lowKey, fieldSerdes, "cbf");
-
- // Build high key.
- ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
- ArrayTupleReference highKey = new ArrayTupleReference();
- TupleUtils.createTuple(highKeyTb, highKey, fieldSerdes, "cc7");
-
- rangeSearch(btree, indexAccessor, fieldSerdes, lowKey, highKey);
-
- btree.close();
+
+ protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparator[] cmps) throws TreeIndexException {
+ return BTreeUtils.createBTree(harness.getBufferCache(), harness.getBTreeFileId(), typeTraits, cmps,
+ BTreeLeafFrameType.REGULAR_NSM);
}
-
- /**
- * Deletion Example.
- *
- * Create a BTree with one variable-length key field and one variable-length
- * value field. Fill B-tree with random values using insertions, then delete
- * entries one-by-one. Repeat procedure a few times on same BTree.
- */
- @Test
- public void deleteExample() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Deletion Example");
- }
-
- // Declare fields.
- int fieldCount = 2;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
- typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
- // Declare field serdes.
- ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
-
- // Declare keys.
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY).createBinaryComparator();
-
- BTree btree = BTreeUtils
- .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
-
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- ArrayTupleReference tuple = new ArrayTupleReference();
- ITreeIndexAccessor indexAccessor = btree.createAccessor();
- // Max string length to be generated.
- int runs = 3;
- for (int run = 0; run < runs; run++) {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Deletion example run: " + (run + 1) + "/" + runs);
- LOGGER.info("Inserting into tree...");
- }
- int maxLength = 10;
- int ins = 10000;
- String[] f0s = new String[ins];
- String[] f1s = new String[ins];
- int insDone = 0;
- int[] insDoneCmp = new int[ins];
- for (int i = 0; i < ins; i++) {
- String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
- String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
- TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
- f0s[i] = f0;
- f1s[i] = f1;
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("Inserting " + i);
- }
- }
- try {
- indexAccessor.insert(tuple);
- insDone++;
- } catch (TreeIndexException e) {
- }
- insDoneCmp[i] = insDone;
- }
-
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Deleting from tree...");
- }
- int delDone = 0;
- for (int i = 0; i < ins; i++) {
- TupleUtils.createTuple(tb, tuple, fieldSerdes, f0s[i], f1s[i]);
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("Deleting " + i);
- }
- }
- try {
- indexAccessor.delete(tuple);
- delDone++;
- } catch (TreeIndexException e) {
- }
- if (insDoneCmp[i] != delDone) {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("INCONSISTENT STATE, ERROR IN DELETION EXAMPLE.");
- LOGGER.info("INSDONECMP: " + insDoneCmp[i] + " " + delDone);
- }
- break;
- }
- }
- if (insDone != delDone) {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("ERROR! INSDONE: " + insDone + " DELDONE: " + delDone);
- }
- break;
- }
- }
- btree.close();
+
+ protected int getIndexFileId() {
+ return harness.getBTreeFileId();
}
-
- /**
- * Update example.
- *
- * Create a BTree with one variable-length key field and one variable-length
- * value field. Fill B-tree with random values using insertions, then update
- * entries one-by-one. Repeat procedure a few times on same BTree.
- */
- @Test
- public void updateExample() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Update example");
- }
-
- // Declare fields.
- int fieldCount = 2;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
- typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
- // Declare field serdes.
- ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
-
- // Declare keys.
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY).createBinaryComparator();
-
- BTree btree = BTreeUtils
- .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
-
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Inserting into tree...");
- }
- ITreeIndexAccessor indexAccessor = btree.createAccessor();
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- ArrayTupleReference tuple = new ArrayTupleReference();
- int maxLength = 10;
- int ins = 10000;
- String[] keys = new String[10000];
- for (int i = 0; i < ins; i++) {
- String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
- String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
- TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
- keys[i] = f0;
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("Inserting " + i);
- }
- }
- try {
- indexAccessor.insert(tuple);
- } catch (TreeIndexException e) {
- }
- }
- // Print before doing any updates.
- orderedScan(btree, indexAccessor, fieldSerdes);
-
- int runs = 3;
- for (int run = 0; run < runs; run++) {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Update test run: " + (run + 1) + "/" + runs);
- LOGGER.info("Updating BTree");
- }
- for (int i = 0; i < ins; i++) {
- // Generate a new random value for f1.
- String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
- TupleUtils.createTuple(tb, tuple, fieldSerdes, keys[i], f1);
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("UPDATING " + i);
- }
- }
- try {
- indexAccessor.update(tuple);
- } catch (TreeIndexException e) {
- }
- }
- // Do another scan after a round of updates.
- orderedScan(btree, indexAccessor, fieldSerdes);
- }
- btree.close();
- }
-
- /**
- * Bulk load example.
- *
- * Load a tree with 100,000 tuples. BTree has a composite key to "simulate"
- * non-unique index creation.
- *
- */
- @Test
- public void bulkLoadExample() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Bulk load example");
- }
- // Declare fields.
- int fieldCount = 3;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = IntegerPointable.TYPE_TRAITS;
- typeTraits[1] = IntegerPointable.TYPE_TRAITS;
- typeTraits[2] = IntegerPointable.TYPE_TRAITS;
- // Declare field serdes.
- ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
-
- // declare keys
- int keyFieldCount = 2;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
- cmps[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
-
- BTree btree = BTreeUtils
- .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
-
- // Load sorted records.
- int ins = 100000;
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Bulk loading " + ins + " tuples");
- }
- long start = System.currentTimeMillis();
- IIndexBulkLoadContext bulkLoadCtx = btree.beginBulkLoad(0.7f);
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- ArrayTupleReference tuple = new ArrayTupleReference();
- for (int i = 0; i < ins; i++) {
- TupleUtils.createIntegerTuple(tb, tuple, i, i, 5);
- btree.bulkLoadAddTuple(tuple, bulkLoadCtx);
- }
- btree.endBulkLoad(bulkLoadCtx);
- long end = System.currentTimeMillis();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(ins + " tuples loaded in " + (end - start) + "ms");
- }
-
- ITreeIndexAccessor indexAccessor = btree.createAccessor();
-
- // Build low key.
- ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
- ArrayTupleReference lowKey = new ArrayTupleReference();
- TupleUtils.createIntegerTuple(lowKeyTb, lowKey, 44444);
-
- // Build high key.
- ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
- ArrayTupleReference highKey = new ArrayTupleReference();
- TupleUtils.createIntegerTuple(highKeyTb, highKey, 44500);
-
- // Prefix-Range search in [44444, 44500]
- rangeSearch(btree, indexAccessor, fieldSerdes, lowKey, highKey);
-
- btree.close();
- }
-
- private void orderedScan(BTree btree, ITreeIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes)
- throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Ordered Scan:");
- }
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
- ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(leafFrame, false);
- RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
- indexAccessor.search(scanCursor, nullPred);
- try {
- while (scanCursor.hasNext()) {
- scanCursor.next();
- ITupleReference frameTuple = scanCursor.getTuple();
- String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(rec);
- }
- }
- } finally {
- scanCursor.close();
- }
- }
-
- private void diskOrderScan(BTree btree, ITreeIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes)
- throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Disk-Order Scan:");
- }
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
- TreeDiskOrderScanCursor diskOrderCursor = new TreeDiskOrderScanCursor(leafFrame);
- indexAccessor.diskOrderScan(diskOrderCursor);
- try {
- while (diskOrderCursor.hasNext()) {
- diskOrderCursor.next();
- ITupleReference frameTuple = diskOrderCursor.getTuple();
- String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(rec);
- }
- }
- } finally {
- diskOrderCursor.close();
- }
- }
-
- private void rangeSearch(BTree btree, ITreeIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes,
- ITupleReference lowKey, ITupleReference highKey) throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- String lowKeyString = TupleUtils.printTuple(lowKey, fieldSerdes);
- String highKeyString = TupleUtils.printTuple(highKey, fieldSerdes);
- LOGGER.info("Range-Search in: [" + lowKeyString + ", " + highKeyString + "]");
- }
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
- MultiComparator lowKeySearchCmp = BTreeUtils.getSearchMultiComparator(btree.getMultiComparator(), lowKey);
- MultiComparator highKeySearchCmp = BTreeUtils.getSearchMultiComparator(btree.getMultiComparator(), highKey);
- ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor(leafFrame, false);
- RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, true, true, lowKeySearchCmp,
- highKeySearchCmp);
- indexAccessor.search(rangeCursor, rangePred);
- try {
- while (rangeCursor.hasNext()) {
- rangeCursor.next();
- ITupleReference frameTuple = rangeCursor.getTuple();
- String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(rec);
- }
- }
- } finally {
- rangeCursor.close();
- }
- }
-
- public static String randomString(int length, Random random) {
- String s = Long.toHexString(Double.doubleToLongBits(random.nextDouble()));
- StringBuilder strBuilder = new StringBuilder();
- for (int i = 0; i < s.length() && i < length; i++) {
- strBuilder.append(s.charAt(Math.abs(random.nextInt()) % s.length()));
- }
- return strBuilder.toString();
- }
-}
\ No newline at end of file
+}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeInsertTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeInsertTest.java
new file mode 100644
index 0000000..a4d9570
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeInsertTest.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.tests.IOrderedIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexInsertTest;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
+
+/**
+ * Tests the BTree insert operation with strings and integer fields using
+ * various numbers of key and payload fields.
+ *
+ * Each tests first fills a BTree with randomly generated tuples. We compare the
+ * following operations against expected results: 1. Point searches for all
+ * tuples. 2. Ordered scan. 3. Disk-order scan. 4. Range search (and prefix
+ * search for composite keys).
+ *
+ */
+@SuppressWarnings("rawtypes")
+public class BTreeInsertTest extends OrderedIndexInsertTest {
+
+ public BTreeInsertTest() {
+ super(BTreeTestHarness.LEAF_FRAMES_TO_TEST);
+ }
+
+ private final BTreeTestHarness harness = new BTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected IOrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType) throws Exception {
+ return BTreeTestContext.create(harness.getBufferCache(),
+ harness.getBTreeFileId(), fieldSerdes, numKeys, leafType);
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/RangeSearchCursorTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeSearchCursorTest.java
similarity index 97%
rename from hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/RangeSearchCursorTest.java
rename to hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeSearchCursorTest.java
index 4031e7f..764cbb6 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/RangeSearchCursorTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeSearchCursorTest.java
@@ -58,8 +58,9 @@
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.am.common.util.IndexUtils;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-public class RangeSearchCursorTest extends AbstractBTreeTest {
+public class BTreeSearchCursorTest extends AbstractBTreeTest {
// Declare fields
int fieldCount = 2;
ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
@@ -83,6 +84,9 @@
LOGGER.info("TESTING RANGE SEARCH CURSOR ON UNIQUE INDEX");
}
+ IBufferCache bufferCache = harness.getBufferCache();
+ int btreeFileId = harness.getBTreeFileId();
+
// declare keys
int keyFieldCount = 1;
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
@@ -160,6 +164,9 @@
LOGGER.info("TESTING RANGE SEARCH CURSOR ON NONUNIQUE INDEX");
}
+ IBufferCache bufferCache = harness.getBufferCache();
+ int btreeFileId = harness.getBTreeFileId();
+
// declare keys
int keyFieldCount = 2;
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
@@ -235,6 +242,9 @@
LOGGER.info("TESTING RANGE SEARCH CURSOR ON NONUNIQUE FIELD-PREFIX COMPRESSED INDEX");
}
+ IBufferCache bufferCache = harness.getBufferCache();
+ int btreeFileId = harness.getBTreeFileId();
+
// declare keys
int keyFieldCount = 2;
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StatsTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java
similarity index 96%
rename from hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StatsTest.java
rename to hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java
index 1ae80d6..e1b19ef 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StatsTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java
@@ -47,11 +47,8 @@
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
import edu.uci.ics.hyracks.test.support.TestUtils;
-public class StatsTest extends AbstractBTreeTest {
-
- // private static final int PAGE_SIZE = 256;
- // private static final int NUM_PAGES = 10;
- // private static final int PAGE_SIZE = 32768;
+@SuppressWarnings("rawtypes")
+public class BTreeStatsTest extends AbstractBTreeTest {
private static final int PAGE_SIZE = 4096;
private static final int NUM_PAGES = 1000;
private static final int MAX_OPEN_FILES = 10;
@@ -64,7 +61,7 @@
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
+ FileReference file = new FileReference(new File(harness.getFileName()));
bufferCache.createFile(file);
int fileId = fmp.lookupFileId(file);
bufferCache.openFile(fileId);
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateSearchTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
similarity index 95%
rename from hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateSearchTest.java
rename to hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
index 5d43ae1..f1059dd 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateSearchTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
@@ -33,12 +33,17 @@
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.am.common.util.IndexUtils;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-public class UpdateSearchTest extends AbstractBTreeTest {
+@SuppressWarnings("rawtypes")
+public class BTreeUpdateSearchTest extends AbstractBTreeTest {
// Update scan test on fixed-length tuples.
@Test
public void test01() throws Exception {
+ IBufferCache bufferCache = harness.getBufferCache();
+ int btreeFileId = harness.getBTreeFileId();
+
// declare fields
int fieldCount = 2;
ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
@@ -81,7 +86,7 @@
ITreeIndexAccessor indexAccessor = btree.createAccessor();
int numInserts = 10000;
- for (int i = 0; i < 10000; i++) {
+ for (int i = 0; i < numInserts; i++) {
int f0 = rnd.nextInt() % 10000;
int f1 = 5;
TupleUtils.createIntegerTuple(tb, insertTuple, f0, f1);
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateTest.java
new file mode 100644
index 0000000..14f8d18
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateTest.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.tests.IOrderedIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexUpdateTest;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
+
+@SuppressWarnings("rawtypes")
+public class BTreeUpdateTest extends OrderedIndexUpdateTest {
+
+ public BTreeUpdateTest() {
+ super(BTreeTestHarness.LEAF_FRAMES_TO_TEST);
+ }
+
+ private final BTreeTestHarness harness = new BTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected IOrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType) throws Exception {
+ return BTreeTestContext.create(harness.getBufferCache(),
+ harness.getBTreeFileId(), fieldSerdes, numKeys, leafType);
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BulkLoadTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BulkLoadTest.java
deleted file mode 100644
index 49d08a3..0000000
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BulkLoadTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.btree;
-
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
-import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
-import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestUtils;
-
-@SuppressWarnings("rawtypes")
-public class BulkLoadTest extends BTreeTestDriver {
-
- @Override
- protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception {
- BTreeTestContext testCtx = BTreeTestUtils.createBTreeTestContext(bufferCache, btreeFileId, fieldSerdes, numKeys, leafType);
-
- // We assume all fieldSerdes are of the same type. Check the first one to determine which field types to generate.
- if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
- BTreeTestUtils.bulkLoadIntTuples(testCtx, numTuplesToInsert, rnd);
- } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
- BTreeTestUtils.bulkLoadStringTuples(testCtx, numTuplesToInsert, rnd);
- }
-
- BTreeTestUtils.checkPointSearches(testCtx);
- BTreeTestUtils.checkOrderedScan(testCtx);
- BTreeTestUtils.checkDiskOrderScan(testCtx);
- BTreeTestUtils.checkRangeSearch(testCtx, lowKey, highKey, true, true);
- if (prefixLowKey != null && prefixHighKey != null) {
- BTreeTestUtils.checkRangeSearch(testCtx, prefixLowKey, prefixHighKey, true, true);
- }
- }
-
- @Override
- protected String getTestOpName() {
- return "BulkLoad";
- }
-}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/DeleteTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/DeleteTest.java
deleted file mode 100644
index 2157adb..0000000
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/DeleteTest.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.btree;
-
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
-import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
-import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestUtils;
-
-@SuppressWarnings("rawtypes")
-public class DeleteTest extends BTreeTestDriver {
-
- private static final int numInsertRounds = 3;
- private static final int numDeleteRounds = 3;
-
- @Override
- protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception {
- BTreeTestContext testCtx = BTreeTestUtils.createBTreeTestContext(bufferCache, btreeFileId, fieldSerdes, numKeys, leafType);
- for (int i = 0; i < numInsertRounds; i++) {
-
- // We assume all fieldSerdes are of the same type. Check the first one to determine which field types to generate.
- if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
- BTreeTestUtils.insertIntTuples(testCtx, numTuplesToInsert, rnd);
- } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
- BTreeTestUtils.insertStringTuples(testCtx, numTuplesToInsert, rnd);
- }
-
- int numTuplesPerDeleteRound = (int)Math.ceil((float)testCtx.checkTuples.size() / (float)numDeleteRounds);
- for(int j = 0; j < numDeleteRounds; j++) {
- BTreeTestUtils.deleteTuples(testCtx, numTuplesPerDeleteRound, rnd);
-
- BTreeTestUtils.checkPointSearches(testCtx);
- BTreeTestUtils.checkOrderedScan(testCtx);
- BTreeTestUtils.checkDiskOrderScan(testCtx);
- BTreeTestUtils.checkRangeSearch(testCtx, lowKey, highKey, true, true);
- if (prefixLowKey != null && prefixHighKey != null) {
- BTreeTestUtils.checkRangeSearch(testCtx, prefixLowKey, prefixHighKey, true, true);
- }
- }
- }
- }
-
- @Override
- protected String getTestOpName() {
- return "Delete";
- }
-}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java
index e8aa00a..d61d16a 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java
@@ -45,9 +45,11 @@
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexUtils;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+@SuppressWarnings("rawtypes")
public class FieldPrefixNSMTest extends AbstractBTreeTest {
private static final int PAGE_SIZE = 32768; // 32K
@@ -55,6 +57,10 @@
private static final int MAX_OPEN_FILES = 10;
private static final int HYRACKS_FRAME_SIZE = 128;
+ public FieldPrefixNSMTest() {
+ super(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES, HYRACKS_FRAME_SIZE);
+ }
+
private ITupleReference createTuple(IHyracksTaskContext ctx, int f0, int f1, int f2, boolean print)
throws HyracksDataException {
if (print) {
@@ -67,7 +73,7 @@
FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
ArrayTupleBuilder tb = new ArrayTupleBuilder(3);
DataOutput dos = tb.getDataOutput();
-
+
ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
@@ -92,8 +98,8 @@
}
@Test
- public void test01() throws Exception {
-
+ public void test01() throws Exception {
+
// declare fields
int fieldCount = 3;
ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
@@ -116,6 +122,9 @@
Random rnd = new Random();
rnd.setSeed(50);
+ IBufferCache bufferCache = harness.getBufferCache();
+ int btreeFileId = harness.getBTreeFileId();
+ IHyracksTaskContext ctx = harness.getHyracksTaskContext();
ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(btreeFileId, 0), false);
try {
@@ -213,20 +222,4 @@
bufferCache.unpin(page);
}
}
-
- public int getPageSize() {
- return PAGE_SIZE;
- }
-
- public int getNumPages() {
- return NUM_PAGES;
- }
-
- public int getHyracksFrameSize() {
- return HYRACKS_FRAME_SIZE;
- }
-
- public int getMaxOpenFiles() {
- return MAX_OPEN_FILES;
- }
}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/InsertTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/InsertTest.java
deleted file mode 100644
index fa3f895..0000000
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/InsertTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.btree;
-
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
-import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
-import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestUtils;
-
-/**
- * Tests the BTree insert operation with strings and integer fields using
- * various numbers of key and payload fields.
- *
- * Each tests first fills a BTree with randomly generated tuples.
- * We compare the following operations against expected results:
- * 1. Point searches for all tuples.
- * 2. Ordered scan.
- * 3. Disk-order scan.
- * 4. Range search (and prefix search for composite keys).
- *
- */
-@SuppressWarnings("rawtypes")
-public class InsertTest extends BTreeTestDriver {
- @Override
- protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception {
- BTreeTestContext testCtx = BTreeTestUtils.createBTreeTestContext(bufferCache, btreeFileId, fieldSerdes, numKeys, leafType);
- // We assume all fieldSerdes are of the same type. Check the first one to determine which field types to generate.
- if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
- BTreeTestUtils.insertIntTuples(testCtx, numTuplesToInsert, rnd);
- } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
- BTreeTestUtils.insertStringTuples(testCtx, numTuplesToInsert, rnd);
- }
-
- BTreeTestUtils.checkPointSearches(testCtx);
- BTreeTestUtils.checkOrderedScan(testCtx);
- BTreeTestUtils.checkDiskOrderScan(testCtx);
-
- BTreeTestUtils.checkRangeSearch(testCtx, lowKey, highKey, true, true);
- if (prefixLowKey != null && prefixHighKey != null) {
- BTreeTestUtils.checkRangeSearch(testCtx, prefixLowKey, prefixHighKey, true, true);
- }
- testCtx.btree.close();
- }
-
- @Override
- protected String getTestOpName() {
- return "Insert";
- }
-}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java
index ac4133d..9ec64b5 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java
@@ -15,7 +15,6 @@
package edu.uci.ics.hyracks.storage.am.btree;
-import java.io.File;
import java.util.LinkedList;
import java.util.List;
import java.util.Random;
@@ -24,21 +23,13 @@
import org.junit.Test;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.btree.util.AbstractBTreeTest;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
import edu.uci.ics.hyracks.storage.common.sync.LatchType;
-import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
-public class StorageManagerTest extends AbstractBTreeTest {
- private static final int PAGE_SIZE = 256;
- private static final int NUM_PAGES = 10;
- private static final int MAX_OPEN_FILES = 10;
- private static final int HYRACKS_FRAME_SIZE = 32768;
-
+public class StorageManagerTest extends AbstractBTreeTest {
public class PinnedLatchedPage {
public final ICachedPage page;
public final LatchType latch;
@@ -258,38 +249,11 @@
}
@Test
- public void oneThreadOneFileTest() throws Exception {
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- Thread worker = new Thread(new FileAccessWorker(0, bufferCache, FileAccessType.FTA_UNLATCHED, fileId, 10, 10,
- 100, 10, 0));
-
+ public void oneThreadOneFileTest() throws Exception {
+ Thread worker = new Thread(new FileAccessWorker(0,
+ harness.getBufferCache(), FileAccessType.FTA_UNLATCHED,
+ harness.getBTreeFileId(), 10, 10, 100, 10, 0));
worker.start();
-
worker.join();
-
- bufferCache.close();
- }
-
- public int getPageSize() {
- return PAGE_SIZE;
- }
-
- public int getNumPages() {
- return NUM_PAGES;
- }
-
- public int getHyracksFrameSize() {
- return HYRACKS_FRAME_SIZE;
- }
-
- public int getMaxOpenFiles() {
- return MAX_OPEN_FILES;
}
}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateTest.java
deleted file mode 100644
index b40539f..0000000
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.btree;
-
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
-import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
-import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestUtils;
-
-@SuppressWarnings("rawtypes")
-public class UpdateTest extends BTreeTestDriver {
- private static final int numUpdateRounds = 3;
-
- @Override
- protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception {
- // This is a noop because we can only update non-key fields.
- if (fieldSerdes.length == numKeys) {
- return;
- }
-
- BTreeTestContext testCtx = BTreeTestUtils.createBTreeTestContext(bufferCache, btreeFileId, fieldSerdes, numKeys, leafType);
-
- // We assume all fieldSerdes are of the same type. Check the first one to determine which field types to generate.
- if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
- BTreeTestUtils.insertIntTuples(testCtx, numTuplesToInsert, rnd);
- } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
- BTreeTestUtils.insertStringTuples(testCtx, numTuplesToInsert, rnd);
- }
-
- int numTuplesPerDeleteRound = (int)Math.ceil((float)testCtx.checkTuples.size() / (float)numUpdateRounds);
- for(int j = 0; j < numUpdateRounds; j++) {
- BTreeTestUtils.updateTuples(testCtx, numTuplesPerDeleteRound, rnd);
-
- BTreeTestUtils.checkPointSearches(testCtx);
- BTreeTestUtils.checkOrderedScan(testCtx);
- BTreeTestUtils.checkDiskOrderScan(testCtx);
- BTreeTestUtils.checkRangeSearch(testCtx, lowKey, highKey, true, true);
- if (prefixLowKey != null && prefixHighKey != null) {
- BTreeTestUtils.checkRangeSearch(testCtx, prefixLowKey, prefixHighKey, true, true);
- }
- }
- }
-
- @Override
- protected String getTestOpName() {
- return "Update";
- }
-}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/AbstractBTreeTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/AbstractBTreeTest.java
index 9630a1b..f4eca1b 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/AbstractBTreeTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/AbstractBTreeTest.java
@@ -1,76 +1,46 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.btree.util;
-import java.io.File;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.Random;
import java.util.logging.Logger;
import org.junit.After;
import org.junit.Before;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
-import edu.uci.ics.hyracks.test.support.TestUtils;
public abstract class AbstractBTreeTest {
- protected static final Logger LOGGER = Logger.getLogger(AbstractBTreeTest.class.getName());
- public static final long RANDOM_SEED = 50;
-
- private static final int PAGE_SIZE = 256;
- private static final int NUM_PAGES = 10;
- private static final int MAX_OPEN_FILES = 10;
- private static final int HYRACKS_FRAME_SIZE = 128;
-
- protected IHyracksTaskContext ctx;
- protected IBufferCache bufferCache;
- protected int btreeFileId;
-
- protected final Random rnd = new Random();
- protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
- protected final static String tmpDir = System.getProperty("java.io.tmpdir");
- protected final static String sep = System.getProperty("file.separator");
- protected String fileName;
-
- @Before
- public void setUp() throws HyracksDataException {
- fileName = tmpDir + sep + simpleDateFormat.format(new Date());
- ctx = TestUtils.create(getHyracksFrameSize());
- TestStorageManagerComponentHolder.init(getPageSize(), getNumPages(), getMaxOpenFiles());
- bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- btreeFileId = fmp.lookupFileId(file);
- bufferCache.openFile(btreeFileId);
- rnd.setSeed(RANDOM_SEED);
+ protected final Logger LOGGER = Logger.getLogger(BTreeTestHarness.class.getName());
+ protected final BTreeTestHarness harness;
+
+ public AbstractBTreeTest() {
+ harness = new BTreeTestHarness();
}
- @After
+ public AbstractBTreeTest(int pageSize, int numPages, int maxOpenFiles, int hyracksFrameSize) {
+ harness = new BTreeTestHarness(pageSize, numPages, maxOpenFiles, hyracksFrameSize);
+ }
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
public void tearDown() throws HyracksDataException {
- bufferCache.closeFile(btreeFileId);
- bufferCache.close();
- File f = new File(fileName);
- f.deleteOnExit();
- }
-
- public int getPageSize() {
- return PAGE_SIZE;
- }
-
- public int getNumPages() {
- return NUM_PAGES;
- }
-
- public int getHyracksFrameSize() {
- return HYRACKS_FRAME_SIZE;
- }
-
- public int getMaxOpenFiles() {
- return MAX_OPEN_FILES;
+ harness.tearDown();
}
}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestContext.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestContext.java
index f1b03c1..c454db3 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestContext.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestContext.java
@@ -15,49 +15,42 @@
package edu.uci.ics.hyracks.storage.am.btree.util;
-import java.util.TreeSet;
-
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
@SuppressWarnings("rawtypes")
-public final class BTreeTestContext {
- public final ISerializerDeserializer[] fieldSerdes;
- public final IBufferCache bufferCache;
- public final BTree btree;
- public final IBTreeLeafFrame leafFrame;
- public final IBTreeInteriorFrame interiorFrame;
- public final ITreeIndexMetaDataFrame metaFrame;
- public final ArrayTupleBuilder tupleBuilder;
- public final ArrayTupleReference tuple = new ArrayTupleReference();
- public final TreeSet<CheckTuple> checkTuples = new TreeSet<CheckTuple>();
- public final ITreeIndexAccessor indexAccessor;
-
- public BTreeTestContext(IBufferCache bufferCache, ISerializerDeserializer[] fieldSerdes, BTree btree,
- IBTreeLeafFrame leafFrame, IBTreeInteriorFrame interiorFrame, ITreeIndexMetaDataFrame metaFrame,
- ITreeIndexAccessor indexAccessor) {
- this.bufferCache = bufferCache;
- this.fieldSerdes = fieldSerdes;
- this.btree = btree;
- this.leafFrame = leafFrame;
- this.interiorFrame = interiorFrame;
- this.metaFrame = metaFrame;
- this.indexAccessor = indexAccessor;
- this.tupleBuilder = new ArrayTupleBuilder(fieldSerdes.length);
+public class BTreeTestContext extends OrderedIndexTestContext {
+
+ public BTreeTestContext(ISerializerDeserializer[] fieldSerdes, ITreeIndex treeIndex) {
+ super(fieldSerdes, treeIndex);
}
- public int getFieldCount() {
- return fieldSerdes.length;
- }
-
+ @Override
public int getKeyFieldCount() {
+ BTree btree = (BTree) treeIndex;
return btree.getMultiComparator().getKeyFieldCount();
}
-}
\ No newline at end of file
+
+ @Override
+ public IBinaryComparator[] getComparators() {
+ BTree btree = (BTree) treeIndex;
+ return btree.getMultiComparator().getComparators();
+ }
+
+ public static BTreeTestContext create(IBufferCache bufferCache, int btreeFileId, ISerializerDeserializer[] fieldSerdes, int numKeyFields, BTreeLeafFrameType leafType) throws Exception {
+ ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
+ IBinaryComparator[] cmps = SerdeUtils.serdesToComparators(fieldSerdes, numKeyFields);
+ BTree btree = BTreeUtils.createBTree(bufferCache, btreeFileId, typeTraits, cmps, leafType);
+ btree.create(btreeFileId);
+ btree.open(btreeFileId);
+ BTreeTestContext testCtx = new BTreeTestContext(fieldSerdes, btree);
+ return testCtx;
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestHarness.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestHarness.java
new file mode 100644
index 0000000..3bf128e
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestHarness.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.util;
+
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Random;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
+import edu.uci.ics.hyracks.test.support.TestUtils;
+
+public class BTreeTestHarness {
+ public static final BTreeLeafFrameType[] LEAF_FRAMES_TO_TEST = new BTreeLeafFrameType[] {
+ BTreeLeafFrameType.REGULAR_NSM, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM };
+
+ private static final long RANDOM_SEED = 50;
+ private static final int DEFAULT_PAGE_SIZE = 256;
+ private static final int DEFAULT_NUM_PAGES = 10;
+ private static final int DEFAULT_MAX_OPEN_FILES = 10;
+ private static final int DEFAULT_HYRACKS_FRAME_SIZE = 128;
+
+ protected final int pageSize;
+ protected final int numPages;
+ protected final int maxOpenFiles;
+ protected final int hyracksFrameSize;
+
+ protected IHyracksTaskContext ctx;
+ protected IBufferCache bufferCache;
+ protected int btreeFileId;
+
+ protected final Random rnd = new Random();
+ protected final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ protected final String tmpDir = System.getProperty("java.io.tmpdir");
+ protected final String sep = System.getProperty("file.separator");
+ protected String fileName;
+
+ public BTreeTestHarness() {
+ this.pageSize = DEFAULT_PAGE_SIZE;
+ this.numPages = DEFAULT_NUM_PAGES;
+ this.maxOpenFiles = DEFAULT_MAX_OPEN_FILES;
+ this.hyracksFrameSize = DEFAULT_HYRACKS_FRAME_SIZE;
+ }
+
+ public BTreeTestHarness(int pageSize, int numPages, int maxOpenFiles, int hyracksFrameSize) {
+ this.pageSize = pageSize;
+ this.numPages = numPages;
+ this.maxOpenFiles = maxOpenFiles;
+ this.hyracksFrameSize = hyracksFrameSize;
+ }
+
+ public void setUp() throws HyracksDataException {
+ fileName = tmpDir + sep + simpleDateFormat.format(new Date());
+ ctx = TestUtils.create(getHyracksFrameSize());
+ TestStorageManagerComponentHolder.init(pageSize, numPages, maxOpenFiles);
+ bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ FileReference file = new FileReference(new File(fileName));
+ bufferCache.createFile(file);
+ btreeFileId = fmp.lookupFileId(file);
+ bufferCache.openFile(btreeFileId);
+ rnd.setSeed(RANDOM_SEED);
+ }
+
+ public void tearDown() throws HyracksDataException {
+ bufferCache.closeFile(btreeFileId);
+ bufferCache.close();
+ File f = new File(fileName);
+ f.deleteOnExit();
+ }
+
+ public IHyracksTaskContext getHyracksTaskContext() {
+ return ctx;
+ }
+
+ public IBufferCache getBufferCache() {
+ return bufferCache;
+ }
+
+ public int getBTreeFileId() {
+ return btreeFileId;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public Random getRandom() {
+ return rnd;
+ }
+
+ public int getPageSize() {
+ return pageSize;
+ }
+
+ public int getNumPages() {
+ return numPages;
+ }
+
+ public int getHyracksFrameSize() {
+ return hyracksFrameSize;
+ }
+
+ public int getMaxOpenFiles() {
+ return maxOpenFiles;
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestUtils.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestUtils.java
deleted file mode 100644
index b5186b0..0000000
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestUtils.java
+++ /dev/null
@@ -1,505 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.btree.util;
-
-import static org.junit.Assert.fail;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.util.Iterator;
-import java.util.NavigableSet;
-import java.util.Random;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
-import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeDuplicateKeyException;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
-import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
-import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
-import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-
-@SuppressWarnings("rawtypes")
-public class BTreeTestUtils {
- private static final Logger LOGGER = Logger.getLogger(BTreeTestUtils.class.getName());
-
- public static BTreeTestContext createBTreeTestContext(IBufferCache bufferCache, int btreeFileId, ISerializerDeserializer[] fieldSerdes, int numKeyFields, BTreeLeafFrameType leafType) throws Exception {
- ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes, fieldSerdes.length);
- IBinaryComparator[] cmps = SerdeUtils.serdesToComparators(fieldSerdes, numKeyFields);
-
- BTree btree = BTreeUtils.createBTree(bufferCache, btreeFileId, typeTraits, cmps, leafType);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
- ITreeIndexAccessor indexAccessor = btree.createAccessor();
-
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
- IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) btree.getInteriorFrameFactory().createFrame();
- ITreeIndexMetaDataFrame metaFrame = btree.getFreePageManager().getMetaDataFrameFactory().createFrame();
- BTreeTestContext testCtx = new BTreeTestContext(bufferCache, fieldSerdes, btree, leafFrame, interiorFrame, metaFrame, indexAccessor);
- return testCtx;
- }
-
- private static void compareActualAndExpected(ITupleReference actual, CheckTuple expected, ISerializerDeserializer[] fieldSerdes) throws HyracksDataException {
- for (int i = 0; i < fieldSerdes.length; i++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(
- actual.getFieldData(i), actual.getFieldStart(i),
- actual.getFieldLength(i));
- DataInput dataIn = new DataInputStream(inStream);
- Object actualObj = fieldSerdes[i].deserialize(dataIn);
- if (!actualObj.equals(expected.get(i))) {
- fail("Actual and expected fields do not match.\nExpected: " + expected.get(i) + "\nActual : " + actualObj);
- }
- }
- }
-
- @SuppressWarnings("unchecked")
- private static CheckTuple createCheckTupleFromTuple(ITupleReference tuple, ISerializerDeserializer[] fieldSerdes, int numKeys) throws HyracksDataException {
- CheckTuple checkTuple = new CheckTuple(fieldSerdes.length, numKeys);
- int fieldCount = Math.min(fieldSerdes.length, tuple.getFieldCount());
- for (int i = 0; i < fieldCount; i++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(
- tuple.getFieldData(i), tuple.getFieldStart(i),
- tuple.getFieldLength(i));
- DataInput dataIn = new DataInputStream(inStream);
- Comparable fieldObj = (Comparable)fieldSerdes[i].deserialize(dataIn);
- checkTuple.add(fieldObj);
- }
- return checkTuple;
- }
-
- @SuppressWarnings("unchecked")
- private static void createTupleFromCheckTuple(CheckTuple checkTuple, ArrayTupleBuilder tupleBuilder, ArrayTupleReference tuple, ISerializerDeserializer[] fieldSerdes) throws HyracksDataException {
- int fieldCount = tupleBuilder.getFieldEndOffsets().length;
- DataOutput dos = tupleBuilder.getDataOutput();
- tupleBuilder.reset();
- for (int i = 0; i < fieldCount; i++) {
- fieldSerdes[i].serialize(checkTuple.get(i), dos);
- tupleBuilder.addFieldEndOffset();
- }
- tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
- }
-
- public static void checkOrderedScan(BTreeTestContext testCtx) throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Testing Ordered Scan.");
- }
- ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(testCtx.leafFrame, false);
- RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
- testCtx.indexAccessor.search(scanCursor, nullPred);
- Iterator<CheckTuple> checkIter = testCtx.checkTuples.iterator();
- int actualCount = 0;
- try {
- while (scanCursor.hasNext()) {
- if (!checkIter.hasNext()) {
- fail("Ordered scan returned more answers than expected.\nExpected: " + testCtx.checkTuples.size());
- }
- scanCursor.next();
- CheckTuple expectedTuple = checkIter.next();
- ITupleReference tuple = scanCursor.getTuple();
- compareActualAndExpected(tuple, expectedTuple, testCtx.fieldSerdes);
- actualCount++;
- }
- if (actualCount < testCtx.checkTuples.size()) {
- fail("Ordered scan returned fewer answers than expected.\nExpected: " + testCtx.checkTuples.size() + "\nActual : " + actualCount);
- }
- } finally {
- scanCursor.close();
- }
- }
-
- public static void checkDiskOrderScan(BTreeTestContext testCtx) throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Testing Disk-Order Scan.");
- }
- ITreeIndexCursor diskOrderCursor = new TreeDiskOrderScanCursor(testCtx.leafFrame);
- testCtx.indexAccessor.diskOrderScan(diskOrderCursor);
- int actualCount = 0;
- try {
- while (diskOrderCursor.hasNext()) {
- diskOrderCursor.next();
- ITupleReference tuple = diskOrderCursor.getTuple();
- CheckTuple checkTuple = createCheckTupleFromTuple(tuple, testCtx.fieldSerdes, testCtx.btree.getMultiComparator().getKeyFieldCount());
- if (!testCtx.checkTuples.contains(checkTuple)) {
- fail("Disk-order scan returned unexpected answer: " + checkTuple.toString());
- }
- actualCount++;
- }
- if (actualCount < testCtx.checkTuples.size()) {
- fail("Disk-order scan returned fewer answers than expected.\nExpected: " + testCtx.checkTuples.size() + "\nActual : " + actualCount);
- }
- if (actualCount > testCtx.checkTuples.size()) {
- fail("Disk-order scan returned more answers than expected.\nExpected: " + testCtx.checkTuples.size() + "\nActual : " + actualCount);
- }
- } finally {
- diskOrderCursor.close();
- }
- }
-
- public static void checkRangeSearch(BTreeTestContext testCtx, ITupleReference lowKey, ITupleReference highKey, boolean lowKeyInclusive, boolean highKeyInclusive) throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Testing Range Search.");
- }
- MultiComparator lowKeyCmp = BTreeUtils.getSearchMultiComparator(testCtx.btree.getMultiComparator(), lowKey);
- MultiComparator highKeyCmp = BTreeUtils.getSearchMultiComparator(testCtx.btree.getMultiComparator(), highKey);
- ITreeIndexCursor searchCursor = new BTreeRangeSearchCursor(testCtx.leafFrame, false);
- RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, lowKeyInclusive, highKeyInclusive, lowKeyCmp, highKeyCmp);
- testCtx.indexAccessor.search(searchCursor, rangePred);
- // Get the subset of elements from the expected set within given key range.
- CheckTuple lowKeyCheck = createCheckTupleFromTuple(lowKey, testCtx.fieldSerdes, lowKeyCmp.getKeyFieldCount());
- CheckTuple highKeyCheck = createCheckTupleFromTuple(highKey, testCtx.fieldSerdes, highKeyCmp.getKeyFieldCount());
- NavigableSet<CheckTuple> expectedSubset = null;
- if (lowKeyCmp.getKeyFieldCount() < testCtx.btree.getMultiComparator().getKeyFieldCount() ||
- highKeyCmp.getKeyFieldCount() < testCtx.btree.getMultiComparator().getKeyFieldCount()) {
- // Searching on a key prefix (low key or high key or both).
- expectedSubset = getPrefixExpectedSubset(testCtx.checkTuples, lowKeyCheck, highKeyCheck);
- } else {
- // Searching on all key fields.
- expectedSubset = testCtx.checkTuples.subSet(lowKeyCheck, lowKeyInclusive, highKeyCheck, highKeyInclusive);
- }
- Iterator<CheckTuple> checkIter = expectedSubset.iterator();
- int actualCount = 0;
- try {
- while (searchCursor.hasNext()) {
- if (!checkIter.hasNext()) {
- fail("Range search returned more answers than expected.\nExpected: " + expectedSubset.size());
- }
- searchCursor.next();
- CheckTuple expectedTuple = checkIter.next();
- ITupleReference tuple = searchCursor.getTuple();
- compareActualAndExpected(tuple, expectedTuple, testCtx.fieldSerdes);
- actualCount++;
- }
- if (actualCount < expectedSubset.size()) {
- fail("Range search returned fewer answers than expected.\nExpected: " + expectedSubset.size() + "\nActual : " + actualCount);
- }
- } finally {
- searchCursor.close();
- }
- }
-
- public static void checkPointSearches(BTreeTestContext testCtx) throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Testing Point Searches On All Expected Keys.");
- }
- ITreeIndexCursor searchCursor = new BTreeRangeSearchCursor(testCtx.leafFrame, false);
-
- ArrayTupleBuilder lowKeyBuilder = new ArrayTupleBuilder(testCtx.btree.getMultiComparator().getKeyFieldCount());
- ArrayTupleReference lowKey = new ArrayTupleReference();
- ArrayTupleBuilder highKeyBuilder = new ArrayTupleBuilder(testCtx.btree.getMultiComparator().getKeyFieldCount());
- ArrayTupleReference highKey = new ArrayTupleReference();
- RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, true, true, null, null);
-
- // Iterate through expected tuples, and perform a point search in the BTree to verify the tuple can be reached.
- for (CheckTuple checkTuple : testCtx.checkTuples) {
- createTupleFromCheckTuple(checkTuple, lowKeyBuilder, lowKey, testCtx.fieldSerdes);
- createTupleFromCheckTuple(checkTuple, highKeyBuilder, highKey, testCtx.fieldSerdes);
- MultiComparator lowKeyCmp = BTreeUtils.getSearchMultiComparator(testCtx.btree.getMultiComparator(), lowKey);
- MultiComparator highKeyCmp = BTreeUtils.getSearchMultiComparator(testCtx.btree.getMultiComparator(), highKey);
-
- rangePred.setLowKey(lowKey, true);
- rangePred.setHighKey(highKey, true);
- rangePred.setLowKeyComparator(lowKeyCmp);
- rangePred.setHighKeyComparator(highKeyCmp);
-
- testCtx.indexAccessor.search(searchCursor, rangePred);
-
- try {
- // We expect exactly one answer.
- if (searchCursor.hasNext()) {
- searchCursor.next();
- ITupleReference tuple = searchCursor.getTuple();
- compareActualAndExpected(tuple, checkTuple, testCtx.fieldSerdes);
- }
- if (searchCursor.hasNext()) {
- fail("Point search returned more than one answer.");
- }
- } finally {
- searchCursor.close();
- }
- }
- }
-
- @SuppressWarnings("unchecked")
- // Create a new TreeSet containing the elements satisfying the prefix search.
- // Implementing prefix search by changing compareTo() in CheckTuple does not work.
- public static TreeSet<CheckTuple> getPrefixExpectedSubset(TreeSet<CheckTuple> checkTuples, CheckTuple lowKey, CheckTuple highKey) {
- TreeSet<CheckTuple> expectedSubset = new TreeSet<CheckTuple>();
- Iterator<CheckTuple> iter = checkTuples.iterator();
- while(iter.hasNext()) {
- CheckTuple t = iter.next();
- boolean geLowKey = true;
- boolean leHighKey = true;
- for (int i = 0; i < lowKey.getNumKeys(); i++) {
- if (t.get(i).compareTo(lowKey.get(i)) < 0) {
- geLowKey = false;
- break;
- }
- }
- for (int i = 0; i < highKey.getNumKeys(); i++) {
- if (t.get(i).compareTo(highKey.get(i)) > 0) {
- leHighKey = false;
- break;
- }
- }
- if (geLowKey && leHighKey) {
- expectedSubset.add(t);
- }
- }
- return expectedSubset;
- }
-
- public static void insertIntTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
- int fieldCount = testCtx.getFieldCount();
- int numKeyFields = testCtx.getKeyFieldCount();
- int[] tupleValues = new int[testCtx.getFieldCount()];
- // Scale range of values according to number of keys.
- // For example, for 2 keys we want the square root of numTuples, for 3 keys the cube root of numTuples, etc.
- int maxValue = (int)Math.ceil(Math.pow(numTuples, 1.0/(double)numKeyFields));
- for (int i = 0; i < numTuples; i++) {
- // Set keys.
- for (int j = 0; j < numKeyFields; j++) {
- tupleValues[j] = rnd.nextInt() % maxValue;
- }
- // Set values.
- for (int j = numKeyFields; j < fieldCount; j++) {
- tupleValues[j] = j;
- }
- TupleUtils.createIntegerTuple(testCtx.tupleBuilder, testCtx.tuple, tupleValues);
- if (LOGGER.isLoggable(Level.INFO)) {
- if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
- LOGGER.info("Inserting Tuple " + (i + 1) + "/" + numTuples);
- }
- }
- try {
- testCtx.indexAccessor.insert(testCtx.tuple);
- // Set expected values. Do this only after insertion succeeds because we ignore duplicate keys.
- CheckTuple<Integer> checkTuple = new CheckTuple<Integer>(fieldCount, numKeyFields);
- for(int v : tupleValues) {
- checkTuple.add(v);
- }
- testCtx.checkTuples.add(checkTuple);
- } catch (BTreeDuplicateKeyException e) {
- // Ignore duplicate key insertions.
- }
- }
- }
-
- public static void insertStringTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
- int fieldCount = testCtx.getFieldCount();
- int numKeyFields = testCtx.getKeyFieldCount();
- Object[] tupleValues = new Object[fieldCount];
- for (int i = 0; i < numTuples; i++) {
- if (LOGGER.isLoggable(Level.INFO)) {
- if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
- LOGGER.info("Inserting Tuple " + (i + 1) + "/" + numTuples);
- }
- }
- // Set keys.
- for (int j = 0; j < numKeyFields; j++) {
- int length = (Math.abs(rnd.nextInt()) % 10) + 1;
- tupleValues[j] = getRandomString(length, rnd);
- }
- // Set values.
- for (int j = numKeyFields; j < fieldCount; j++) {
- tupleValues[j] = getRandomString(5, rnd);
- }
- TupleUtils.createTuple(testCtx.tupleBuilder, testCtx.tuple, testCtx.fieldSerdes, tupleValues);
- try {
- testCtx.indexAccessor.insert(testCtx.tuple);
- // Set expected values. Do this only after insertion succeeds because we ignore duplicate keys.
- CheckTuple<String> checkTuple = new CheckTuple<String>(fieldCount, numKeyFields);
- for(Object v : tupleValues) {
- checkTuple.add((String)v);
- }
- testCtx.checkTuples.add(checkTuple);
- } catch (BTreeDuplicateKeyException e) {
- // Ignore duplicate key insertions.
- }
- }
- }
-
- public static void bulkLoadIntTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
- int fieldCount = testCtx.getFieldCount();
- int numKeyFields = testCtx.getKeyFieldCount();
- int[] tupleValues = new int[testCtx.getFieldCount()];
- int maxValue = (int)Math.ceil(Math.pow(numTuples, 1.0/(double)numKeyFields));
- for (int i = 0; i < numTuples; i++) {
- // Set keys.
- for (int j = 0; j < numKeyFields; j++) {
- tupleValues[j] = rnd.nextInt() % maxValue;
- }
- // Set values.
- for (int j = numKeyFields; j < fieldCount; j++) {
- tupleValues[j] = j;
- }
-
- // Set expected values. We also use these as the pre-sorted stream for bulk loading.
- CheckTuple<Integer> checkTuple = new CheckTuple<Integer>(fieldCount, numKeyFields);
- for(int v : tupleValues) {
- checkTuple.add(v);
- }
- testCtx.checkTuples.add(checkTuple);
- }
-
- bulkLoadCheckTuples(testCtx, numTuples);
- }
-
- public static void bulkLoadStringTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
- int fieldCount = testCtx.getFieldCount();
- int numKeyFields = testCtx.getKeyFieldCount();
- String[] tupleValues = new String[fieldCount];
- for (int i = 0; i < numTuples; i++) {
- // Set keys.
- for (int j = 0; j < numKeyFields; j++) {
- int length = (Math.abs(rnd.nextInt()) % 10) + 1;
- tupleValues[j] = getRandomString(length, rnd);
- }
- // Set values.
- for (int j = numKeyFields; j < fieldCount; j++) {
- tupleValues[j] = getRandomString(5, rnd);
- }
- // Set expected values. We also use these as the pre-sorted stream for bulk loading.
- CheckTuple<String> checkTuple = new CheckTuple<String>(fieldCount, numKeyFields);
- for(String v : tupleValues) {
- checkTuple.add(v);
- }
- testCtx.checkTuples.add(checkTuple);
- }
-
- bulkLoadCheckTuples(testCtx, numTuples);
- }
-
- private static void bulkLoadCheckTuples(BTreeTestContext testCtx, int numTuples) throws HyracksDataException, TreeIndexException, PageAllocationException {
- int fieldCount = testCtx.getFieldCount();
- ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
- ArrayTupleReference tuple = new ArrayTupleReference();
- // Perform bulk load.
- IIndexBulkLoadContext bulkLoadCtx = testCtx.btree.beginBulkLoad(0.7f);
- int c = 1;
- for (CheckTuple checkTuple : testCtx.checkTuples) {
- if (LOGGER.isLoggable(Level.INFO)) {
- if (c % (numTuples / 10) == 0) {
- LOGGER.info("Bulk Loading Tuple " + c + "/" + numTuples);
- }
- }
- createTupleFromCheckTuple(checkTuple, tupleBuilder, tuple, testCtx.fieldSerdes);
- testCtx.btree.bulkLoadAddTuple(tuple, bulkLoadCtx);
- c++;
- }
- testCtx.btree.endBulkLoad(bulkLoadCtx);
- }
-
- public static void deleteTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
- ArrayTupleBuilder deleteTupleBuilder = new ArrayTupleBuilder(testCtx.btree.getMultiComparator().getKeyFieldCount());
- ArrayTupleReference deleteTuple = new ArrayTupleReference();
- int numCheckTuples = testCtx.checkTuples.size();
- // Copy CheckTuple references into array, so we can randomly pick from there.
- CheckTuple[] checkTuples = new CheckTuple[numCheckTuples];
- int idx = 0;
- for (CheckTuple checkTuple : testCtx.checkTuples) {
- checkTuples[idx++] = checkTuple;
- }
- for (int i = 0; i < numTuples && numCheckTuples > 0; i++) {
- if (LOGGER.isLoggable(Level.INFO)) {
- if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
- LOGGER.info("Deleting Tuple " + (i + 1) + "/" + numTuples);
- }
- }
- int checkTupleIdx = Math.abs(rnd.nextInt() % numCheckTuples);
- CheckTuple checkTuple = checkTuples[checkTupleIdx];
- createTupleFromCheckTuple(checkTuple, deleteTupleBuilder, deleteTuple, testCtx.fieldSerdes);
- testCtx.indexAccessor.delete(deleteTuple);
-
- // Remove check tuple from expected results.
- testCtx.checkTuples.remove(checkTuple);
-
- // Swap with last "valid" CheckTuple.
- CheckTuple tmp = checkTuples[numCheckTuples - 1];
- checkTuples[numCheckTuples - 1] = checkTuple;
- checkTuples[checkTupleIdx] = tmp;
- numCheckTuples--;
- }
- }
-
- @SuppressWarnings("unchecked")
- public static void updateTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
- int fieldCount = testCtx.btree.getFieldCount();
- int keyFieldCount = testCtx.btree.getMultiComparator().getKeyFieldCount();
- // This is a noop because we can only update non-key fields.
- if (fieldCount == keyFieldCount) {
- return;
- }
- ArrayTupleBuilder updateTupleBuilder = new ArrayTupleBuilder(fieldCount);
- ArrayTupleReference updateTuple = new ArrayTupleReference();
- int numCheckTuples = testCtx.checkTuples.size();
- // Copy CheckTuple references into array, so we can randomly pick from there.
- CheckTuple[] checkTuples = new CheckTuple[numCheckTuples];
- int idx = 0;
- for (CheckTuple checkTuple : testCtx.checkTuples) {
- checkTuples[idx++] = checkTuple;
- }
- for (int i = 0; i < numTuples && numCheckTuples > 0; i++) {
- if (LOGGER.isLoggable(Level.INFO)) {
- if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
- LOGGER.info("Updating Tuple " + (i + 1) + "/" + numTuples);
- }
- }
- int checkTupleIdx = Math.abs(rnd.nextInt() % numCheckTuples);
- CheckTuple checkTuple = checkTuples[checkTupleIdx];
- // Update check tuple's non-key fields.
- for (int j = keyFieldCount; j < fieldCount; j++) {
- Comparable newValue = getRandomUpdateValue(testCtx.fieldSerdes[j], rnd);
- checkTuple.set(j, newValue);
- }
-
- createTupleFromCheckTuple(checkTuple, updateTupleBuilder, updateTuple, testCtx.fieldSerdes);
- testCtx.indexAccessor.update(updateTuple);
-
- // Swap with last "valid" CheckTuple.
- CheckTuple tmp = checkTuples[numCheckTuples - 1];
- checkTuples[numCheckTuples - 1] = checkTuple;
- checkTuples[checkTupleIdx] = tmp;
- numCheckTuples--;
- }
- }
-
- private static Comparable getRandomUpdateValue(ISerializerDeserializer serde, Random rnd) {
- if (serde instanceof IntegerSerializerDeserializer) {
- return Integer.valueOf(rnd.nextInt());
- } else if (serde instanceof UTF8StringSerializerDeserializer) {
- return getRandomString(10, rnd);
- }
- return null;
- }
-
- public static String getRandomString(int length, Random rnd) {
- String s = Long.toHexString(Double.doubleToLongBits(rnd.nextDouble()));
- StringBuilder strBuilder = new StringBuilder();
- for (int i = 0; i < s.length() && i < length; i++) {
- strBuilder.append(s.charAt(Math.abs(rnd.nextInt()) % s.length()));
- }
- return strBuilder.toString();
- }
-}
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchPerfTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchPerfTest.java
index 1299fcb..38b8bfa 100644
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchPerfTest.java
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchPerfTest.java
@@ -28,7 +28,6 @@
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
@@ -68,7 +67,7 @@
loadData();
}
- public void loadData() throws HyracksDataException, TreeIndexException, PageAllocationException {
+ public void loadData() throws HyracksDataException, TreeIndexException {
tokens.add("compilers");
tokens.add("computer");
tokens.add("databases");
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchTest.java
index eb2b39c..fc99244 100644
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchTest.java
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchTest.java
@@ -30,7 +30,6 @@
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
@@ -130,7 +129,7 @@
}
}
- public void loadData() throws IOException, TreeIndexException, PageAllocationException {
+ public void loadData() throws IOException, TreeIndexException {
List<TokenIdPair> pairs = new ArrayList<TokenIdPair>();
// generate pairs for subsequent sorting and bulk-loading
int id = 0;
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml b/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
new file mode 100644
index 0000000..dbc02c8
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
@@ -0,0 +1,55 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-btree-test</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-tests</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <type>jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-control-nc</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-btree</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-test-support</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
new file mode 100644
index 0000000..8103191
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.tests.IOrderedIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexBulkLoadTest;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
+
+@SuppressWarnings("rawtypes")
+public class LSMBTreeBulkLoadTest extends OrderedIndexBulkLoadTest {
+
+ public LSMBTreeBulkLoadTest() {
+ super(LSMBTreeTestHarness.LEAF_FRAMES_TO_TEST);
+ }
+
+ private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected IOrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
+ BTreeLeafFrameType leafType) throws Exception {
+ return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
+ harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ numKeys, harness.getFileId());
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
new file mode 100644
index 0000000..27ef60f
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.tests.IOrderedIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexDeleteTest;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
+
+@SuppressWarnings("rawtypes")
+public class LSMBTreeDeleteTest extends OrderedIndexDeleteTest {
+
+ public LSMBTreeDeleteTest() {
+ super(LSMBTreeTestHarness.LEAF_FRAMES_TO_TEST);
+ }
+
+ private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected IOrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
+ BTreeLeafFrameType leafType) throws Exception {
+ return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
+ harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ numKeys, harness.getFileId());
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
new file mode 100644
index 0000000..a46142e
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexExamplesTest;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
+import edu.uci.ics.hyracks.storage.am.lsm.util.LSMBTreeUtils;
+
+public class LSMBTreeExamplesTest extends OrderedIndexExamplesTest {
+ private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
+
+ @Override
+ protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits,
+ IBinaryComparator[] cmps) throws TreeIndexException {
+ return LSMBTreeUtils.createLSMTree(harness.getMemBufferCache(),
+ harness.getMemFreePageManager(), harness.getOnDiskDir(),
+ harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
+ typeTraits, cmps);
+ }
+
+ @Override
+ protected int getIndexFileId() {
+ return harness.getFileId();
+ }
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeFileNameManagerTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeFileNameManagerTest.java
new file mode 100644
index 0000000..87ee59f
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeFileNameManagerTest.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFileNameManager;
+import edu.uci.ics.hyracks.storage.am.lsm.impls.LSMBTreeFileNameManager;
+
+public class LSMBTreeFileNameManagerTest {
+ protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ protected final static String tmpDir = System.getProperty("java.io.tmpdir");
+ protected final static String sep = System.getProperty("file.separator");
+ protected String baseDir;
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ baseDir = tmpDir + sep + "lsm_btree" + simpleDateFormat.format(new Date()) + sep;
+ }
+
+ public void flushFileNamesTest(boolean testFlushFileName) throws InterruptedException {
+ ILSMFileNameManager fileNameManager = new LSMBTreeFileNameManager(baseDir);
+ LinkedList<String> fileNames = new LinkedList<String>();
+
+ int numFileNames = 100;
+ long sleepTime = 5;
+ for (int i = 0; i < numFileNames; i++) {
+ String fileName = null;
+ if (testFlushFileName) {
+ fileName = fileNameManager.getFlushFileName();
+ } else {
+ fileName = fileNameManager.getMergeFileName();
+ }
+ fileNames.addFirst(fileName);
+ Thread.sleep(sleepTime);
+ }
+
+ List<String> sortedFileNames = new ArrayList<String>();
+ sortedFileNames.addAll(fileNames);
+
+ // Make sure the comparator sorts in the correct order (i.e., the
+ // reverse insertion order in this case).
+ Comparator<String> cmp = fileNameManager.getFileNameComparator();
+ Collections.sort(sortedFileNames, cmp);
+ for (int i = 0; i < numFileNames; i++) {
+ assertEquals(fileNames.get(i), sortedFileNames.get(i));
+ }
+ }
+
+ @Test
+ public void individualFileNamesTest() throws InterruptedException {
+ flushFileNamesTest(true);
+ flushFileNamesTest(false);
+ }
+
+ @Test
+ public void mixedFileNamesTest() throws InterruptedException {
+ ILSMFileNameManager fileNameManager = new LSMBTreeFileNameManager(baseDir);
+ List<String> fileNames = new ArrayList<String>();
+ HashSet<String> flushFileNames = new HashSet<String>();
+ HashSet<String> mergeFileNames = new HashSet<String>();
+
+ int numFileNames = 100;
+ long sleepTime = 5;
+ for (int i = 0; i < numFileNames; i++) {
+ String fileName = null;
+ if (i % 2 == 0) {
+ fileName = fileNameManager.getFlushFileName();
+ flushFileNames.add(fileName);
+ } else {
+ fileName = fileNameManager.getMergeFileName();
+ mergeFileNames.add(fileName);
+ }
+ fileNames.add(fileName);
+ Thread.sleep(sleepTime);
+ }
+
+ // Make sure the comparator sorts in the correct order.
+ // We only need to check whether the flushed files and merged files are
+ // clustered.
+ // We verified the secondary sort order (based on timestamp) in the
+ // individualFileNamesTest().
+ Comparator<String> cmp = fileNameManager.getFileNameComparator();
+ Collections.sort(fileNames, cmp);
+ // We expect flush file names at the front.
+ int i = 0;
+ while (flushFileNames.contains(fileNames.get(i))) {
+ i++;
+ }
+ // We expect only merge file names from here on.
+ while (i < numFileNames) {
+ if (!mergeFileNames.contains(fileNames.get(i))) {
+ fail("Expected a merge file name at position: " + i);
+ }
+ i++;
+ }
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
new file mode 100644
index 0000000..4ab58a7
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.tests.IOrderedIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexInsertTest;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
+
+@SuppressWarnings("rawtypes")
+public class LSMBTreeInsertTest extends OrderedIndexInsertTest {
+
+ public LSMBTreeInsertTest() {
+ super(LSMBTreeTestHarness.LEAF_FRAMES_TO_TEST);
+ }
+
+ private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected IOrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
+ BTreeLeafFrameType leafType) throws Exception {
+ return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
+ harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ numKeys, harness.getFileId());
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
new file mode 100644
index 0000000..8fbe9da
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.tests.IOrderedIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexUpdateTest;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
+
+@SuppressWarnings("rawtypes")
+public class LSMBTreeUpdateTest extends OrderedIndexUpdateTest {
+
+ public LSMBTreeUpdateTest() {
+ super(LSMBTreeTestHarness.LEAF_FRAMES_TO_TEST);
+ }
+
+ private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected IOrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
+ BTreeLeafFrameType leafType) throws Exception {
+ return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
+ harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ numKeys, harness.getFileId());
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/BTreeBulkLoadRunner.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/BTreeBulkLoadRunner.java
new file mode 100644
index 0000000..1590b17
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/BTreeBulkLoadRunner.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree.perf;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
+import edu.uci.ics.hyracks.storage.am.common.datagen.TupleBatch;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+
+public class BTreeBulkLoadRunner extends BTreeRunner {
+
+ protected final float fillFactor;
+
+ public BTreeBulkLoadRunner(int numBatches, int pageSize, int numPages, ITypeTraits[] typeTraits, MultiComparator cmp, float fillFactor)
+ throws HyracksDataException, BTreeException {
+ super(numBatches, pageSize, numPages, typeTraits, cmp);
+ this.fillFactor = fillFactor;
+ }
+
+ @Override
+ public long runExperiment(DataGenThread dataGen, int numThreads) throws Exception {
+ btree.create(btreeFileId);
+ long start = System.currentTimeMillis();
+ IIndexBulkLoadContext bulkLoadCtx = btree.beginBulkLoad(1.0f);
+ for (int i = 0; i < numBatches; i++) {
+ TupleBatch batch = dataGen.tupleBatchQueue.take();
+ for (int j = 0; j < batch.size(); j++) {
+ btree.bulkLoadAddTuple(batch.get(j), bulkLoadCtx);
+ }
+ }
+ btree.endBulkLoad(bulkLoadCtx);
+ long end = System.currentTimeMillis();
+ long time = end - start;
+ return time;
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/BTreePageSizePerf.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/BTreePageSizePerf.java
new file mode 100644
index 0000000..31a4c92
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/BTreePageSizePerf.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree.perf;
+
+import java.util.Enumeration;
+import java.util.logging.Level;
+import java.util.logging.LogManager;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
+import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+
+public class BTreePageSizePerf {
+ public static void main(String[] args) throws Exception {
+ // Disable logging so we can better see the output times.
+ Enumeration<String> loggers = LogManager.getLogManager().getLoggerNames();
+ while(loggers.hasMoreElements()) {
+ String loggerName = loggers.nextElement();
+ Logger logger = LogManager.getLogManager().getLogger(loggerName);
+ logger.setLevel(Level.OFF);
+ }
+
+ int numTuples = 1000000;
+ int batchSize = 10000;
+ int numBatches = numTuples / batchSize;
+
+ ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE };
+ ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes, 30);
+
+ IBinaryComparator[] cmps = SerdeUtils.serdesToComparators(fieldSerdes, fieldSerdes.length);
+ MultiComparator cmp = new MultiComparator(cmps);
+
+ runExperiment(numBatches, batchSize, 1024, 100000, fieldSerdes, cmp, typeTraits);
+ runExperiment(numBatches, batchSize, 2048, 100000, fieldSerdes, cmp, typeTraits);
+ runExperiment(numBatches, batchSize, 4096, 25000, fieldSerdes, cmp, typeTraits);
+ runExperiment(numBatches, batchSize, 8192, 12500, fieldSerdes, cmp, typeTraits);
+ runExperiment(numBatches, batchSize, 16384, 6250, fieldSerdes, cmp, typeTraits);
+ runExperiment(numBatches, batchSize, 32768, 3125, fieldSerdes, cmp, typeTraits);
+ runExperiment(numBatches, batchSize, 65536, 1564, fieldSerdes, cmp, typeTraits);
+ runExperiment(numBatches, batchSize, 131072, 782, fieldSerdes, cmp, typeTraits);
+ runExperiment(numBatches, batchSize, 262144, 391, fieldSerdes, cmp, typeTraits);
+ }
+
+ private static void runExperiment(int numBatches, int batchSize, int pageSize, int numPages, ISerializerDeserializer[] fieldSerdes, MultiComparator cmp, ITypeTraits[] typeTraits) throws Exception {
+ System.out.println("PAGE SIZE: " + pageSize);
+ System.out.println("NUM PAGES: " + numPages);
+ System.out.println("MEMORY: " + (pageSize * numPages));
+ int repeats = 5;
+ long[] times = new long[repeats];
+ //BTreeRunner runner = new BTreeRunner(numTuples, pageSize, numPages, typeTraits, cmp);
+ InMemoryBTreeRunner runner = new InMemoryBTreeRunner(numBatches, pageSize, numPages, typeTraits, cmp);
+ runner.init();
+ int numThreads = 1;
+ for (int i = 0; i < repeats; i++) {
+ DataGenThread dataGen = new DataGenThread(numBatches, batchSize, 10, numThreads, fieldSerdes, 30, 50, false);
+ dataGen.start();
+ times[i] = runner.runExperiment(dataGen, numThreads);
+ System.out.println("TIME " + i + ": " + times[i] + "ms");
+ }
+ runner.deinit();
+ long avgTime = 0;
+ for (int i = 0; i < repeats; i++) {
+ avgTime += times[i];
+ }
+ avgTime /= repeats;
+ System.out.println("AVG TIME: " + avgTime + "ms");
+ System.out.println("-------------------------------");
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/BTreeRunner.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/BTreeRunner.java
new file mode 100644
index 0000000..f110941
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/BTreeRunner.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree.perf;
+
+import java.io.File;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
+import edu.uci.ics.hyracks.test.support.TestUtils;
+
+public class BTreeRunner extends InMemoryBTreeRunner {
+ protected static final int MAX_OPEN_FILES = 10;
+ protected static final int HYRACKS_FRAME_SIZE = 128;
+
+ public BTreeRunner(int numTuples, int pageSize, int numPages, ITypeTraits[] typeTraits, MultiComparator cmp) throws HyracksDataException, BTreeException {
+ super(numTuples, pageSize, numPages, typeTraits, cmp);
+ }
+
+ @Override
+ protected void init(int pageSize, int numPages, ITypeTraits[] typeTraits, MultiComparator cmp) throws HyracksDataException, BTreeException {
+ IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
+ TestStorageManagerComponentHolder.init(pageSize, numPages, MAX_OPEN_FILES);
+ bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ FileReference file = new FileReference(new File(fileName));
+ bufferCache.createFile(file);
+ btreeFileId = fmp.lookupFileId(file);
+ bufferCache.openFile(btreeFileId);
+ btree = BTreeUtils
+ .createBTree(bufferCache, btreeFileId, typeTraits, cmp.getComparators(), BTreeLeafFrameType.REGULAR_NSM);
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/ConcurrentSkipListRunner.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/ConcurrentSkipListRunner.java
new file mode 100644
index 0000000..8f44966
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/ConcurrentSkipListRunner.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree.perf;
+
+import java.nio.ByteBuffer;
+import java.util.Comparator;
+import java.util.concurrent.ConcurrentSkipListSet;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
+import edu.uci.ics.hyracks.storage.am.common.datagen.TupleBatch;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+
+public class ConcurrentSkipListRunner implements IExperimentRunner {
+ public class TupleComparator implements Comparator<ITupleReference> {
+ private final MultiComparator cmp;
+
+ public TupleComparator(MultiComparator cmp) {
+ this.cmp = cmp;
+ }
+
+ @Override
+ public int compare(ITupleReference o1, ITupleReference o2) {
+ return cmp.compare(o1, o2);
+ }
+ }
+
+ private final TupleComparator tupleCmp;
+ private final int numBatches;
+ private final int batchSize;
+ private final int tupleSize;
+ private final ITypeTraits[] typeTraits;
+
+ public ConcurrentSkipListRunner(int numBatches, int batchSize, int tupleSize, ITypeTraits[] typeTraits, MultiComparator cmp) {
+ this.numBatches = numBatches;
+ this.tupleSize = tupleSize;
+ this.batchSize = batchSize;
+ this.typeTraits = typeTraits;
+ tupleCmp = new TupleComparator(cmp);
+ }
+
+ @Override
+ public long runExperiment(DataGenThread dataGen, int numThreads) throws InterruptedException {
+ ConcurrentSkipListSet<ITupleReference> skipList = new ConcurrentSkipListSet<ITupleReference>(tupleCmp);
+ SkipListThread[] threads = new SkipListThread[numThreads];
+ int threadNumBatches = numBatches / numThreads;
+ for (int i = 0; i < numThreads; i++) {
+ threads[i] = new SkipListThread(dataGen, skipList, threadNumBatches, batchSize);
+ }
+ // Wait until the tupleBatchQueue is completely full.
+ while (dataGen.tupleBatchQueue.remainingCapacity() != 0) {
+ Thread.sleep(10);
+ }
+
+ long start = System.currentTimeMillis();
+ for (int i = 0; i < numThreads; i++) {
+ threads[i].start();
+ }
+ for (int i = 0; i < numThreads; i++) {
+ threads[i].join();
+ }
+ long end = System.currentTimeMillis();
+ long time = end - start;
+ return time;
+ }
+
+ @Override
+ public void init() throws Exception {
+ }
+
+ @Override
+ public void deinit() throws Exception {
+ }
+
+ public void reset() throws Exception {
+ }
+
+ public class SkipListThread extends Thread {
+ private final DataGenThread dataGen;
+ private final ConcurrentSkipListSet<ITupleReference> skipList;
+ private final int numBatches;
+ public final TypeAwareTupleWriterFactory tupleWriterFactory;
+ public final TypeAwareTupleWriter tupleWriter;
+ public final TypeAwareTupleReference[] tuples;
+ public final ByteBuffer tupleBuf;
+
+ public SkipListThread(DataGenThread dataGen, ConcurrentSkipListSet<ITupleReference> skipList, int numBatches, int batchSize) {
+ this.dataGen = dataGen;
+ this.numBatches = numBatches;
+ this.skipList = skipList;
+ tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
+ tupleWriter = (TypeAwareTupleWriter) tupleWriterFactory.createTupleWriter();
+ int numTuples = numBatches * batchSize;
+ tuples = new TypeAwareTupleReference[numTuples];
+ tupleBuf = ByteBuffer.allocate(numTuples * tupleSize);
+ for (int i = 0; i < numTuples; i++) {
+ tuples[i] = (TypeAwareTupleReference) tupleWriter.createTupleReference();
+ }
+ }
+
+ @Override
+ public void run() {
+ int tupleIndex = 0;
+ try {
+ for (int i = 0; i < numBatches; i++) {
+ TupleBatch batch = dataGen.tupleBatchQueue.take();
+ for (int j = 0; j < batch.size(); j++) {
+ // Copy the tuple to the buffer and set the pre-created tuple ref.
+ tupleWriter.writeTuple(batch.get(j), tupleBuf.array(), tupleIndex * tupleSize);
+ tuples[tupleIndex].resetByTupleOffset(tupleBuf, tupleIndex * tupleSize);
+ skipList.add(tuples[tupleIndex]);
+ tupleIndex++;
+ }
+ }
+ } catch (Exception e) {
+ System.out.println(tupleIndex);
+ e.printStackTrace();
+ }
+ }
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/IExperimentRunner.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/IExperimentRunner.java
new file mode 100644
index 0000000..0ea3a71
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/IExperimentRunner.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree.perf;
+
+import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
+
+public interface IExperimentRunner {
+ public static int DEFAULT_MAX_OUTSTANDING = 100000;
+
+ public void init() throws Exception;
+
+ public long runExperiment(DataGenThread dataGen, int numThreads) throws Exception;
+
+ public void reset() throws Exception;
+
+ public void deinit() throws Exception;
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java
new file mode 100644
index 0000000..1afd3d8
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java
@@ -0,0 +1,140 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree.perf;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
+import edu.uci.ics.hyracks.storage.am.common.datagen.TupleBatch;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
+
+public class InMemoryBTreeRunner extends Thread implements IExperimentRunner {
+ protected IBufferCache bufferCache;
+ protected int btreeFileId;
+
+ protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ protected final static String tmpDir = System.getProperty("java.io.tmpdir");
+ protected final static String sep = System.getProperty("file.separator");
+ protected String fileName;
+
+ protected final int numBatches;
+ protected BTree btree;
+
+ public InMemoryBTreeRunner(int numBatches, int pageSize, int numPages, ITypeTraits[] typeTraits, MultiComparator cmp) throws HyracksDataException, BTreeException {
+ this.numBatches = numBatches;
+ fileName = tmpDir + sep + simpleDateFormat.format(new Date());
+ init(pageSize, numPages, typeTraits, cmp);
+ }
+
+ protected void init(int pageSize, int numPages, ITypeTraits[] typeTraits, MultiComparator cmp) throws HyracksDataException, BTreeException {
+ ICacheMemoryAllocator allocator = new HeapBufferAllocator();
+ bufferCache = new InMemoryBufferCache(allocator, pageSize, numPages);
+ // Chose an aribtrary file id.
+ btreeFileId = 0;
+ TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+ IFreePageManager freePageManager = new InMemoryFreePageManager(bufferCache.getNumPages(), metaFrameFactory);
+ btree = new BTree(bufferCache, typeTraits.length, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ }
+
+ @Override
+ public long runExperiment(DataGenThread dataGen, int numThreads) throws Exception {
+ BTreeThread[] threads = new BTreeThread[numThreads];
+ int threadNumBatches = numBatches / numThreads;
+ for (int i = 0; i < numThreads; i++) {
+ threads[i] = new BTreeThread(dataGen, btree, threadNumBatches);
+ }
+ // Wait until the tupleBatchQueue is completely full.
+ while (dataGen.tupleBatchQueue.remainingCapacity() != 0) {
+ Thread.sleep(10);
+ }
+
+ long start = System.currentTimeMillis();
+ for (int i = 0; i < numThreads; i++) {
+ threads[i].start();
+ }
+ for (int i = 0; i < numThreads; i++) {
+ threads[i].join();
+ }
+ long end = System.currentTimeMillis();
+ long time = end - start;
+ return time;
+ }
+
+ @Override
+ public void init() throws Exception {
+ }
+
+ @Override
+ public void deinit() throws Exception {
+ bufferCache.closeFile(btreeFileId);
+ bufferCache.close();
+ }
+
+ @Override
+ public void reset() throws Exception {
+ btree.create(btreeFileId);
+ }
+
+ public class BTreeThread extends Thread {
+ private final DataGenThread dataGen;
+ private final int numBatches;
+ private final ITreeIndexAccessor indexAccessor;
+ public BTreeThread(DataGenThread dataGen, BTree btree, int numBatches) {
+ this.dataGen = dataGen;
+ this.numBatches = numBatches;
+ indexAccessor = btree.createAccessor();
+ }
+
+ @Override
+ public void run() {
+ try {
+ for (int i = 0; i < numBatches; i++) {
+ TupleBatch batch = dataGen.tupleBatchQueue.take();
+ for (int j = 0; j < batch.size(); j++) {
+ try {
+ indexAccessor.insert(batch.get(j));
+ } catch (TreeIndexException e) {
+ }
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/InMemorySortRunner.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/InMemorySortRunner.java
new file mode 100644
index 0000000..53fbd88
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/InMemorySortRunner.java
@@ -0,0 +1,153 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree.perf;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.concurrent.ConcurrentSkipListSet;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
+import edu.uci.ics.hyracks.storage.am.common.datagen.TupleBatch;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+
+public class InMemorySortRunner implements IExperimentRunner {
+ public class TupleComparator implements Comparator<ITupleReference> {
+ private final MultiComparator cmp;
+
+ public TupleComparator(MultiComparator cmp) {
+ this.cmp = cmp;
+ }
+
+ @Override
+ public int compare(ITupleReference o1, ITupleReference o2) {
+ return cmp.compare(o1, o2);
+ }
+ }
+
+ private final TupleComparator tupleCmp;
+ private final int numBatches;
+ private final int batchSize;
+ private final int tupleSize;
+ private final ITypeTraits[] typeTraits;
+
+ private final TypeAwareTupleWriterFactory tupleWriterFactory;
+ private final TypeAwareTupleWriter tupleWriter;
+ private final ArrayList<TypeAwareTupleReference> tuples;
+ private final ByteBuffer tupleBuf;
+
+ public InMemorySortRunner(int numBatches, int batchSize, int tupleSize, ITypeTraits[] typeTraits, MultiComparator cmp) {
+ this.numBatches = numBatches;
+ this.tupleSize = tupleSize;
+ this.batchSize = batchSize;
+ this.typeTraits = typeTraits;
+ tupleCmp = new TupleComparator(cmp);
+ tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
+ tupleWriter = (TypeAwareTupleWriter) tupleWriterFactory.createTupleWriter();
+ int numTuples = numBatches * batchSize;
+ tuples = new ArrayList<TypeAwareTupleReference>();
+ tupleBuf = ByteBuffer.allocate(numTuples * tupleSize);
+ for (int i = 0; i < numTuples; i++) {
+ tuples.add((TypeAwareTupleReference) tupleWriter.createTupleReference());
+ }
+ }
+
+ @Override
+ public long runExperiment(DataGenThread dataGen, int numThreads) throws InterruptedException {
+ // Wait until the tupleBatchQueue is completely full.
+ while (dataGen.tupleBatchQueue.remainingCapacity() != 0) {
+ Thread.sleep(10);
+ }
+
+ long start = System.currentTimeMillis();
+ int tupleIndex = 0;
+ for (int i = 0; i < numBatches; i++) {
+ TupleBatch batch = dataGen.tupleBatchQueue.take();
+ for (int j = 0; j < batch.size(); j++) {
+ // Copy the tuple to the buffer and set the pre-created tuple ref.
+ tupleWriter.writeTuple(batch.get(j), tupleBuf.array(), tupleIndex * tupleSize);
+ tuples.get(tupleIndex).resetByTupleOffset(tupleBuf, tupleIndex * tupleSize);
+ tupleIndex++;
+ }
+ }
+ // Perform the sort.
+ Collections.sort(tuples, tupleCmp);
+ long end = System.currentTimeMillis();
+ long time = end - start;
+ return time;
+ }
+
+ @Override
+ public void init() throws Exception {
+ }
+
+ @Override
+ public void deinit() throws Exception {
+ }
+
+ public void reset() throws Exception {
+ }
+
+ public class SkipListThread extends Thread {
+ private final DataGenThread dataGen;
+ private final ConcurrentSkipListSet<ITupleReference> skipList;
+ private final int numBatches;
+ public final TypeAwareTupleWriterFactory tupleWriterFactory;
+ public final TypeAwareTupleWriter tupleWriter;
+ public final TypeAwareTupleReference[] tuples;
+ public final ByteBuffer tupleBuf;
+
+ public SkipListThread(DataGenThread dataGen, ConcurrentSkipListSet<ITupleReference> skipList, int numBatches, int batchSize) {
+ this.dataGen = dataGen;
+ this.numBatches = numBatches;
+ this.skipList = skipList;
+ tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
+ tupleWriter = (TypeAwareTupleWriter) tupleWriterFactory.createTupleWriter();
+ int numTuples = numBatches * batchSize;
+ tuples = new TypeAwareTupleReference[numTuples];
+ tupleBuf = ByteBuffer.allocate(numTuples * tupleSize);
+ for (int i = 0; i < numTuples; i++) {
+ tuples[i] = (TypeAwareTupleReference) tupleWriter.createTupleReference();
+ }
+ }
+
+ @Override
+ public void run() {
+ int tupleIndex = 0;
+ try {
+ for (int i = 0; i < numBatches; i++) {
+ TupleBatch batch = dataGen.tupleBatchQueue.take();
+ for (int j = 0; j < batch.size(); j++) {
+ // Copy the tuple to the buffer and set the pre-created tuple ref.
+ tupleWriter.writeTuple(batch.get(j), tupleBuf.array(), tupleIndex * tupleSize);
+ tuples[tupleIndex].resetByTupleOffset(tupleBuf, tupleIndex * tupleSize);
+ skipList.add(tuples[tupleIndex]);
+ tupleIndex++;
+ }
+ }
+ } catch (Exception e) {
+ System.out.println(tupleIndex);
+ e.printStackTrace();
+ }
+ }
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
new file mode 100644
index 0000000..28e58c7
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
@@ -0,0 +1,151 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree.perf;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
+import edu.uci.ics.hyracks.storage.am.common.datagen.TupleBatch;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsm.impls.LSMBTree;
+import edu.uci.ics.hyracks.storage.am.lsm.util.LSMBTreeUtils;
+import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
+import edu.uci.ics.hyracks.test.support.TestUtils;
+
+public class LSMTreeRunner implements IExperimentRunner {
+
+ private static final int MAX_OPEN_FILES = 10000;
+ private static final int HYRACKS_FRAME_SIZE = 128;
+
+ protected IHyracksTaskContext ctx;
+ protected IBufferCache bufferCache;
+ protected int lsmtreeFileId;
+
+ protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ protected final static String tmpDir = System.getProperty("java.io.tmpdir");
+ protected final static String sep = System.getProperty("file.separator");
+ protected final static String classDir = "/lsmtree/";
+ protected String onDiskDir;
+
+ protected final int numBatches;
+ protected final LSMBTree lsmtree;
+ protected IBufferCache memBufferCache;
+ private final int onDiskPageSize;
+ private final int onDiskNumPages;
+
+ public LSMTreeRunner(int numBatches, int inMemPageSize, int inMemNumPages, int onDiskPageSize, int onDiskNumPages, ITypeTraits[] typeTraits, MultiComparator cmp) throws HyracksDataException, BTreeException {
+ this.numBatches = numBatches;
+
+ this.onDiskPageSize = onDiskPageSize;
+ this.onDiskNumPages = onDiskNumPages;
+
+ onDiskDir = tmpDir + classDir + sep + simpleDateFormat.format(new Date()) + sep;
+ ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
+
+ TestStorageManagerComponentHolder.init(this.onDiskPageSize, this.onDiskNumPages, MAX_OPEN_FILES);
+ bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+
+ InMemoryBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), inMemPageSize, inMemNumPages);
+ InMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(inMemNumPages, new LIFOMetaDataFrameFactory());
+
+ lsmtree = LSMBTreeUtils.createLSMTree(memBufferCache, memFreePageManager, onDiskDir, bufferCache, fmp, typeTraits, cmp.getComparators());
+ }
+ @Override
+ public void init() throws Exception {
+ }
+
+ @Override
+ public long runExperiment(DataGenThread dataGen, int numThreads) throws Exception {
+ LSMTreeThread[] threads = new LSMTreeThread[numThreads];
+ int threadNumBatches = numBatches / numThreads;
+ for (int i = 0; i < numThreads; i++) {
+ threads[i] = new LSMTreeThread(dataGen, lsmtree, threadNumBatches);
+ }
+ // Wait until the tupleBatchQueue is completely full.
+ while (dataGen.tupleBatchQueue.remainingCapacity() != 0) {
+ Thread.sleep(10);
+ }
+
+ long start = System.currentTimeMillis();
+ for (int i = 0; i < numThreads; i++) {
+ threads[i].start();
+ }
+ for (int i = 0; i < numThreads; i++) {
+ threads[i].join();
+ }
+ long end = System.currentTimeMillis();
+ long time = end - start;
+ return time;
+ }
+
+ @Override
+ public void reset() throws Exception {
+ lsmtree.create(lsmtreeFileId);
+ }
+
+ @Override
+ public void deinit() throws Exception {
+ bufferCache.closeFile(lsmtreeFileId);
+ bufferCache.close();
+ memBufferCache.closeFile(lsmtreeFileId);
+ memBufferCache.close();
+ }
+
+ public class LSMTreeThread extends Thread {
+ private final DataGenThread dataGen;
+ private final LSMBTree lsmTree;
+ private final int numBatches;
+ private final ITreeIndexAccessor lsmTreeAccessor;
+ public LSMTreeThread(DataGenThread dataGen, LSMBTree lsmTree, int numBatches) {
+ this.dataGen = dataGen;
+ this.lsmTree = lsmTree;
+ this.numBatches = numBatches;
+ lsmTreeAccessor = lsmTree.createAccessor();
+ }
+
+ @Override
+ public void run() {
+ try {
+ for (int i = 0; i < numBatches; i++) {
+ TupleBatch batch = dataGen.tupleBatchQueue.take();
+ for (int j = 0; j < batch.size(); j++) {
+ try {
+ lsmTreeAccessor.insert(batch.get(j));
+ } catch (TreeIndexException e) {
+ }
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/PerfExperiment.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/PerfExperiment.java
new file mode 100644
index 0000000..f7cf57b
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/PerfExperiment.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree.perf;
+
+import java.io.FileOutputStream;
+import java.io.PrintStream;
+import java.util.Enumeration;
+import java.util.logging.Level;
+import java.util.logging.LogManager;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
+import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+
+public class PerfExperiment {
+ public static void main(String[] args) throws Exception {
+ // Disable logging so we can better see the output times.
+ Enumeration<String> loggers = LogManager.getLogManager().getLoggerNames();
+ while(loggers.hasMoreElements()) {
+ String loggerName = loggers.nextElement();
+ Logger logger = LogManager.getLogManager().getLogger(loggerName);
+ logger.setLevel(Level.OFF);
+ }
+
+ //int numTuples = 1000;
+ //int batchSize = 100;
+ int numTuples = 100000; // 100K
+ //int numTuples = 1000000; // 1M
+ //int numTuples = 2000000; // 2M
+ //int numTuples = 3000000; // 3M
+ //int numTuples = 10000000; // 10M
+ //int numTuples = 20000000; // 20M
+ //int numTuples = 30000000; // 30M
+ //int numTuples = 40000000; // 40M
+ //int numTuples = 60000000; // 60M
+ //int numTuples = 100000000; // 100M
+ //int numTuples = 200000000; // 200M
+ int batchSize = 10000;
+ int numBatches = numTuples / batchSize;
+
+ ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE };
+ ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes, 30);
+
+ FileOutputStream out;
+ PrintStream p;
+ out = new FileOutputStream("/tmp/testOutput.txt");
+ p = new PrintStream( out );
+
+
+ // Add 1 byte for the null flags.
+ // TODO: hide this in some method.
+ int tupleSize = 4 + 30 + 1;
+
+ IBinaryComparator[] cmps = SerdeUtils.serdesToComparators(fieldSerdes, fieldSerdes.length);
+ MultiComparator cmp = new MultiComparator(cmps);
+
+
+ //Test 30M Btree
+ try{
+ p.println ("Start for test 30M BTree");
+ }catch (Exception e){
+ System.err.println ("Error writing to file");
+ }
+
+
+ //int repeats = 1000;
+ int repeats = 1;
+ long[] times = new long[repeats];
+
+ int numThreads = 1;
+ for (int i = 0; i < repeats; i++) {
+ //ConcurrentSkipListRunner runner = new ConcurrentSkipListRunner(numBatches, batchSize, tupleSize, typeTraits, cmp);
+
+ //InMemoryBTreeRunner runner = new InMemoryBTreeRunner(numBatches, 8192, 100000, typeTraits, cmp);
+ //BTreeBulkLoadRunner runner = new BTreeBulkLoadRunner(numBatches, 8192, 100000, typeTraits, cmp, 1.0f);
+
+ //BTreeRunner runner = new BTreeRunner(numBatches, 8192, 100000, typeTraits, cmp);
+ //String btreeName = "071211";
+ //BTreeSearchRunner runner = new BTreeSearchRunner(btreeName, 10, numBatches, 8192, 25000, typeTraits, cmp);
+ LSMTreeRunner runner = new LSMTreeRunner(numBatches, 8192, 100, 8192, 250, typeTraits, cmp);
+ //LSMTreeSearchRunner runner = new LSMTreeSearchRunner(100000, numBatches, 8192, 24750, 8192, 250, typeTraits, cmp);
+ DataGenThread dataGen = new DataGenThread(numBatches, batchSize, 10, numThreads, fieldSerdes, 30, 50, false);
+ dataGen.start();
+ runner.reset();
+ times[i] = runner.runExperiment(dataGen, numThreads);
+ System.out.println("TIME " + i + ": " + times[i] + "ms");
+ try{
+ p.println ("TIME " + i + ": " + times[i] + "ms");
+ }catch (Exception e){
+ System.err.println ("Error writing to file");
+ }
+ runner.deinit();
+ }
+
+ long avgTime = 0;
+ for (int i = 0; i < repeats; i++) {
+ avgTime += times[i];
+ }
+
+ avgTime /= repeats;
+ System.out.println("AVG TIME: " + avgTime + "ms");
+ try{
+ p.println ("AVG TIME: " + avgTime + "ms");
+ }catch (Exception e){
+ System.err.println ("Error writing to file");
+ }
+
+
+
+/*
+ //Test 30M Btree
+ try{
+ p.println ("Start for test 40M LSMTree");
+ }catch (Exception e){
+ System.err.println ("Error writing to file");
+ }
+
+ numTuples = 40000000; // 40M
+ //numTuples = 1000000; // 100K
+ numBatches = numTuples / batchSize;
+ runner = new BTreeRunner(numBatches, 8192, 100000, typeTraits, cmp);
+
+ runner.init();
+ for (int i = 0; i < repeats; i++) {
+ // LSMTreeRunner runner = new LSMTreeRunner(numBatches, 8192, 100000, typeTraits, cmp);
+ DataGenThread dataGen = new DataGenThread(numBatches, batchSize, 10, numThreads, fieldSerdes, 30, 50, false);
+ dataGen.start();
+ runner.reset();
+ times[i] = runner.runExperiment(dataGen, numThreads);
+ System.out.println("TIME " + i + ": " + times[i] + "ms");
+ try{
+ p.println ("TIME " + i + ": " + times[i] + "ms");
+ }catch (Exception e){
+ System.err.println ("Error writing to file");
+ }
+ // runner.deinit();
+ }
+ runner.deinit();
+
+ avgTime = 0;
+ for (int i = 0; i < repeats; i++) {
+ avgTime += times[i];
+ }
+
+ avgTime /= repeats;
+ System.out.println("AVG TIME: " + avgTime + "ms");
+ try{
+ p.println ("AVG TIME: " + avgTime + "ms");
+ }catch (Exception e){
+ System.err.println ("Error writing to file");
+ }
+ */
+ p.close();
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
new file mode 100644
index 0000000..ff462b5
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree.util;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
+import edu.uci.ics.hyracks.storage.am.btree.tests.CheckTuple;
+import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsm.impls.LSMBTree;
+import edu.uci.ics.hyracks.storage.am.lsm.util.LSMBTreeUtils;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+
+@SuppressWarnings("rawtypes")
+public final class LSMBTreeTestContext extends OrderedIndexTestContext {
+
+ public LSMBTreeTestContext(ISerializerDeserializer[] fieldSerdes, ITreeIndex treeIndex) {
+ super(fieldSerdes, treeIndex);
+ }
+
+ @Override
+ public int getKeyFieldCount() {
+ LSMBTree lsmTree = (LSMBTree) treeIndex;
+ return lsmTree.getMultiComparator().getKeyFieldCount();
+ }
+
+ @Override
+ public IBinaryComparator[] getComparators() {
+ LSMBTree lsmTree = (LSMBTree) treeIndex;
+ return lsmTree.getMultiComparator().getComparators();
+ }
+
+ /**
+ * Override to provide upsert semantics for the check tuples.
+ */
+ @Override
+ public void insertIntCheckTuple(int[] fieldValues) {
+ CheckTuple<Integer> checkTuple = new CheckTuple<Integer>(getFieldCount(), getKeyFieldCount());
+ for(int v : fieldValues) {
+ checkTuple.add(v);
+ }
+ if (checkTuples.contains(checkTuple)) {
+ checkTuples.remove(checkTuple);
+ }
+ checkTuples.add(checkTuple);
+ }
+
+ /**
+ * Override to provide upsert semantics for the check tuples.
+ */
+ @Override
+ public void insertStringCheckTuple(String[] fieldValues) {
+ CheckTuple<String> checkTuple = new CheckTuple<String>(getFieldCount(), getKeyFieldCount());
+ for(String s : fieldValues) {
+ checkTuple.add((String)s);
+ }
+ if (checkTuples.contains(checkTuple)) {
+ checkTuples.remove(checkTuple);
+ }
+ checkTuples.add(checkTuple);
+ }
+
+ public static LSMBTreeTestContext create(InMemoryBufferCache memBufferCache,
+ InMemoryFreePageManager memFreePageManager, String onDiskDir, IBufferCache diskBufferCache,
+ IFileMapProvider diskFileMapProvider, ISerializerDeserializer[] fieldSerdes, int numKeyFields, int fileId)
+ throws Exception {
+ ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
+ IBinaryComparator[] cmps = SerdeUtils.serdesToComparators(fieldSerdes, numKeyFields);
+ LSMBTree lsmTree = LSMBTreeUtils.createLSMTree(memBufferCache, memFreePageManager, onDiskDir, diskBufferCache,
+ diskFileMapProvider, typeTraits, cmps);
+ lsmTree.create(fileId);
+ lsmTree.open(fileId);
+ LSMBTreeTestContext testCtx = new LSMBTreeTestContext(fieldSerdes, lsmTree);
+ return testCtx;
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
new file mode 100644
index 0000000..b2db656
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
@@ -0,0 +1,163 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree.util;
+
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Random;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
+import edu.uci.ics.hyracks.test.support.TestUtils;
+
+public class LSMBTreeTestHarness {
+ protected static final Logger LOGGER = Logger.getLogger(LSMBTreeTestHarness.class.getName());
+
+ public static final BTreeLeafFrameType[] LEAF_FRAMES_TO_TEST = new BTreeLeafFrameType[] { BTreeLeafFrameType.REGULAR_NSM };
+
+ private static final long RANDOM_SEED = 50;
+ private static final int DEFAULT_DISK_PAGE_SIZE = 256;
+ private static final int DEFAULT_DISK_NUM_PAGES = 1000;
+ private static final int DEFAULT_DISK_MAX_OPEN_FILES = 200;
+ private static final int DEFAULT_MEM_PAGE_SIZE = 256;
+ private static final int DEFAULT_MEM_NUM_PAGES = 100;
+ //private static final int DEFAULT_MEM_NUM_PAGES = 10;
+ private static final int DEFAULT_HYRACKS_FRAME_SIZE = 128;
+ private static final int DUMMY_FILE_ID = -1;
+
+ protected final int diskPageSize;
+ protected final int diskNumPages;
+ protected final int diskMaxOpenFiles;
+ protected final int memPageSize;
+ protected final int memNumPages;
+ protected final int hyracksFrameSize;
+
+ protected IBufferCache diskBufferCache;
+ protected IFileMapProvider diskFileMapProvider;
+ protected InMemoryBufferCache memBufferCache;
+ protected InMemoryFreePageManager memFreePageManager;
+ protected IHyracksTaskContext ctx;
+
+ protected final Random rnd = new Random();
+ protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ protected final static String tmpDir = System.getProperty("java.io.tmpdir");
+ protected final static String sep = System.getProperty("file.separator");
+ protected String onDiskDir;
+
+ public LSMBTreeTestHarness() {
+ this.diskPageSize = DEFAULT_DISK_PAGE_SIZE;
+ this.diskNumPages = DEFAULT_DISK_NUM_PAGES;
+ this.diskMaxOpenFiles = DEFAULT_DISK_MAX_OPEN_FILES;
+ this.memPageSize = DEFAULT_MEM_PAGE_SIZE;
+ this.memNumPages = DEFAULT_MEM_NUM_PAGES;
+ this.hyracksFrameSize = DEFAULT_HYRACKS_FRAME_SIZE;
+ }
+
+ public LSMBTreeTestHarness(int diskPageSize, int diskNumPages,
+ int diskMaxOpenFiles, int memPageSize, int memNumPages,
+ int hyracksFrameSize) {
+ this.diskPageSize = diskPageSize;
+ this.diskNumPages = diskNumPages;
+ this.diskMaxOpenFiles = diskMaxOpenFiles;
+ this.memPageSize = memPageSize;
+ this.memNumPages = memNumPages;
+ this.hyracksFrameSize = hyracksFrameSize;
+ }
+
+ public void setUp() throws HyracksDataException {
+ onDiskDir = tmpDir + sep + "lsm_btree_" + simpleDateFormat.format(new Date()) + sep;
+ ctx = TestUtils.create(getHyracksFrameSize());
+ TestStorageManagerComponentHolder.init(diskPageSize, diskNumPages, diskMaxOpenFiles);
+ diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ diskFileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), memPageSize, memNumPages);
+ memFreePageManager = new InMemoryFreePageManager(memNumPages, new LIFOMetaDataFrameFactory());
+ rnd.setSeed(RANDOM_SEED);
+ }
+
+ public void tearDown() throws HyracksDataException {
+ diskBufferCache.close();
+ File f = new File(onDiskDir);
+ // TODO: For some reason the dir fails to be deleted. Ask Vinayak about this.
+ f.deleteOnExit();
+ }
+
+ public int getDiskPageSize() {
+ return diskPageSize;
+ }
+
+ public int getDiskNumPages() {
+ return diskNumPages;
+ }
+
+ public int getDiskMaxOpenFiles() {
+ return diskMaxOpenFiles;
+ }
+
+ public int getMemPageSize() {
+ return memPageSize;
+ }
+
+ public int getMemNumPages() {
+ return memNumPages;
+ }
+
+ public int getHyracksFrameSize() {
+ return hyracksFrameSize;
+ }
+
+ public int getFileId() {
+ return DUMMY_FILE_ID;
+ }
+
+ public IBufferCache getDiskBufferCache() {
+ return diskBufferCache;
+ }
+
+ public IFileMapProvider getDiskFileMapProvider() {
+ return diskFileMapProvider;
+ }
+
+ public InMemoryBufferCache getMemBufferCache() {
+ return memBufferCache;
+ }
+
+ public InMemoryFreePageManager getMemFreePageManager() {
+ return memFreePageManager;
+ }
+
+ public IHyracksTaskContext getHyracksTastContext() {
+ return ctx;
+ }
+
+ public String getOnDiskDir() {
+ return onDiskDir;
+ }
+
+ public Random getRandom() {
+ return rnd;
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-common-test/pom.xml b/hyracks-tests/hyracks-storage-am-lsm-common-test/pom.xml
new file mode 100644
index 0000000..092d3b6
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-common-test/pom.xml
@@ -0,0 +1,55 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-common-test</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-tests</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <type>jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-control-nc</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-common</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-test-support</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/InMemoryBufferCacheTest.java b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/InMemoryBufferCacheTest.java
new file mode 100644
index 0000000..2238e64
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/InMemoryBufferCacheTest.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.util.HashSet;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+
+public class InMemoryBufferCacheTest{
+ private static final int PAGE_SIZE = 256;
+ private static final int NUM_PAGES = 100;
+ private HashSet<ICachedPage> pinnedPages = new HashSet<ICachedPage>();
+
+ @Test
+ public void test01() throws Exception {
+ InMemoryBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), PAGE_SIZE, NUM_PAGES);
+ int dummyFileId = 0;
+ // Pin all pages, and make sure they return unique ICachedPages.
+ // We expect no overflow pages.
+ for (int i = 0; i < NUM_PAGES; i++) {
+ ICachedPage page = memBufferCache.pin(BufferedFileHandle.getDiskPageId(dummyFileId, i), false);
+ if (pinnedPages.contains(page)) {
+ fail("Id collision for ICachedPage, caused by id: " + i);
+ }
+ pinnedPages.add(page);
+ assertEquals(0, memBufferCache.getNumOverflowPages());
+ }
+ // Pin pages above capacity. We expect to be given new overflow pages.
+ // Going above capacity should be very rare, but nevertheless succeed.
+ for (int i = 0; i < 100; i++) {
+ ICachedPage page = memBufferCache.pin(BufferedFileHandle.getDiskPageId(dummyFileId, i + NUM_PAGES), false);
+ if (pinnedPages.contains(page)) {
+ fail("Id collision for ICachedPage, caused by overflow id: " + i);
+ }
+ pinnedPages.add(page);
+ assertEquals(i + 1, memBufferCache.getNumOverflowPages());
+ }
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/InMemoryFreePageManagerTest.java b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/InMemoryFreePageManagerTest.java
new file mode 100644
index 0000000..bd09a3f
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/InMemoryFreePageManagerTest.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
+
+public class InMemoryFreePageManagerTest {
+
+ private final int NUM_PAGES = 100;
+
+ private void testInMemoryFreePageManager(InMemoryFreePageManager memFreePageManager) throws HyracksDataException {
+ // The first two pages are reserved for the BTree's metadata page and
+ // root page.
+ // The "actual" capacity is therefore numPages - 2.
+ int capacity = memFreePageManager.getCapacity();
+ assertEquals(capacity, NUM_PAGES - 2);
+ for (int i = 0; i < capacity; i++) {
+ int pageId = memFreePageManager.getFreePage(null);
+ // The free pages start from page 2;
+ assertEquals(i + 2, pageId);
+ assertFalse(memFreePageManager.isFull());
+ }
+ // Start asking for 100 pages above the capacity.
+ // Asking for pages above the capacity should be very rare, but
+ // nevertheless succeed.
+ // We expect isFull() to return true.
+ for (int i = 0; i < 100; i++) {
+ int pageId = memFreePageManager.getFreePage(null);
+ assertEquals(capacity + i + 2, pageId);
+ assertTrue(memFreePageManager.isFull());
+ }
+ }
+
+ @Test
+ public void test01() throws HyracksDataException {
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+ InMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(NUM_PAGES, metaFrameFactory);
+ testInMemoryFreePageManager(memFreePageManager);
+ // We expect exactly the same behavior after a reset().
+ memFreePageManager.reset();
+ testInMemoryFreePageManager(memFreePageManager);
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/LSMTreeFileNameManagerTest.java b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/LSMTreeFileNameManagerTest.java
new file mode 100644
index 0000000..d43f34f
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/LSMTreeFileNameManagerTest.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFileNameManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeFileNameManager;
+
+public class LSMTreeFileNameManagerTest {
+ protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ protected final static String tmpDir = System.getProperty("java.io.tmpdir");
+ protected final static String sep = System.getProperty("file.separator");
+ protected String baseDir;
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ baseDir = tmpDir + sep + "lsm_tree" + simpleDateFormat.format(new Date()) + sep;
+ }
+
+ public void flushFileNamesTest(boolean testFlushFileName) throws InterruptedException {
+ ILSMFileNameManager fileNameManager = new LSMTreeFileNameManager(baseDir);
+ LinkedList<String> fileNames = new LinkedList<String>();
+
+ int numFileNames = 100;
+ long sleepTime = 5;
+ for (int i = 0; i < numFileNames; i++) {
+ String fileName = null;
+ if (testFlushFileName) {
+ fileName = fileNameManager.getFlushFileName();
+ } else {
+ fileName = fileNameManager.getMergeFileName();
+ }
+ fileNames.addFirst(fileName);
+ Thread.sleep(sleepTime);
+ }
+
+ List<String> sortedFileNames = new ArrayList<String>();
+ sortedFileNames.addAll(fileNames);
+
+ // Make sure the comparator sorts in the correct order (i.e., the
+ // reverse insertion order in this case).
+ Comparator<String> cmp = fileNameManager.getFileNameComparator();
+ Collections.sort(sortedFileNames, cmp);
+ for (int i = 0; i < numFileNames; i++) {
+ assertEquals(fileNames.get(i), sortedFileNames.get(i));
+ }
+ }
+
+ @Test
+ public void individualFileNamesTest() throws InterruptedException {
+ flushFileNamesTest(true);
+ flushFileNamesTest(false);
+ }
+
+ @Test
+ public void mixedFileNamesTest() throws InterruptedException {
+ ILSMFileNameManager fileNameManager = new LSMTreeFileNameManager(baseDir);
+ List<String> fileNames = new ArrayList<String>();
+ HashSet<String> flushFileNames = new HashSet<String>();
+ HashSet<String> mergeFileNames = new HashSet<String>();
+
+ int numFileNames = 100;
+ long sleepTime = 5;
+ for (int i = 0; i < numFileNames; i++) {
+ String fileName = null;
+ if (i % 2 == 0) {
+ fileName = fileNameManager.getFlushFileName();
+ flushFileNames.add(fileName);
+ } else {
+ fileName = fileNameManager.getMergeFileName();
+ mergeFileNames.add(fileName);
+ }
+ fileNames.add(fileName);
+ Thread.sleep(sleepTime);
+ }
+
+ // Make sure the comparator sorts in the correct order.
+ // We only need to check whether the flushed files and merged files are
+ // clustered.
+ // We verified the secondary sort order (based on timestamp) in the
+ // individualFileNamesTest().
+ Comparator<String> cmp = fileNameManager.getFileNameComparator();
+ Collections.sort(fileNames, cmp);
+ // We expect flush file names at the front.
+ int i = 0;
+ while (flushFileNames.contains(fileNames.get(i))) {
+ i++;
+ }
+ // We expect only merge file names from here on.
+ while (i < numFileNames) {
+ if (!mergeFileNames.contains(fileNames.get(i))) {
+ fail("Expected a merge file name at position: " + i);
+ }
+ i++;
+ }
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/pom.xml b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/pom.xml
new file mode 100644
index 0000000..d39acf3
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/pom.xml
@@ -0,0 +1,55 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-rtree-test</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-tests</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <type>jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-control-nc</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-rtree</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-test-support</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/AbstractLSMRTreeTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/AbstractLSMRTreeTest.java
new file mode 100644
index 0000000..0c61186
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/AbstractLSMRTreeTest.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.rtree;
+
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Random;
+import java.util.logging.Logger;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTreeInMemoryBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTreeInMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
+import edu.uci.ics.hyracks.test.support.TestUtils;
+
+public class AbstractLSMRTreeTest {
+ protected static final Logger LOGGER = Logger.getLogger(AbstractLSMRTreeTest.class.getName());
+
+ private static final long RANDOM_SEED = 50;
+ private static final int DEFAULT_DISK_PAGE_SIZE = 256;
+ private static final int DEFAULT_DISK_NUM_PAGES = 100;
+ private static final int DEFAULT_DISK_MAX_OPEN_FILES = 100;
+ private static final int DEFAULT_MEM_PAGE_SIZE = 256;
+ private static final int DEFAULT_MEM_NUM_PAGES = 100;
+ private static final int DEFAULT_HYRACKS_FRAME_SIZE = 128;
+ private static final int DUMMY_FILE_ID = -1;
+
+ protected final int diskPageSize;
+ protected final int diskNumPages;
+ protected final int diskMaxOpenFiles;
+ protected final int memPageSize;
+ protected final int memNumPages;
+ protected final int hyracksFrameSize;
+
+ protected IBufferCache diskBufferCache;
+ protected IFileMapProvider diskFileMapProvider;
+ protected InMemoryBufferCache memBufferCache;
+ protected InMemoryFreePageManager memFreePageManager;
+ protected IHyracksTaskContext ctx;
+
+ protected final Random rnd = new Random();
+ protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ protected final static String tmpDir = System.getProperty("java.io.tmpdir");
+ protected final static String sep = System.getProperty("file.separator");
+ protected String onDiskDir;
+
+ public AbstractLSMRTreeTest() {
+ this.diskPageSize = DEFAULT_DISK_PAGE_SIZE;
+ this.diskNumPages = DEFAULT_DISK_NUM_PAGES;
+ this.diskMaxOpenFiles = DEFAULT_DISK_MAX_OPEN_FILES;
+ this.memPageSize = DEFAULT_MEM_PAGE_SIZE;
+ this.memNumPages = DEFAULT_MEM_NUM_PAGES;
+ this.hyracksFrameSize = DEFAULT_HYRACKS_FRAME_SIZE;
+ }
+
+ public AbstractLSMRTreeTest(int diskPageSize, int diskNumPages, int diskMaxOpenFiles, int memPageSize,
+ int memNumPages, int hyracksFrameSize) {
+ this.diskPageSize = diskPageSize;
+ this.diskNumPages = diskNumPages;
+ this.diskMaxOpenFiles = diskMaxOpenFiles;
+ this.memPageSize = memPageSize;
+ this.memNumPages = memNumPages;
+ this.hyracksFrameSize = hyracksFrameSize;
+ }
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ onDiskDir = tmpDir + sep + "lsm_rtree_" + simpleDateFormat.format(new Date()) + sep;
+ ctx = TestUtils.create(getHyracksFrameSize());
+ TestStorageManagerComponentHolder.init(diskPageSize, diskNumPages, diskMaxOpenFiles);
+ diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ diskFileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ memBufferCache = new LSMRTreeInMemoryBufferCache(new HeapBufferAllocator(), getMemPageSize(), getMemNumPages());
+ memFreePageManager = new LSMRTreeInMemoryFreePageManager(memNumPages, new LIFOMetaDataFrameFactory());
+ rnd.setSeed(RANDOM_SEED);
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ diskBufferCache.close();
+ File f = new File(onDiskDir);
+ // TODO: For some reason the dir fails to be deleted. Ask Vinayak about
+ // this.
+ f.deleteOnExit();
+ }
+
+ public int getDiskPageSize() {
+ return diskPageSize;
+ }
+
+ public int getDiskNumPages() {
+ return diskNumPages;
+ }
+
+ public int getDiskMaxOpenFiles() {
+ return diskMaxOpenFiles;
+ }
+
+ public int getMemPageSize() {
+ return memPageSize;
+ }
+
+ public int getMemNumPages() {
+ return memNumPages;
+ }
+
+ public int getHyracksFrameSize() {
+ return hyracksFrameSize;
+ }
+
+ public int getFileId() {
+ return DUMMY_FILE_ID;
+ }
+
+ public IBufferCache getDiskBufferCache() {
+ return diskBufferCache;
+ }
+
+ public IFileMapProvider getDiskFileMapProvider() {
+ return diskFileMapProvider;
+ }
+
+ public InMemoryBufferCache getMemBufferCache() {
+ return memBufferCache;
+ }
+
+ public InMemoryFreePageManager getMemFreePageManager() {
+ return memFreePageManager;
+ }
+
+ public IHyracksTaskContext getHyracksTastContext() {
+ return ctx;
+ }
+
+ public String getOnDiskDir() {
+ return onDiskDir;
+ }
+
+ public Random getRandom() {
+ return rnd;
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeSerachTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeSerachTest.java
new file mode 100644
index 0000000..6fe5ad3
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeSerachTest.java
@@ -0,0 +1,248 @@
+package edu.uci.ics.hyracks.storage.am.lsm.rtree;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Random;
+import java.util.logging.Level;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFileNameManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeFileNameManager;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.BTreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTree;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTreeInMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTreeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.RTreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.tuples.LSMTypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.SearchPredicate;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
+
+public class LSMRTreeSerachTest extends AbstractLSMRTreeTest {
+
+ // create LSM-RTree of two dimensions
+ // fill the tree with random values using insertions
+ // and then perform range search
+ @Test
+ public void Test1() throws Exception {
+
+ // declare r-tree keys
+ int rtreeKeyFieldCount = 4;
+ IBinaryComparator[] rtreeCmps = new IBinaryComparator[rtreeKeyFieldCount];
+ rtreeCmps[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
+ rtreeCmps[1] = rtreeCmps[0];
+ rtreeCmps[2] = rtreeCmps[0];
+ rtreeCmps[3] = rtreeCmps[0];
+
+ // declare b-tree keys
+ int btreeKeyFieldCount = 5;
+ IBinaryComparator[] btreeCmps = new IBinaryComparator[btreeKeyFieldCount];
+ btreeCmps[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
+ btreeCmps[1] = btreeCmps[0];
+ btreeCmps[2] = btreeCmps[0];
+ btreeCmps[3] = btreeCmps[0];
+ btreeCmps[4] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+
+ // declare tuple fields
+ int fieldCount = 5;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = DoublePointable.TYPE_TRAITS;
+ typeTraits[1] = DoublePointable.TYPE_TRAITS;
+ typeTraits[2] = DoublePointable.TYPE_TRAITS;
+ typeTraits[3] = DoublePointable.TYPE_TRAITS;
+ typeTraits[4] = IntegerPointable.TYPE_TRAITS;
+
+ MultiComparator rtreeCmp = new MultiComparator(rtreeCmps);
+ MultiComparator btreeCmp = new MultiComparator(btreeCmps);
+
+ // create value providers
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ rtreeCmps.length, DoublePointable.FACTORY);
+
+ LSMTypeAwareTupleWriterFactory rtreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
+ LSMTypeAwareTupleWriterFactory btreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
+
+ ITreeIndexFrameFactory rtreeInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(rtreeTupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexFrameFactory rtreeLeafFrameFactory = new RTreeNSMLeafFrameFactory(rtreeTupleWriterFactory,
+ valueProviderFactories);
+
+ ITreeIndexFrameFactory btreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(btreeTupleWriterFactory);
+ ITreeIndexFrameFactory btreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(btreeTupleWriterFactory);
+
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+
+ InMemoryFreePageManager memFreePageManager = new LSMRTreeInMemoryFreePageManager(1000, metaFrameFactory);
+
+ LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
+ metaFrameFactory);
+
+ RTreeFactory diskRTreeFactory = new RTreeFactory(diskBufferCache, freePageManagerFactory, rtreeCmp, fieldCount,
+ rtreeInteriorFrameFactory, rtreeLeafFrameFactory);
+ BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, btreeCmp, fieldCount,
+ btreeInteriorFrameFactory, btreeLeafFrameFactory);
+
+ ILSMFileNameManager fileNameManager = new LSMTreeFileNameManager(onDiskDir);
+ LSMRTree lsmRTree = new LSMRTree(memBufferCache, memFreePageManager, rtreeInteriorFrameFactory,
+ rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager,
+ diskRTreeFactory, diskBTreeFactory, diskFileMapProvider, fieldCount, rtreeCmp, btreeCmp);
+
+ lsmRTree.create(getFileId());
+ lsmRTree.open(getFileId());
+
+ ByteBuffer frame = ctx.allocateFrame();
+ FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ DataOutput dos = tb.getDataOutput();
+
+ @SuppressWarnings("rawtypes")
+ ISerializerDeserializer[] recDescSers = { DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+ RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
+
+ IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
+ accessor.reset(frame);
+
+ FrameTupleReference tuple = new FrameTupleReference();
+
+ ITreeIndexAccessor lsmTreeAccessor = lsmRTree.createAccessor();
+
+ Random rnd = new Random();
+ rnd.setSeed(50);
+
+ for (int i = 0; i < 5000; i++) {
+
+ double p1x = rnd.nextDouble();
+ double p1y = rnd.nextDouble();
+ double p2x = rnd.nextDouble();
+ double p2y = rnd.nextDouble();
+
+ int pk = rnd.nextInt();
+
+ tb.reset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(pk, dos);
+ tb.addFieldEndOffset();
+
+ appender.reset(frame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
+
+ tuple.reset(accessor, 0);
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("INSERTING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
+ }
+ }
+
+ try {
+ lsmTreeAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (int i = 0; i < 50; i++) {
+ double p1x = rnd.nextDouble();
+ double p1y = rnd.nextDouble();
+ double p2x = rnd.nextDouble();
+ double p2y = rnd.nextDouble();
+
+ int pk = rnd.nextInt();
+
+ tb.reset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(pk, dos);
+ tb.addFieldEndOffset();
+
+ appender.reset(frame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
+
+ tuple.reset(accessor, 0);
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(i + " Searching for: " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
+ }
+
+ ITreeIndexCursor searchCursor = new LSMRTreeSearchCursor();
+ SearchPredicate searchPredicate = new SearchPredicate(tuple, rtreeCmp);
+
+ lsmTreeAccessor.search(searchCursor, searchPredicate);
+
+ ArrayList<Integer> results = new ArrayList<Integer>();
+ try {
+ while (searchCursor.hasNext()) {
+ searchCursor.next();
+ ITupleReference frameTuple = searchCursor.getTuple();
+ ByteArrayInputStream inStream = new ByteArrayInputStream(frameTuple.getFieldData(4),
+ frameTuple.getFieldStart(4), frameTuple.getFieldLength(4));
+ DataInput dataIn = new DataInputStream(inStream);
+ Integer res = IntegerSerializerDeserializer.INSTANCE.deserialize(dataIn);
+ results.add(res);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ searchCursor.close();
+ }
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("There are " + results.size() + " objects that satisfy the query");
+ }
+ }
+
+ lsmRTree.close();
+ memBufferCache.close();
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeTest.java
new file mode 100644
index 0000000..867efcc
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeTest.java
@@ -0,0 +1,712 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.rtree;
+
+import java.io.DataOutput;
+import java.nio.ByteBuffer;
+import java.util.Random;
+import java.util.logging.Level;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFileNameManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeFileNameManager;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.BTreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTree;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTreeInMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.RTreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.tuples.LSMTypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
+
+public class LSMRTreeTest extends AbstractLSMRTreeTest {
+
+ // create an LSM-RTree of two dimensions
+ // fill the tree with random values using insertions
+ @Test
+ public void test01() throws Exception {
+
+ // declare r-tree keys
+ int rtreeKeyFieldCount = 4;
+ IBinaryComparator[] rtreeCmps = new IBinaryComparator[rtreeKeyFieldCount];
+ rtreeCmps[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
+ rtreeCmps[1] = rtreeCmps[0];
+ rtreeCmps[2] = rtreeCmps[0];
+ rtreeCmps[3] = rtreeCmps[0];
+
+ // declare b-tree keys
+ int btreeKeyFieldCount = 7;
+ IBinaryComparator[] btreeCmps = new IBinaryComparator[btreeKeyFieldCount];
+ btreeCmps[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
+ btreeCmps[1] = btreeCmps[0];
+ btreeCmps[2] = btreeCmps[0];
+ btreeCmps[3] = btreeCmps[0];
+ btreeCmps[4] = btreeCmps[0];
+ btreeCmps[5] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+ btreeCmps[6] = btreeCmps[0];
+
+ // declare tuple fields
+ int fieldCount = 7;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = DoublePointable.TYPE_TRAITS;
+ typeTraits[1] = DoublePointable.TYPE_TRAITS;
+ typeTraits[2] = DoublePointable.TYPE_TRAITS;
+ typeTraits[3] = DoublePointable.TYPE_TRAITS;
+ typeTraits[4] = DoublePointable.TYPE_TRAITS;
+ typeTraits[5] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[6] = DoublePointable.TYPE_TRAITS;
+
+ MultiComparator rtreeCmp = new MultiComparator(rtreeCmps);
+ MultiComparator btreeCmp = new MultiComparator(btreeCmps);
+
+ // create value providers
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ rtreeCmps.length, DoublePointable.FACTORY);
+
+ LSMTypeAwareTupleWriterFactory rtreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
+ LSMTypeAwareTupleWriterFactory btreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
+
+ ITreeIndexFrameFactory rtreeInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(rtreeTupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexFrameFactory rtreeLeafFrameFactory = new RTreeNSMLeafFrameFactory(rtreeTupleWriterFactory,
+ valueProviderFactories);
+
+ ITreeIndexFrameFactory btreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(btreeTupleWriterFactory);
+ ITreeIndexFrameFactory btreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(btreeTupleWriterFactory);
+
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+
+ InMemoryFreePageManager memFreePageManager = new LSMRTreeInMemoryFreePageManager(1000, metaFrameFactory);
+
+ LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
+ metaFrameFactory);
+
+ RTreeFactory diskRTreeFactory = new RTreeFactory(diskBufferCache, freePageManagerFactory, rtreeCmp, fieldCount,
+ rtreeInteriorFrameFactory, rtreeLeafFrameFactory);
+ BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, btreeCmp, fieldCount,
+ btreeInteriorFrameFactory, btreeLeafFrameFactory);
+
+ ILSMFileNameManager fileNameManager = new LSMTreeFileNameManager(onDiskDir);
+ LSMRTree lsmRTree = new LSMRTree(memBufferCache, memFreePageManager, rtreeInteriorFrameFactory,
+ rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager,
+ diskRTreeFactory, diskBTreeFactory, diskFileMapProvider, fieldCount, rtreeCmp, btreeCmp);
+
+ lsmRTree.create(getFileId());
+ lsmRTree.open(getFileId());
+
+ ByteBuffer frame = ctx.allocateFrame();
+ FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ DataOutput dos = tb.getDataOutput();
+
+ @SuppressWarnings("rawtypes")
+ ISerializerDeserializer[] recDescSers = { DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
+ RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
+ IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
+ accessor.reset(frame);
+ FrameTupleReference tuple = new FrameTupleReference();
+
+ ITreeIndexAccessor indexAccessor = lsmRTree.createAccessor();
+
+ Random rnd = new Random();
+ rnd.setSeed(50);
+
+ Random rnd2 = new Random();
+ rnd2.setSeed(50);
+ for (int i = 0; i < 5000; i++) {
+
+ double p1x = rnd.nextDouble();
+ double p1y = rnd.nextDouble();
+ double p2x = rnd.nextDouble();
+ double p2y = rnd.nextDouble();
+
+ double pk1 = rnd2.nextDouble();
+ int pk2 = rnd2.nextInt();
+ double pk3 = rnd2.nextDouble();
+
+ tb.reset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
+ tb.addFieldEndOffset();
+
+ appender.reset(frame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
+
+ tuple.reset(accessor, 0);
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("INSERTING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
+ }
+ }
+
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ lsmRTree.close();
+ memBufferCache.close();
+
+ }
+
+ // create an LSM-RTree of two dimensions
+ // fill the tree with random values using insertions
+ // and then delete all the tuples which result of an empty LSM-RTree
+ @Test
+ public void test02() throws Exception {
+
+ // declare r-tree keys
+ int rtreeKeyFieldCount = 4;
+ IBinaryComparator[] rtreeCmps = new IBinaryComparator[rtreeKeyFieldCount];
+ rtreeCmps[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
+ rtreeCmps[1] = rtreeCmps[0];
+ rtreeCmps[2] = rtreeCmps[0];
+ rtreeCmps[3] = rtreeCmps[0];
+
+ // declare b-tree keys
+ int btreeKeyFieldCount = 7;
+ IBinaryComparator[] btreeCmps = new IBinaryComparator[btreeKeyFieldCount];
+ btreeCmps[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
+ btreeCmps[1] = btreeCmps[0];
+ btreeCmps[2] = btreeCmps[0];
+ btreeCmps[3] = btreeCmps[0];
+ btreeCmps[4] = btreeCmps[0];
+ btreeCmps[5] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+ btreeCmps[6] = btreeCmps[0];
+
+ // declare tuple fields
+ int fieldCount = 7;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = DoublePointable.TYPE_TRAITS;
+ typeTraits[1] = DoublePointable.TYPE_TRAITS;
+ typeTraits[2] = DoublePointable.TYPE_TRAITS;
+ typeTraits[3] = DoublePointable.TYPE_TRAITS;
+ typeTraits[4] = DoublePointable.TYPE_TRAITS;
+ typeTraits[5] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[6] = DoublePointable.TYPE_TRAITS;
+
+ MultiComparator rtreeCmp = new MultiComparator(rtreeCmps);
+ MultiComparator btreeCmp = new MultiComparator(btreeCmps);
+
+ // create value providers
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ rtreeCmps.length, DoublePointable.FACTORY);
+
+ LSMTypeAwareTupleWriterFactory rtreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
+ LSMTypeAwareTupleWriterFactory btreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
+
+ ITreeIndexFrameFactory rtreeInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(rtreeTupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexFrameFactory rtreeLeafFrameFactory = new RTreeNSMLeafFrameFactory(rtreeTupleWriterFactory,
+ valueProviderFactories);
+
+ ITreeIndexFrameFactory btreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(btreeTupleWriterFactory);
+ ITreeIndexFrameFactory btreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(btreeTupleWriterFactory);
+
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+
+ InMemoryFreePageManager memFreePageManager = new LSMRTreeInMemoryFreePageManager(1000, metaFrameFactory);
+
+ LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
+ metaFrameFactory);
+
+ RTreeFactory diskRTreeFactory = new RTreeFactory(diskBufferCache, freePageManagerFactory, rtreeCmp, fieldCount,
+ rtreeInteriorFrameFactory, rtreeLeafFrameFactory);
+ BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, btreeCmp, fieldCount,
+ btreeInteriorFrameFactory, btreeLeafFrameFactory);
+
+ ILSMFileNameManager fileNameManager = new LSMTreeFileNameManager(onDiskDir);
+ LSMRTree lsmRTree = new LSMRTree(memBufferCache, memFreePageManager, rtreeInteriorFrameFactory,
+ rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager,
+ diskRTreeFactory, diskBTreeFactory, diskFileMapProvider, fieldCount, rtreeCmp, btreeCmp);
+
+ lsmRTree.create(getFileId());
+ lsmRTree.open(getFileId());
+
+ ByteBuffer frame = ctx.allocateFrame();
+ FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ DataOutput dos = tb.getDataOutput();
+
+ @SuppressWarnings("rawtypes")
+ ISerializerDeserializer[] recDescSers = { DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
+ RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
+ IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
+ accessor.reset(frame);
+ FrameTupleReference tuple = new FrameTupleReference();
+
+ ITreeIndexAccessor indexAccessor = lsmRTree.createAccessor();
+
+ Random rnd = new Random();
+ rnd.setSeed(50);
+
+ for (int i = 0; i < 5000; i++) {
+
+ double p1x = rnd.nextDouble();
+ double p1y = rnd.nextDouble();
+ double p2x = rnd.nextDouble();
+ double p2y = rnd.nextDouble();
+
+ double pk1 = rnd.nextDouble();
+ int pk2 = rnd.nextInt();
+ double pk3 = rnd.nextDouble();
+
+ tb.reset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
+ tb.addFieldEndOffset();
+
+ appender.reset(frame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
+
+ tuple.reset(accessor, 0);
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("INSERTING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
+ }
+ }
+
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ rnd.setSeed(50);
+ for (int i = 0; i < 5000; i++) {
+
+ double p1x = rnd.nextDouble();
+ double p1y = rnd.nextDouble();
+ double p2x = rnd.nextDouble();
+ double p2y = rnd.nextDouble();
+
+ double pk1 = rnd.nextDouble();
+ int pk2 = rnd.nextInt();
+ double pk3 = rnd.nextDouble();
+
+ tb.reset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
+ tb.addFieldEndOffset();
+
+ appender.reset(frame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
+
+ tuple.reset(accessor, 0);
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("DELETING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
+ }
+ }
+
+ try {
+ indexAccessor.delete(tuple);
+ } catch (TreeIndexException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ lsmRTree.close();
+ memBufferCache.close();
+
+ }
+
+ // create an LSM-RTree of three dimensions
+ // fill the tree with random values using insertions
+ @Test
+ public void test03() throws Exception {
+
+ // declare r-tree keys
+ int rtreeKeyFieldCount = 6;
+ IBinaryComparator[] rtreeCmps = new IBinaryComparator[rtreeKeyFieldCount];
+ rtreeCmps[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
+ rtreeCmps[1] = rtreeCmps[0];
+ rtreeCmps[2] = rtreeCmps[0];
+ rtreeCmps[3] = rtreeCmps[0];
+ rtreeCmps[4] = rtreeCmps[0];
+ rtreeCmps[5] = rtreeCmps[0];
+
+ // declare b-tree keys
+ int btreeKeyFieldCount = 9;
+ IBinaryComparator[] btreeCmps = new IBinaryComparator[btreeKeyFieldCount];
+ btreeCmps[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
+ btreeCmps[1] = btreeCmps[0];
+ btreeCmps[2] = btreeCmps[0];
+ btreeCmps[3] = btreeCmps[0];
+ btreeCmps[4] = btreeCmps[0];
+ btreeCmps[5] = btreeCmps[0];
+ btreeCmps[6] = btreeCmps[0];
+ btreeCmps[7] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+ btreeCmps[8] = btreeCmps[0];
+
+ // declare tuple fields
+ int fieldCount = 9;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = DoublePointable.TYPE_TRAITS;
+ typeTraits[1] = DoublePointable.TYPE_TRAITS;
+ typeTraits[2] = DoublePointable.TYPE_TRAITS;
+ typeTraits[3] = DoublePointable.TYPE_TRAITS;
+ typeTraits[4] = DoublePointable.TYPE_TRAITS;
+ typeTraits[5] = DoublePointable.TYPE_TRAITS;
+ typeTraits[6] = DoublePointable.TYPE_TRAITS;
+ typeTraits[7] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[8] = DoublePointable.TYPE_TRAITS;
+
+ MultiComparator rtreeCmp = new MultiComparator(rtreeCmps);
+ MultiComparator btreeCmp = new MultiComparator(btreeCmps);
+
+ // create value providers
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ rtreeCmps.length, DoublePointable.FACTORY);
+
+ LSMTypeAwareTupleWriterFactory rtreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
+ LSMTypeAwareTupleWriterFactory btreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
+
+ ITreeIndexFrameFactory rtreeInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(rtreeTupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexFrameFactory rtreeLeafFrameFactory = new RTreeNSMLeafFrameFactory(rtreeTupleWriterFactory,
+ valueProviderFactories);
+
+ ITreeIndexFrameFactory btreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(btreeTupleWriterFactory);
+ ITreeIndexFrameFactory btreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(btreeTupleWriterFactory);
+
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+
+ InMemoryFreePageManager memFreePageManager = new LSMRTreeInMemoryFreePageManager(1000, metaFrameFactory);
+
+ LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
+ metaFrameFactory);
+
+ RTreeFactory diskRTreeFactory = new RTreeFactory(diskBufferCache, freePageManagerFactory, rtreeCmp, fieldCount,
+ rtreeInteriorFrameFactory, rtreeLeafFrameFactory);
+ BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, btreeCmp, fieldCount,
+ btreeInteriorFrameFactory, btreeLeafFrameFactory);
+
+ ILSMFileNameManager fileNameManager = new LSMTreeFileNameManager(onDiskDir);
+ LSMRTree lsmRTree = new LSMRTree(memBufferCache, memFreePageManager, rtreeInteriorFrameFactory,
+ rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager,
+ diskRTreeFactory, diskBTreeFactory, diskFileMapProvider, fieldCount, rtreeCmp, btreeCmp);
+
+ lsmRTree.create(getFileId());
+ lsmRTree.open(getFileId());
+
+ ByteBuffer frame = ctx.allocateFrame();
+ FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ DataOutput dos = tb.getDataOutput();
+
+ @SuppressWarnings("rawtypes")
+ ISerializerDeserializer[] recDescSers = { DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
+ RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
+ IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
+ accessor.reset(frame);
+ FrameTupleReference tuple = new FrameTupleReference();
+
+ ITreeIndexAccessor indexAccessor = lsmRTree.createAccessor();
+
+ Random rnd = new Random();
+ rnd.setSeed(50);
+
+ for (int i = 0; i < 5000; i++) {
+
+ double p1x = rnd.nextDouble();
+ double p1y = rnd.nextDouble();
+ double p1z = rnd.nextDouble();
+ double p2x = rnd.nextDouble();
+ double p2y = rnd.nextDouble();
+ double p2z = rnd.nextDouble();
+
+ double pk1 = rnd.nextDouble();
+ int pk2 = rnd.nextInt();
+ double pk3 = rnd.nextDouble();
+
+ tb.reset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1z, p2z), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1z, p2z), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
+ tb.addFieldEndOffset();
+
+ appender.reset(frame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
+
+ tuple.reset(accessor, 0);
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("INSERTING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.min(p1z, p2z) + " " + " " + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y) + " "
+ + Math.max(p1z, p2z));
+ }
+ }
+
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ lsmRTree.close();
+ memBufferCache.close();
+
+ }
+
+ // create an LSM-RTree of two dimensions
+ // fill the tree with random integer key values using insertions
+ @Test
+ public void test04() throws Exception {
+
+ // declare r-tree keys
+ int rtreeKeyFieldCount = 4;
+ IBinaryComparator[] rtreeCmps = new IBinaryComparator[rtreeKeyFieldCount];
+ rtreeCmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+ rtreeCmps[1] = rtreeCmps[0];
+ rtreeCmps[2] = rtreeCmps[0];
+ rtreeCmps[3] = rtreeCmps[0];
+
+ // declare b-tree keys
+ int btreeKeyFieldCount = 7;
+ IBinaryComparator[] btreeCmps = new IBinaryComparator[btreeKeyFieldCount];
+ btreeCmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+ btreeCmps[1] = btreeCmps[0];
+ btreeCmps[2] = btreeCmps[0];
+ btreeCmps[3] = btreeCmps[0];
+ btreeCmps[4] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
+ btreeCmps[5] = btreeCmps[0];
+ btreeCmps[6] = btreeCmps[4];
+
+ // declare tuple fields
+ int fieldCount = 7;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[3] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[4] = DoublePointable.TYPE_TRAITS;
+ typeTraits[5] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[6] = DoublePointable.TYPE_TRAITS;
+
+ MultiComparator rtreeCmp = new MultiComparator(rtreeCmps);
+ MultiComparator btreeCmp = new MultiComparator(btreeCmps);
+
+ // create value providers
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ rtreeCmps.length, IntegerPointable.FACTORY);
+
+ LSMTypeAwareTupleWriterFactory rtreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
+ LSMTypeAwareTupleWriterFactory btreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
+
+ ITreeIndexFrameFactory rtreeInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(rtreeTupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexFrameFactory rtreeLeafFrameFactory = new RTreeNSMLeafFrameFactory(rtreeTupleWriterFactory,
+ valueProviderFactories);
+
+ ITreeIndexFrameFactory btreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(btreeTupleWriterFactory);
+ ITreeIndexFrameFactory btreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(btreeTupleWriterFactory);
+
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+
+ InMemoryFreePageManager memFreePageManager = new LSMRTreeInMemoryFreePageManager(1000, metaFrameFactory);
+
+ LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
+ metaFrameFactory);
+
+ RTreeFactory diskRTreeFactory = new RTreeFactory(diskBufferCache, freePageManagerFactory, rtreeCmp, fieldCount,
+ rtreeInteriorFrameFactory, rtreeLeafFrameFactory);
+ BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, btreeCmp, fieldCount,
+ btreeInteriorFrameFactory, btreeLeafFrameFactory);
+
+ ILSMFileNameManager fileNameManager = new LSMTreeFileNameManager(onDiskDir);
+ LSMRTree lsmRTree = new LSMRTree(memBufferCache, memFreePageManager, rtreeInteriorFrameFactory,
+ rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager,
+ diskRTreeFactory, diskBTreeFactory, diskFileMapProvider, fieldCount, rtreeCmp, btreeCmp);
+
+ lsmRTree.create(getFileId());
+ lsmRTree.open(getFileId());
+
+ ByteBuffer frame = ctx.allocateFrame();
+ FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ DataOutput dos = tb.getDataOutput();
+
+ @SuppressWarnings("rawtypes")
+ ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
+ RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
+ IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
+ accessor.reset(frame);
+ FrameTupleReference tuple = new FrameTupleReference();
+
+ ITreeIndexAccessor indexAccessor = lsmRTree.createAccessor();
+
+ Random rnd = new Random();
+ rnd.setSeed(50);
+
+ Random rnd2 = new Random();
+ rnd2.setSeed(50);
+ for (int i = 0; i < 5000; i++) {
+
+ int p1x = rnd.nextInt();
+ int p1y = rnd.nextInt();
+ int p2x = rnd.nextInt();
+ int p2y = rnd.nextInt();
+
+ double pk1 = rnd2.nextDouble();
+ int pk2 = rnd2.nextInt();
+ double pk3 = rnd2.nextDouble();
+
+ tb.reset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
+ tb.addFieldEndOffset();
+
+ appender.reset(frame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
+
+ tuple.reset(accessor, 0);
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("INSERTING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
+ }
+ }
+
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ lsmRTree.close();
+ memBufferCache.close();
+
+ }
+}
\ No newline at end of file
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTest.java b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTest.java
index 9fe2c94..bb744c9 100644
--- a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTest.java
+++ b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTest.java
@@ -102,7 +102,7 @@
typeTraits[2] = DoublePointable.TYPE_TRAITS;
typeTraits[3] = DoublePointable.TYPE_TRAITS;
typeTraits[4] = DoublePointable.TYPE_TRAITS;
- typeTraits[5] = DoublePointable.TYPE_TRAITS;
+ typeTraits[5] = IntegerPointable.TYPE_TRAITS;
typeTraits[6] = DoublePointable.TYPE_TRAITS;
// create value providers
@@ -267,7 +267,7 @@
typeTraits[2] = DoublePointable.TYPE_TRAITS;
typeTraits[3] = DoublePointable.TYPE_TRAITS;
typeTraits[4] = DoublePointable.TYPE_TRAITS;
- typeTraits[5] = DoublePointable.TYPE_TRAITS;
+ typeTraits[5] = IntegerPointable.TYPE_TRAITS;
typeTraits[6] = DoublePointable.TYPE_TRAITS;
// create value providers
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/SearchCursorTest.java b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/SearchCursorTest.java
index a232e37..2eac485 100644
--- a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/SearchCursorTest.java
+++ b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/SearchCursorTest.java
@@ -120,7 +120,6 @@
ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(tupleWriterFactory,
valueProviderFactories);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
IRTreeInteriorFrame interiorFrame = (IRTreeInteriorFrame) interiorFrameFactory.createFrame();
IRTreeLeafFrame leafFrame = (IRTreeLeafFrame) leafFrameFactory.createFrame();
diff --git a/hyracks-tests/pom.xml b/hyracks-tests/pom.xml
index b120ba82..4596297 100644
--- a/hyracks-tests/pom.xml
+++ b/hyracks-tests/pom.xml
@@ -16,5 +16,8 @@
<module>hyracks-storage-am-btree-test</module>
<module>hyracks-storage-am-invertedindex-test</module>
<module>hyracks-storage-am-rtree-test</module>
+ <module>hyracks-storage-am-lsm-common-test</module>
+ <module>hyracks-storage-am-lsm-btree-test</module>
+ <module>hyracks-storage-am-lsm-rtree-test</module>
</modules>
</project>
diff --git a/pom.xml b/pom.xml
index 0ab0646..ef46dab 100644
--- a/pom.xml
+++ b/pom.xml
@@ -94,6 +94,9 @@
<module>hyracks-storage-am-common</module>
<module>hyracks-storage-am-btree</module>
<module>hyracks-storage-am-invertedindex</module>
+ <module>hyracks-storage-am-lsm-common</module>
+ <module>hyracks-storage-am-lsm-btree</module>
+ <module>hyracks-storage-am-lsm-rtree</module>
<module>hyracks-storage-am-rtree</module>
<module>hyracks-test-support</module>
<module>hyracks-tests</module>