Merged hyracks_dev_next into this branch.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_lsm_tree@1032 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/SerdeUtils.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/SerdeUtils.java
index 87d9b35..00575f4 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/SerdeUtils.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/SerdeUtils.java
@@ -35,13 +35,41 @@
@SuppressWarnings("rawtypes")
public class SerdeUtils {
- public static ITypeTraits[] serdesToTypeTraits(ISerializerDeserializer[] serdes, int numSerdes) {
- ITypeTraits[] typeTraits = new ITypeTraits[numSerdes];
- for (int i = 0; i < numSerdes; i++) {
+ public static class PayloadTypeTraits implements ITypeTraits {
+ private static final long serialVersionUID = 1L;
+ final int payloadSize;
+
+ public PayloadTypeTraits(int payloadSize) {
+ this.payloadSize = payloadSize;
+ }
+
+ @Override
+ public boolean isFixedLength() {
+ return true;
+ }
+
+ @Override
+ public int getFixedLength() {
+ return payloadSize;
+ }
+ }
+
+ public static ITypeTraits[] serdesToTypeTraits(ISerializerDeserializer[] serdes) {
+ ITypeTraits[] typeTraits = new ITypeTraits[serdes.length];
+ for (int i = 0; i < serdes.length; i++) {
typeTraits[i] = serdeToTypeTrait(serdes[i]);
}
return typeTraits;
}
+
+ public static ITypeTraits[] serdesToTypeTraits(ISerializerDeserializer[] serdes, int payloadSize) {
+ ITypeTraits[] typeTraits = new ITypeTraits[serdes.length + 1];
+ for (int i = 0; i < serdes.length; i++) {
+ typeTraits[i] = serdeToTypeTrait(serdes[i]);
+ }
+ typeTraits[serdes.length] = new PayloadTypeTraits(payloadSize);
+ return typeTraits;
+ }
public static ITypeTraits serdeToTypeTrait(ISerializerDeserializer serde) {
if (serde instanceof IntegerSerializerDeserializer) {
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
index 4113e9f..edcc6b2 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
@@ -1128,7 +1128,9 @@
return new BTreeAccessor(this);
}
- private class BTreeAccessor implements ITreeIndexAccessor {
+ // TODO: Class should be private. But currently we need to expose the
+ // setOpContext() API to the LSM Tree for it to work correctly.
+ public class BTreeAccessor implements ITreeIndexAccessor {
private BTree btree;
private BTreeOpContext ctx;
@@ -1167,5 +1169,13 @@
ctx.reset(IndexOp.DISKORDERSCAN);
btree.diskOrderScan(cursor, ctx);
}
+
+ // TODO: Ideally, this method should not exist. But we need it for
+ // the LSM tree to work correctly, so we can use the LSMOpContext inside
+ // a BTreeAccessor.
+ // Making the appropriate change will involve changing lots of code.
+ public void setOpContext(BTreeOpContext ctx) {
+ this.ctx = ctx;
+ }
}
}
diff --git a/hyracks-storage-am-lsmtree-btree/pom.xml b/hyracks-storage-am-lsmtree-btree/pom.xml
new file mode 100644
index 0000000..4175e72
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/pom.xml
@@ -0,0 +1,49 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsmtree-btree</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-test-support</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-btree</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <type>jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/DataGenThread.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/DataGenThread.java
new file mode 100644
index 0000000..494534d
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/DataGenThread.java
@@ -0,0 +1,86 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.datagen;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+
+/**
+ * Quick & dirty data generator for performance testing.
+ *
+ */
+public class DataGenThread extends Thread {
+ public final BlockingQueue<TupleBatch> tupleBatchQueue;
+ private final int maxNumBatches;
+ private final int maxOutstandingBatches;
+ private int numBatches;
+ private final boolean sorted;
+ private final Random rnd;
+
+ // maxOutstandingBatches pre-created tuple-batches for populating the queue.
+ private TupleBatch[] tupleBatches;
+ private int ringPos;
+
+ public DataGenThread(int maxNumBatches, int batchSize, int maxOutstandingBatches, int numConsumers, ISerializerDeserializer[] fieldSerdes, int payloadSize, int rndSeed, boolean sorted) {
+ this.maxNumBatches = maxNumBatches;
+ this.maxOutstandingBatches = maxOutstandingBatches;
+ this.sorted = sorted;
+ rnd = new Random(rndSeed);
+ tupleBatches = new TupleBatch[maxOutstandingBatches];
+ IFieldValueGenerator[] fieldGens = new IFieldValueGenerator[fieldSerdes.length];
+ for (int i = 0; i < fieldSerdes.length; i++) {
+ fieldGens[i] = getFieldGenFromSerde(fieldSerdes[i]);
+ }
+ for (int i = 0; i < maxOutstandingBatches; i++) {
+ tupleBatches[i] = new TupleBatch(batchSize, fieldGens, fieldSerdes, payloadSize);
+ }
+ // make sure we don't overwrite tuples that are in use by consumers.
+ // -1 because we first generate a new tuple, and then try to put it into the queue.
+ int capacity = Math.max(maxOutstandingBatches - numConsumers - 1, 1);
+ tupleBatchQueue = new LinkedBlockingQueue<TupleBatch>(capacity);
+ ringPos = 0;
+ }
+
+ @Override
+ public void run() {
+ while(numBatches < maxNumBatches) {
+ try {
+ tupleBatches[ringPos].generate();
+ tupleBatchQueue.put(tupleBatches[ringPos]);
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ numBatches++;
+ ringPos++;
+ if (ringPos >= maxOutstandingBatches) {
+ ringPos = 0;
+ }
+ }
+ }
+
+ public IFieldValueGenerator getFieldGenFromSerde(ISerializerDeserializer serde) {
+ if (serde instanceof IntegerSerializerDeserializer) {
+ if (sorted) {
+ return new SortedIntegerFieldValueGenerator();
+ } else {
+ return new IntegerFieldValueGenerator(rnd);
+ }
+ }
+ System.out.println("NULL");
+ //if (serde instanceof Integer64SerializerDeserializer) {
+ // throw new UnsupportedOperationException("Binary comparator factory for Integer64 not implemented.");
+ //}
+ //if (serde instanceof FloatSerializerDeserializer) {
+ // return FloatBinaryComparatorFactory.INSTANCE;
+ //}
+ //if (serde instanceof DoubleSerializerDeserializer) {
+ // return DoubleBinaryComparatorFactory.INSTANCE;
+ //}
+ return null;
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/IFieldValueGenerator.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/IFieldValueGenerator.java
new file mode 100644
index 0000000..32e6ab3
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/IFieldValueGenerator.java
@@ -0,0 +1,5 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.datagen;
+
+public interface IFieldValueGenerator<T> {
+ public T next();
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/IntegerFieldValueGenerator.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/IntegerFieldValueGenerator.java
new file mode 100644
index 0000000..aa21959
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/IntegerFieldValueGenerator.java
@@ -0,0 +1,16 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.datagen;
+
+import java.util.Random;
+
+public class IntegerFieldValueGenerator implements IFieldValueGenerator<Integer> {
+ protected final Random rnd;
+
+ public IntegerFieldValueGenerator(Random rnd) {
+ this.rnd = rnd;
+ }
+
+ @Override
+ public Integer next() {
+ return rnd.nextInt();
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/SortedIntegerFieldValueGenerator.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/SortedIntegerFieldValueGenerator.java
new file mode 100644
index 0000000..cf42490
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/SortedIntegerFieldValueGenerator.java
@@ -0,0 +1,17 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.datagen;
+
+public class SortedIntegerFieldValueGenerator implements IFieldValueGenerator<Integer> {
+ private int val = 0;
+
+ public SortedIntegerFieldValueGenerator() {
+ }
+
+ public SortedIntegerFieldValueGenerator(int startVal) {
+ val = startVal;
+ }
+
+ @Override
+ public Integer next() {
+ return val++;
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/TupleBatch.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/TupleBatch.java
new file mode 100644
index 0000000..7363854
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/TupleBatch.java
@@ -0,0 +1,33 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.datagen;
+
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+public class TupleBatch {
+ private final int size;
+ private final TupleGenerator[] tupleGens;
+
+ public TupleBatch(int size, IFieldValueGenerator[] fieldGens, ISerializerDeserializer[] fieldSerdes, int payloadSize) {
+ this.size = size;
+ tupleGens = new TupleGenerator[size];
+ for (int i = 0; i < size; i++) {
+ tupleGens[i] = new TupleGenerator(fieldGens, fieldSerdes, payloadSize);
+ }
+ }
+
+ public void generate() throws IOException {
+ for(TupleGenerator tupleGen : tupleGens) {
+ tupleGen.next();
+ }
+ }
+
+ public int size() {
+ return size;
+ }
+
+ public ITupleReference get(int ix) {
+ return tupleGens[ix].get();
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/TupleGenerator.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/TupleGenerator.java
new file mode 100644
index 0000000..c035f9d
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/datagen/TupleGenerator.java
@@ -0,0 +1,51 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.datagen;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+@SuppressWarnings({"rawtypes", "unchecked" })
+public class TupleGenerator {
+ protected final ISerializerDeserializer[] fieldSerdes;
+ protected final IFieldValueGenerator[] fieldGens;
+ protected final ArrayTupleBuilder tb;
+ protected final ArrayTupleReference tuple;
+ protected final byte[] payload;
+ protected final DataOutput tbDos;
+
+ public TupleGenerator(IFieldValueGenerator[] fieldGens, ISerializerDeserializer[] fieldSerdes, int payloadSize) {
+ this.fieldSerdes = fieldSerdes;
+ this.fieldGens = fieldGens;
+ tuple = new ArrayTupleReference();
+ if (payloadSize > 0) {
+ tb = new ArrayTupleBuilder(fieldSerdes.length + 1);
+ payload = new byte[payloadSize];
+ } else {
+ tb = new ArrayTupleBuilder(fieldSerdes.length);
+ payload = null;
+ }
+ tbDos = tb.getDataOutput();
+ }
+
+ public ITupleReference next() throws IOException {
+ tb.reset();
+ for (int i = 0; i < fieldSerdes.length; i++) {
+ fieldSerdes[i].serialize(fieldGens[i].next(), tbDos);
+ tb.addFieldEndOffset();
+ }
+ if (payload != null) {
+ tbDos.write(payload);
+ tb.addFieldEndOffset();
+ }
+ tuple.reset(tb.getFieldEndOffsets(), tb.getByteArray());
+ return tuple;
+ }
+
+ public ITupleReference get() {
+ return tuple;
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/FreePageManagerFactory.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/FreePageManagerFactory.java
new file mode 100644
index 0000000..4285490
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/FreePageManagerFactory.java
@@ -0,0 +1,23 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.freepage;
+
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+
+public class FreePageManagerFactory {
+
+ private final ITreeIndexMetaDataFrameFactory metaDataFrameFactory;
+ private final IBufferCache bufferCache;
+
+ public FreePageManagerFactory(IBufferCache bufferCache, ITreeIndexMetaDataFrameFactory metaDataFrameFactory) {
+ this.metaDataFrameFactory = metaDataFrameFactory;
+ this.bufferCache = bufferCache;
+ }
+
+ public IFreePageManager createFreePageManager(int fileId) {
+ return new LinkedListFreePageManager(bufferCache, fileId, 0, metaDataFrameFactory);
+ }
+
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/InMemoryBufferCache.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/InMemoryBufferCache.java
new file mode 100644
index 0000000..c523440
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/InMemoryBufferCache.java
@@ -0,0 +1,146 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.freepage;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCacheInternal;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPageInternal;
+import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+
+public class InMemoryBufferCache implements IBufferCacheInternal {
+
+ private final int pageSize;
+ private final int numPages;
+ private final CachedPage[] cachedPages;
+
+ //Constructor
+ public InMemoryBufferCache(ICacheMemoryAllocator allocator, int pageSize, int numPages){
+
+ this.pageSize = pageSize;
+ this.numPages = numPages;
+ ByteBuffer[] buffers = allocator.allocate(this.pageSize, this.numPages);
+ cachedPages = new CachedPage[buffers.length];
+ for (int i = 0; i < buffers.length; ++i) {
+ cachedPages[i] = new CachedPage(i, buffers[i]);
+ }
+ }
+
+ @Override
+ public void createFile(FileReference fileRef) throws HyracksDataException {
+ // Do nothing
+ }
+
+ @Override
+ public void openFile(int fileId) throws HyracksDataException {
+ // Do nothing
+ }
+
+ @Override
+ public void closeFile(int fileId) throws HyracksDataException {
+ // Do nothing
+ }
+
+ @Override
+ public void deleteFile(int fileId) throws HyracksDataException {
+ // Do nothing
+ }
+
+ @Override
+ public ICachedPage tryPin(long dpid) throws HyracksDataException {
+ // Just call pin!
+ return null;
+ }
+
+ @Override
+ public ICachedPage pin(long dpid, boolean newPage){
+ return cachedPages[BufferedFileHandle.getPageId(dpid)];
+ }
+
+ @Override
+ public void unpin(ICachedPage page) throws HyracksDataException {
+ //Do Nothing
+ }
+
+ @Override
+ public int getPageSize() {
+ return pageSize;
+ }
+
+ @Override
+ public int getNumPages() {
+ return numPages;
+ }
+
+ @Override
+ public void close() {
+ // Do nothing
+ }
+
+ @Override
+ public ICachedPageInternal getPage(int cpid) {
+ return cachedPages[cpid];
+ }
+
+ private class CachedPage implements ICachedPageInternal {
+ private final int cpid;
+ private final ByteBuffer buffer;
+ private final ReadWriteLock latch;
+
+ public CachedPage(int cpid, ByteBuffer buffer) {
+ this.cpid = cpid;
+ this.buffer = buffer;
+ latch = new ReentrantReadWriteLock(true);
+ }
+
+ @Override
+ public ByteBuffer getBuffer() {
+ return buffer;
+ }
+
+ @Override
+ public Object getReplacementStrategyObject() {
+ //Do nothing
+ return null;
+ }
+
+ @Override
+ public boolean pinIfGoodVictim() {
+ //Do nothing
+ return false;
+ }
+
+ @Override
+ public int getCachedPageId() {
+ return cpid;
+ }
+
+ @Override
+ public void acquireReadLatch() {
+ latch.readLock().lock();
+ }
+
+ private void acquireWriteLatch(boolean markDirty) {
+ latch.writeLock().lock();
+ }
+
+ @Override
+ public void acquireWriteLatch() {
+ acquireWriteLatch(true);
+ }
+
+ @Override
+ public void releaseReadLatch() {
+ latch.readLock().unlock();
+ }
+
+ @Override
+ public void releaseWriteLatch() {
+ latch.writeLock().unlock();
+ }
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/InMemoryBufferCacheFactory.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/InMemoryBufferCacheFactory.java
new file mode 100644
index 0000000..07e1c15
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/InMemoryBufferCacheFactory.java
@@ -0,0 +1,26 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.freepage;
+
+import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
+
+public class InMemoryBufferCacheFactory {
+
+ private IBufferCache bufferCache;
+ private final int pageSize;
+ private final int numPages;
+
+ public InMemoryBufferCacheFactory(int pageSize, int numPages) {
+ this.pageSize = pageSize;
+ this.numPages = numPages;
+ bufferCache = null;
+ }
+
+ public synchronized IBufferCache createInMemoryBufferCache() {
+ if (bufferCache == null) {
+ ICacheMemoryAllocator allocator = new HeapBufferAllocator();
+ bufferCache = new InMemoryBufferCache(allocator, pageSize, numPages);
+ }
+ return bufferCache;
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/InMemoryBufferCacheTest.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/InMemoryBufferCacheTest.java
new file mode 100644
index 0000000..418ed5c
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/InMemoryBufferCacheTest.java
@@ -0,0 +1,117 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.freepage;
+
+import static org.junit.Assert.fail;
+
+import org.junit.*;
+
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCacheInternal;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+
+public class InMemoryBufferCacheTest{
+
+ private static final int PAGE_SIZE = 256;
+ private static final int NUM_PAGES = 10;
+
+ //TEST InMemoryBufferCache.pin()
+ @Test
+ public void InMemoryBufferCacheTest01() throws Exception {
+
+ InMemoryBufferCacheFactory InMemBufferCacheFactory = new InMemoryBufferCacheFactory(PAGE_SIZE, NUM_PAGES);
+ IBufferCache memBufferCache = InMemBufferCacheFactory.createInMemoryBufferCache();
+
+ try {
+ ICachedPage memCachedPage = memBufferCache.pin(10, true);
+ if(memCachedPage != null) {
+ return;
+ }
+ else {
+ fail("fail to pin");
+ }
+ }
+ catch (ArrayIndexOutOfBoundsException e){
+ System.out.println("Catch exception: " + e);
+ return;
+ }
+ catch (Exception e) {
+ fail("Unexpected exception!");
+ }
+ }
+
+ //TEST InMemoryBufferCache.pin()
+ //expect ArrayIndexOutOfBoundsException
+ @Test
+ public void InMemoryBufferCacheTest02() throws Exception {
+
+ InMemoryBufferCacheFactory InMemBufferCacheFactory = new InMemoryBufferCacheFactory(PAGE_SIZE, NUM_PAGES);
+ IBufferCache memBufferCache = InMemBufferCacheFactory.createInMemoryBufferCache();
+
+ try {
+ ICachedPage memCachedPage = memBufferCache.pin(500, true);
+ if(memCachedPage != null) {
+ return;
+ }
+ else {
+ fail("fail to pin");
+ }
+ }
+ catch (ArrayIndexOutOfBoundsException e){
+ System.out.println("Catch exception: " + e);
+ return;
+ }
+ catch (Exception e) {
+ fail("Unexpected exception!");
+ }
+ }
+
+ //TEST InMemoryBufferCache.getPage()
+ @Test
+ public void InMemoryBufferCacheTest03() throws Exception {
+
+ InMemoryBufferCacheFactory InMemBufferCacheFactory = new InMemoryBufferCacheFactory(PAGE_SIZE, NUM_PAGES);
+ IBufferCache memBufferCache = InMemBufferCacheFactory.createInMemoryBufferCache();
+
+ try {
+ ICachedPage memCachedPage = ((IBufferCacheInternal) memBufferCache).getPage(20);
+ if(memCachedPage != null) {
+ return;
+ }
+ else {
+ fail("fail to pin");
+ }
+ }
+ catch (ArrayIndexOutOfBoundsException e){
+ System.out.println("Catch exception: " + e);
+ return;
+ }
+ catch (Exception e) {
+ fail("Unexpected exception!");
+ }
+ }
+
+ //TEST InMemoryBufferCache.getPage()
+ //expect ArrayIndexOutOfBoundsException
+ @Test
+ public void InMemoryBufferCacheTest04() throws Exception {
+
+ InMemoryBufferCacheFactory InMemBufferCacheFactory = new InMemoryBufferCacheFactory(PAGE_SIZE, NUM_PAGES);
+ IBufferCache memBufferCache = InMemBufferCacheFactory.createInMemoryBufferCache();
+
+ try {
+ ICachedPage memCachedPage = ((IBufferCacheInternal) memBufferCache).getPage(1000);
+ if(memCachedPage != null) {
+ return;
+ }
+ else {
+ fail("fail to pin");
+ }
+ }
+ catch (ArrayIndexOutOfBoundsException e){
+ System.out.println("Catch exception: " + e);
+ return;
+ }
+ catch (Exception e) {
+ fail("Unexpected exception!");
+ }
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/InMemoryFreePageManager.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/InMemoryFreePageManager.java
new file mode 100644
index 0000000..9331c04
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/InMemoryFreePageManager.java
@@ -0,0 +1,87 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.freepage;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
+
+public class InMemoryFreePageManager implements IFreePageManager{
+ private final int maxCapacity;
+ private int currentCapacity;
+ private final ITreeIndexMetaDataFrameFactory metaDataFrameFactory;
+
+ public InMemoryFreePageManager(int maxCapacity, ITreeIndexMetaDataFrameFactory metaDataFrameFactory) {
+ this.maxCapacity = maxCapacity-1; // Since the range of CacheArray in InMemoryBufferCache is 0 ~ maxCapacity-1
+ currentCapacity = 1;
+ this.metaDataFrameFactory = metaDataFrameFactory;
+ }
+
+ public int getCurrentCapacity() {
+ return currentCapacity;
+ }
+
+ @Override
+ public synchronized int getFreePage(ITreeIndexMetaDataFrame metaFrame)
+ throws HyracksDataException, PageAllocationException {
+
+ if(currentCapacity == maxCapacity) {
+ throw new PageAllocationException("In-mem tree capacity reaches max capacity");
+ }
+ currentCapacity++;
+ return currentCapacity;
+ }
+
+
+ @Override
+ public void addFreePage(ITreeIndexMetaDataFrame metaFrame, int freePage)
+ throws HyracksDataException {
+ System.out.println("InMemoryFreePageManager.addFreePage()");
+ }
+
+ @Override
+ public int getMaxPage(ITreeIndexMetaDataFrame metaFrame)
+ throws HyracksDataException {
+ System.out.println("InMemoryFreePageManager.getMaxPage()");
+ return 0;
+ }
+
+ @Override
+ public void init(ITreeIndexMetaDataFrame metaFrame, int currentMaxPage)
+ throws HyracksDataException {
+ currentCapacity = 1;
+ }
+
+ @Override
+ public ITreeIndexMetaDataFrameFactory getMetaDataFrameFactory() {
+ return metaDataFrameFactory;
+ }
+
+ @Override
+ public byte getMetaPageLevelIndicator() {
+ System.out.println("InMemoryFreePageManager.getMetaPageLevelIndicator()");
+ return 0;
+ }
+
+ @Override
+ public byte getFreePageLevelIndicator() {
+ System.out.println("InMemoryFreePageManager.getFreePageLevelIndicator()");
+ return 0;
+ }
+
+ @Override
+ public boolean isMetaPage(ITreeIndexMetaDataFrame metaFrame) {
+ System.out.println("InMemoryFreePageManager.isMetaPage()");
+ return false;
+ }
+
+ @Override
+ public boolean isFreePage(ITreeIndexMetaDataFrame metaFrame) {
+ System.out.println("InMemoryFreePageManager.isFreePage()");
+ return false;
+ }
+
+ public void reset(){
+ currentCapacity = 1;
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/InMemoryFreePageManagerTest.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/InMemoryFreePageManagerTest.java
new file mode 100644
index 0000000..71a781c
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/freepage/InMemoryFreePageManagerTest.java
@@ -0,0 +1,56 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.freepage;
+
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+
+public class InMemoryFreePageManagerTest{
+
+ //Get the free pages
+ @Test
+ public void InMemoryFreePageManagerTest01() throws Exception {
+
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+ IFreePageManager memFreePageManager = new InMemoryFreePageManager(10, metaFrameFactory);
+
+ try{
+ for(int i = 0; i < 5; i++) {
+ memFreePageManager.getFreePage(null);
+ }
+ }
+ catch (PageAllocationException e){
+ System.out.println("Catch exception: " + e);
+ return;
+ }
+ catch (Exception e) {
+ fail("Unexpected exception!");
+ }
+ }
+
+ //Get free pages more than the max capacity
+ //expect PageAllocationException
+ @Test
+ public void InMemoryFreePageManagerTest02() throws Exception {
+
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+ IFreePageManager memFreePageManager = new InMemoryFreePageManager(10, metaFrameFactory);
+
+ try{
+ for(int i = 0; i < 20; i++) {
+ memFreePageManager.getFreePage(null);
+ }
+ }
+ catch (PageAllocationException e){
+ System.out.println("Catch exception: " + e);
+ return;
+ }
+ catch (Exception e) {
+ fail("Unexpected exception!");
+ }
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/BTreeFactory.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/BTreeFactory.java
new file mode 100644
index 0000000..f5bfaec
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/BTreeFactory.java
@@ -0,0 +1,35 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.impls;
+
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsmtree.freepage.FreePageManagerFactory;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+public class BTreeFactory {
+
+ private IBufferCache bufferCache;
+ private int fieldCount;
+ private MultiComparator cmp;
+ private ITreeIndexFrameFactory interiorFrameFactory;
+ private ITreeIndexFrameFactory leafFrameFactory;
+ private FreePageManagerFactory freePageManagerFactory;
+
+
+ public BTreeFactory(IBufferCache bufferCache, FreePageManagerFactory freePageManagerFactory, MultiComparator cmp,
+ int fieldCount, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
+ this.bufferCache = bufferCache;
+ this.fieldCount = fieldCount;
+ this.cmp = cmp;
+ this.interiorFrameFactory = interiorFrameFactory;
+ this.leafFrameFactory = leafFrameFactory;
+ this.freePageManagerFactory = freePageManagerFactory;
+ }
+
+ public BTree createBTreeInstance(int fileId) {
+ return new BTree(bufferCache, fieldCount, cmp, freePageManagerFactory.createFreePageManager(fileId),
+ interiorFrameFactory, leafFrameFactory);
+ }
+
+
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/InDiskTreeInfo.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/InDiskTreeInfo.java
new file mode 100644
index 0000000..e8d1137
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/InDiskTreeInfo.java
@@ -0,0 +1,16 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.impls;
+
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+
+public class InDiskTreeInfo {
+
+ private BTree bTree;
+
+ public InDiskTreeInfo(BTree bTree) {
+ this.bTree = bTree;
+ }
+
+ public BTree getBTree() {
+ return bTree;
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMPriorityQueueComparator.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMPriorityQueueComparator.java
new file mode 100644
index 0000000..1bb5abb
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMPriorityQueueComparator.java
@@ -0,0 +1,36 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.impls;
+
+import java.util.Comparator;
+
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+
+public class LSMPriorityQueueComparator implements Comparator<LSMPriorityQueueElement> {
+
+ private MultiComparator cmp;
+
+ public LSMPriorityQueueComparator(MultiComparator cmp) {
+ this.cmp = cmp;
+ }
+
+ @Override
+ public int compare(LSMPriorityQueueElement elementA, LSMPriorityQueueElement elementB) {
+
+ int result = cmp.compare(elementA.getTuple(), elementB.getTuple());
+
+ if(result == 1) {
+ return 1;
+ }
+ else if(result == -1) {
+ return -1;
+ }
+ else {
+ if(elementA.getCursorIndex() > elementB.getCursorIndex()) {
+ return 1;
+ }
+ else {
+ return -1;
+ }
+ }
+ }
+
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMPriorityQueueElement.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMPriorityQueueElement.java
new file mode 100644
index 0000000..c8e3241
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMPriorityQueueElement.java
@@ -0,0 +1,25 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.impls;
+
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+public class LSMPriorityQueueElement {
+ private ITupleReference tuple;
+ private int cursorIndex;
+
+ public LSMPriorityQueueElement(ITupleReference tuple, int cursorIndex) {
+ reset(tuple, cursorIndex);
+ }
+
+ public ITupleReference getTuple() {
+ return tuple;
+ }
+
+ public int getCursorIndex() {
+ return cursorIndex;
+ }
+
+ public void reset(ITupleReference tuple, int cursorIndex) {
+ this.tuple = tuple;
+ this.cursorIndex = cursorIndex;
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTree.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTree.java
new file mode 100644
index 0000000..de6b524
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTree.java
@@ -0,0 +1,585 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.impls;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.File;
+import java.util.LinkedList;
+import java.util.ListIterator;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeDuplicateKeyException;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
+import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsmtree.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsmtree.tuples.LSMTypeAwareTupleReference;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapManager;
+
+public class LSMTree implements ITreeIndex {
+
+ private final IBufferCache bufferCache;
+ private BTree memBTree;
+ private String fileName;
+ private int fileId;
+ private boolean created;
+
+ private final IFreePageManager memFreePageManager;
+ private final ITreeIndexFrameFactory interiorFrameFactory;
+ private final ITreeIndexFrameFactory insertLeafFrameFactory;
+ private final ITreeIndexFrameFactory deleteLeafFrameFactory;
+ private final MultiComparator cmp;
+
+ // TODO: change to private, it's public only for LSMTreeSearchTest
+ public LinkedList<InDiskTreeInfo> inDiskTreeInfoList;
+ private LinkedList<InDiskTreeInfo> mergedInDiskTreeInfoList;
+ private int inDiskTreeCounter;
+ private final BTreeFactory bTreeFactory;
+ private final IFileMapManager fileMapManager;
+ private int threadReferenceCounter;
+ private boolean flushFlag;
+
+ public LSMTree(IBufferCache memCache, IBufferCache bufferCache, int fieldCount, MultiComparator cmp,
+ IFreePageManager memFreePageManager, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory,
+ BTreeFactory bTreeFactory, IFileMapManager fileMapManager) {
+ this.bufferCache = bufferCache;
+ this.cmp = cmp;
+ this.interiorFrameFactory = interiorFrameFactory;
+ this.insertLeafFrameFactory = insertLeafFrameFactory;
+ this.deleteLeafFrameFactory = deleteLeafFrameFactory;
+ this.memFreePageManager = memFreePageManager;
+ this.bTreeFactory = bTreeFactory;
+ this.inDiskTreeInfoList = new LinkedList<InDiskTreeInfo>();
+ this.inDiskTreeCounter = 0;
+ this.fileMapManager = fileMapManager;
+ this.threadReferenceCounter = 0;
+ this.created = false;
+ this.flushFlag = false;
+
+ try {
+ this.fileName = this.fileMapManager.lookupFileName(this.fileId).toString();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ memBTree = new BTree(memCache, fieldCount, cmp, memFreePageManager, interiorFrameFactory,
+ insertLeafFrameFactory);
+ }
+
+ @Override
+ public void create(int indexFileId) throws HyracksDataException {
+ if (created) {
+ return;
+ } else {
+ memBTree.create(indexFileId);
+ this.fileId = indexFileId;
+ created = true;
+ }
+ }
+
+ @Override
+ public void open(int indexFileId) {
+ memBTree.open(indexFileId);
+ this.fileId = indexFileId;
+ }
+
+ @Override
+ public void close() {
+ memBTree.close();
+ this.fileId = -1;
+ }
+
+ private void lsmPerformOp(ITupleReference tuple, LSMTreeOpContext ctx) throws Exception {
+ boolean continuePerformOp = false;
+ try {
+ while (continuePerformOp == false) {
+ synchronized (this) {
+ if (!flushFlag) {
+ threadReferenceCounter++;
+ continuePerformOp = true;
+ }
+ }
+ }
+ ctx.memBtreeAccessor.insert(tuple);
+ decreaseThreadReferenceCounter();
+ } catch (BTreeDuplicateKeyException e) {
+ ctx.reset(IndexOp.UPDATE);
+ // We don't need to deal with a nonexistent key here, because a
+ // deleter will actually update the key and it's value, and not
+ // delete it from the BTree.
+ ctx.memBtreeAccessor.update(tuple);
+ decreaseThreadReferenceCounter();
+ } catch (PageAllocationException e) {
+ synchronized (this) {
+ // If flushFlag is false it means we are the first inserter to
+ // trigger the flush. If flushFlag is already set to true,
+ // there's no harm in setting it to true again.
+ flushFlag = true;
+ threadReferenceCounter--;
+ if (threadReferenceCounter == 0) {
+ flushInMemoryBtree();
+ ctx.reset();
+ ctx.memBtreeAccessor.insert(tuple);
+ flushFlag = false;
+ return;
+ } else if (threadReferenceCounter < 0) {
+ throw new Error("Thread reference counter is below zero. This indicates a programming error!");
+ }
+ }
+ lsmPerformOp(tuple, ctx);
+ return;
+ }
+ }
+
+ public void decreaseThreadReferenceCounter() throws Exception {
+ synchronized (this) {
+ threadReferenceCounter--;
+ if (flushFlag == true) {
+ if (threadReferenceCounter == 0) {
+ flushInMemoryBtree();
+ flushFlag = false;
+ return;
+ } else if (threadReferenceCounter < 0) {
+ throw new Error("Thread reference counter is below zero. This indicates a programming error!");
+ }
+ }
+ }
+ }
+
+ private String getNextFileName() {
+ int newDiskBTreeId = inDiskTreeCounter++;
+ String LSMFileName = new String(this.fileName);
+ return LSMFileName.concat("-" + Integer.toString(newDiskBTreeId));
+ }
+
+ public void flushInMemoryBtree() throws Exception {
+ // read the tuple from InMemoryBtree, and bulkload into the disk
+
+ // scan
+ ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(
+ (IBTreeLeafFrame) insertLeafFrameFactory.createFrame(), false);
+ RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
+ ITreeIndexAccessor memBtreeAccessor = memBTree.createAccessor();
+ memBtreeAccessor.search(scanCursor, nullPred);
+
+ // Create a new in-Disk BTree, which have full fillfactor.
+
+ // Register the BTree information into system.
+ FileReference file = new FileReference(new File(getNextFileName()));
+ // TODO: Delete the file during cleanup.
+ bufferCache.createFile(file);
+ int newDiskBTreeId = fileMapManager.lookupFileId(file);
+ // TODO: Close the file during cleanup.
+ bufferCache.openFile(newDiskBTreeId);
+
+ // Create new in-Disk Btree.
+ BTree inDiskBTree = this.bTreeFactory.createBTreeInstance(newDiskBTreeId);
+ inDiskBTree.create(newDiskBTreeId);
+ // TODO: Close the BTree during cleanup.
+ inDiskBTree.open(newDiskBTreeId);
+
+ // BulkLoad the tuples from the in-memory tree into the new disk BTree.
+ IIndexBulkLoadContext bulkLoadCtx = inDiskBTree.beginBulkLoad(1.0f);
+ try {
+ while (scanCursor.hasNext()) {
+ scanCursor.next();
+ ITupleReference frameTuple = scanCursor.getTuple();
+ inDiskBTree.bulkLoadAddTuple(frameTuple, bulkLoadCtx);
+ }
+ } finally {
+ scanCursor.close();
+ }
+ inDiskBTree.endBulkLoad(bulkLoadCtx);
+
+ // After BulkLoading, Clear the in-memTree
+ resetInMemoryTree();
+
+ InDiskTreeInfo newLinkedListNode = new InDiskTreeInfo(inDiskBTree);
+ synchronized (inDiskTreeInfoList) {
+ inDiskTreeInfoList.addFirst(newLinkedListNode);
+ }
+ }
+
+ private void insert(ITupleReference tuple, LSMTreeOpContext ctx) throws HyracksDataException, TreeIndexException,
+ PageAllocationException {
+ try {
+ lsmPerformOp(tuple, ctx);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ private void delete(ITupleReference tuple, LSMTreeOpContext ctx) throws HyracksDataException, TreeIndexException,
+ PageAllocationException {
+ try {
+ lsmPerformOp(tuple, ctx);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public IIndexBulkLoadContext beginBulkLoad(float fillFactor) throws TreeIndexException, HyracksDataException,
+ PageAllocationException {
+ return null;
+ }
+
+ @Override
+ public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException,
+ PageAllocationException {
+ }
+
+ @Override
+ public void endBulkLoad(IIndexBulkLoadContext ictx) throws HyracksDataException {
+ }
+
+ public void search(ITreeIndexCursor cursor, RangePredicate pred, LSMTreeOpContext ctx) throws Exception {
+ int numberOfInDiskTrees;
+ ListIterator<InDiskTreeInfo> inDiskTreeInfoListIterator;
+ boolean continuePerformOp = false;
+
+ ctx.reset(IndexOp.SEARCH);
+
+ while (continuePerformOp == false) {
+ synchronized (this) {
+ if (!flushFlag) {
+ threadReferenceCounter++;
+ continuePerformOp = true;
+ }
+ }
+ }
+
+ // in-disk
+ synchronized (inDiskTreeInfoList) {
+ numberOfInDiskTrees = inDiskTreeInfoList.size();
+ inDiskTreeInfoListIterator = inDiskTreeInfoList.listIterator();
+ }
+
+ LSMTreeCursorInitialState initialState = new LSMTreeCursorInitialState(numberOfInDiskTrees + 1,
+ insertLeafFrameFactory, cmp, this);
+ cursor.open(initialState, pred);
+
+ BTree[] onDiskBtrees = new BTree[numberOfInDiskTrees];
+ ITreeIndexAccessor[] onDiskBtreeAccessors = new ITreeIndexAccessor[numberOfInDiskTrees];
+
+ for (int i = 0; i < numberOfInDiskTrees; i++) {
+ // get btree instances for in-disk trees
+ if (inDiskTreeInfoListIterator.hasNext()) {
+ onDiskBtrees[i] = ((InDiskTreeInfo) inDiskTreeInfoListIterator.next()).getBTree();
+ } else {
+ throw new HyracksDataException("Cannot find in-disk tree instance");
+ }
+ onDiskBtreeAccessors[i] = onDiskBtrees[i].createAccessor();
+ onDiskBtreeAccessors[i].search(((LSMTreeRangeSearchCursor) cursor).getCursor(i + 1), pred);
+ }
+
+ // in-memory
+ ctx.memBtreeAccessor.search(((LSMTreeRangeSearchCursor) cursor).getCursor(0), pred);
+
+ LSMPriorityQueueComparator LSMPriorityQueueCmp = new LSMPriorityQueueComparator(cmp);
+ ((LSMTreeRangeSearchCursor) cursor).initPriorityQueue(numberOfInDiskTrees + 1, LSMPriorityQueueCmp);
+ }
+
+ public void merge() throws Exception {
+ ITreeIndexCursor rangeCursor = new LSMTreeRangeSearchCursor();
+ RangePredicate rangePred = new RangePredicate(true, null, null, true, true, null, null);
+
+ // Cursor setting -- almost the same as search, only difference is
+ // "no cursor for in-memory tree"
+ int numberOfInDiskTrees;
+ ListIterator<InDiskTreeInfo> inDiskTreeInfoListIterator;
+ boolean continuePerformOp = false;
+ while (continuePerformOp == false) {
+ synchronized (this) {
+ if (!flushFlag) {
+ threadReferenceCounter++;
+ continuePerformOp = true;
+ }
+ }
+ }
+
+ synchronized (inDiskTreeInfoList) {
+ numberOfInDiskTrees = inDiskTreeInfoList.size();
+ inDiskTreeInfoListIterator = inDiskTreeInfoList.listIterator();
+ }
+
+ LSMTreeCursorInitialState initialState = new LSMTreeCursorInitialState(numberOfInDiskTrees,
+ insertLeafFrameFactory, cmp, this);
+ rangeCursor.open(initialState, rangePred);
+
+ BTree[] onDiskBtrees = new BTree[numberOfInDiskTrees];
+ ITreeIndexAccessor[] onDiskBtreeAccessors = new ITreeIndexAccessor[numberOfInDiskTrees];
+
+ for (int i = 0; i < numberOfInDiskTrees; i++) {
+ // get btree instances for in-disk trees
+ if (inDiskTreeInfoListIterator.hasNext()) {
+ onDiskBtrees[i] = ((InDiskTreeInfo) inDiskTreeInfoListIterator.next()).getBTree();
+ } else {
+ throw new Exception("Cannot find in-disk tree instance");
+ }
+ onDiskBtreeAccessors[i] = onDiskBtrees[i].createAccessor();
+ onDiskBtreeAccessors[i].search(((LSMTreeRangeSearchCursor) rangeCursor).getCursor(i), rangePred);
+ }
+
+ LSMPriorityQueueComparator LSMPriorityQueueCmp = new LSMPriorityQueueComparator(cmp);
+ ((LSMTreeRangeSearchCursor) rangeCursor).initPriorityQueue(numberOfInDiskTrees, LSMPriorityQueueCmp);
+ // End of Cursor setting
+
+ // Create a new Merged BTree, which have full fillfactor.
+ // Register the BTree information into system.
+ // TODO: change the naming schema for the new tree
+ FileReference file = new FileReference(new File(getNextFileName()));
+ // TODO: Delete the file during cleanup.
+ bufferCache.createFile(file);
+ int mergedBTreeId = fileMapManager.lookupFileId(file);
+ // TODO: Close the file during cleanup.
+ bufferCache.openFile(mergedBTreeId);
+
+ // Create new in-Disk BTree.
+ BTree mergedBTree = this.bTreeFactory.createBTreeInstance(mergedBTreeId);
+ mergedBTree.create(mergedBTreeId);
+ // TODO: Close the BTree during cleanup.
+ mergedBTree.open(mergedBTreeId);
+
+ // BulkLoad the tuples from the in-memory tree into the new disk BTree.
+ IIndexBulkLoadContext bulkLoadCtx = mergedBTree.beginBulkLoad(1.0f);
+
+ try {
+ while (rangeCursor.hasNext()) {
+ rangeCursor.next();
+ ITupleReference frameTuple = rangeCursor.getTuple();
+ mergedBTree.bulkLoadAddTuple(frameTuple, bulkLoadCtx);
+ }
+ } finally {
+ rangeCursor.close();
+ }
+
+ mergedBTree.endBulkLoad(bulkLoadCtx);
+
+ InDiskTreeInfo newLinkedListNode = new InDiskTreeInfo(mergedBTree);
+ LinkedList<InDiskTreeInfo> tempInDiskTreeInfo;
+ synchronized (inDiskTreeInfoList) {
+ mergedInDiskTreeInfoList = (LinkedList<InDiskTreeInfo>) inDiskTreeInfoList.clone();
+ // Remove the redundant trees, and add the new merged tree in the
+ // last off the list
+ for (int i = 0; i < numberOfInDiskTrees; i++) {
+ mergedInDiskTreeInfoList.removeLast();
+ }
+ mergedInDiskTreeInfoList.addLast(newLinkedListNode);
+
+ // TODO: to swap the linkedlists
+ /*
+ * tempInDiskTreeInfo = inDiskTreeInfoList; inDiskTreeInfoList =
+ * mergedInDiskTreeInfoList; mergedInDiskTreeInfoList =
+ * tempInDiskTreeInfo;
+ */
+ // TODO: to swap the searchThreadCounters
+
+ // 1. should add the searcherReferenceCounter
+ // 2. Wrap the searcherReferenceCounter as Integer object,
+ // otherwise, the reference cannot be swapped
+ // 3. modify the "decrease counter part" in search(), and let the
+ // searcher remember the localReferences
+
+ }
+ // TODO: to wake up the cleaning thread
+ }
+
+ @Override
+ public ITreeIndexFrameFactory getLeafFrameFactory() {
+ return memBTree.getLeafFrameFactory();
+ }
+
+ @Override
+ public ITreeIndexFrameFactory getInteriorFrameFactory() {
+ return memBTree.getInteriorFrameFactory();
+ }
+
+ @Override
+ public IFreePageManager getFreePageManager() {
+ return memBTree.getFreePageManager();
+ }
+
+ @Override
+ public int getFieldCount() {
+ return memBTree.getFieldCount();
+ }
+
+ @Override
+ public int getRootPageId() {
+ return memBTree.getRootPageId();
+ }
+
+ @Override
+ public IndexType getIndexType() {
+ return memBTree.getIndexType();
+ }
+
+ public void resetInMemoryTree() throws HyracksDataException {
+ ((InMemoryFreePageManager) memFreePageManager).reset();
+ memBTree.create(fileId);
+ }
+
+ // This function is just for testing flushInMemoryBtree()
+ public void scanDiskTree(int treeIndex) throws Exception {
+ BTree onDiskBtree = inDiskTreeInfoList.get(treeIndex).getBTree();
+ ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE };
+ RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
+ ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(
+ (IBTreeLeafFrame) this.insertLeafFrameFactory.createFrame(), false);
+ RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
+ ITreeIndexAccessor onDiskBtreeAccessor = onDiskBtree.createAccessor();
+ onDiskBtreeAccessor.search(scanCursor, nullPred);
+ try {
+ int scanTupleIndex = 0;
+ while (scanCursor.hasNext()) {
+ scanCursor.hasNext();
+ scanCursor.next();
+ ITupleReference frameTuple = scanCursor.getTuple();
+ int numPrintFields = Math.min(frameTuple.getFieldCount(), recDescSers.length);
+
+ for (int i = 0; i < numPrintFields; i++) {
+ ByteArrayInputStream inStream = new ByteArrayInputStream(frameTuple.getFieldData(i),
+ frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
+ DataInput dataIn = new DataInputStream(inStream);
+ Object o = recDescSers[i].deserialize(dataIn);
+
+ if (i == 1)
+ System.out.printf("LSMTree.scanDiskTree(): scanTupleIndex[%d]; Value is [%d]; ",
+ scanTupleIndex, Integer.parseInt(o.toString()));
+
+ }
+
+ if (((LSMTypeAwareTupleReference) frameTuple).isDelete()) {
+ System.out.printf(" DELETE\n");
+ } else {
+ System.out.printf(" INSERT\n");
+ }
+ scanTupleIndex++;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ scanCursor.close();
+ }
+ }
+
+ // This function is just for testing flushInMemoryBtree()
+ public void scanInMemoryTree() throws Exception {
+ ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE };
+ RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
+ ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(
+ (IBTreeLeafFrame) this.insertLeafFrameFactory.createFrame(), false);
+ RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
+ ITreeIndexAccessor inMemBtreeAccessor = memBTree.createAccessor();
+ inMemBtreeAccessor.search(scanCursor, nullPred);
+ try {
+ int scanTupleIndex = 0;
+ while (scanCursor.hasNext()) {
+ scanCursor.hasNext();
+ scanCursor.next();
+ ITupleReference frameTuple = scanCursor.getTuple();
+ int numPrintFields = Math.min(frameTuple.getFieldCount(), recDescSers.length);
+ for (int i = 0; i < numPrintFields; i++) {
+ ByteArrayInputStream inStream = new ByteArrayInputStream(frameTuple.getFieldData(i),
+ frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
+ DataInput dataIn = new DataInputStream(inStream);
+ Object o = recDescSers[i].deserialize(dataIn);
+ if (i == 1)
+ System.out.printf("LSMTree.scanMemoryTree(): scanTupleIndex[%d]; Value is [%d]; ",
+ scanTupleIndex, Integer.parseInt(o.toString()));
+ }
+ if (((LSMTypeAwareTupleReference) frameTuple).isDelete()) {
+ System.out.printf(" DELETE\n");
+ } else {
+ System.out.printf(" INSERT\n");
+ }
+ scanTupleIndex++;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ scanCursor.close();
+ }
+ }
+
+ public LSMTreeOpContext createOpContext() {
+ return new LSMTreeOpContext((BTree.BTreeAccessor) memBTree.createAccessor(), insertLeafFrameFactory,
+ deleteLeafFrameFactory, interiorFrameFactory, memFreePageManager.getMetaDataFrameFactory()
+ .createFrame(), cmp);
+ }
+
+ @Override
+ public ITreeIndexAccessor createAccessor() {
+ return new LSMTreeIndexAccessor(this);
+ }
+
+ private class LSMTreeIndexAccessor implements ITreeIndexAccessor {
+ private LSMTree lsmTree;
+ private LSMTreeOpContext ctx;
+
+ public LSMTreeIndexAccessor(LSMTree lsmTree) {
+ this.lsmTree = lsmTree;
+ this.ctx = lsmTree.createOpContext();
+ }
+
+ @Override
+ public void insert(ITupleReference tuple) throws HyracksDataException, TreeIndexException,
+ PageAllocationException {
+ ctx.reset(IndexOp.INSERT);
+ lsmTree.insert(tuple, ctx);
+ }
+
+ @Override
+ public void update(ITupleReference tuple) throws HyracksDataException, TreeIndexException,
+ PageAllocationException {
+ throw new UnsupportedOperationException("Update not supported by LSMTree");
+ }
+
+ @Override
+ public void delete(ITupleReference tuple) throws HyracksDataException, TreeIndexException,
+ PageAllocationException {
+ ctx.reset(IndexOp.DELETE);
+ lsmTree.delete(tuple, ctx);
+ }
+
+ @Override
+ public void search(ITreeIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException,
+ TreeIndexException, PageAllocationException {
+ ctx.reset(IndexOp.SEARCH);
+ // TODO: fix exception handling throughout LSM tree.
+ try {
+ lsmTree.search(cursor, (RangePredicate) searchPred, ctx);
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void diskOrderScan(ITreeIndexCursor cursor) throws HyracksDataException {
+ throw new UnsupportedOperationException("DiskOrderScan not supported by LSMTree");
+ }
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTreeCursorInitialState.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTreeCursorInitialState.java
new file mode 100644
index 0000000..5391d9c
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTreeCursorInitialState.java
@@ -0,0 +1,47 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.impls;
+
+import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+
+public class LSMTreeCursorInitialState implements ICursorInitialState {
+
+ private int numberOfTrees;
+ private ITreeIndexFrameFactory leafFrameFactory;
+ private MultiComparator cmp;
+ private LSMTree lsm;
+
+ public LSMTreeCursorInitialState(int numberOfTrees, ITreeIndexFrameFactory leafFrameFactory, MultiComparator cmp, LSMTree lsm) {
+ this.numberOfTrees = numberOfTrees;
+ this.leafFrameFactory = leafFrameFactory;
+ this.cmp = cmp;
+ this.lsm = lsm;
+ }
+
+ public int getNumberOfTrees() {
+ return numberOfTrees;
+ }
+
+ public ITreeIndexFrameFactory getLeafFrameFactory() {
+ return leafFrameFactory;
+ }
+
+ public MultiComparator getCmp() {
+ return cmp;
+ }
+
+ @Override
+ public ICachedPage getPage() {
+ return null;
+ }
+
+ @Override
+ public void setPage(ICachedPage page) {
+ }
+
+ public LSMTree getLsm() {
+ return lsm;
+ }
+
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTreeOpContext.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTreeOpContext.java
new file mode 100644
index 0000000..d038ada
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTreeOpContext.java
@@ -0,0 +1,66 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.impls;
+
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+
+public final class LSMTreeOpContext extends BTreeOpContext {
+
+ public ITreeIndexFrameFactory insertLeafFrameFactory;
+ public ITreeIndexFrameFactory deleteLeafFrameFactory;
+ public IBTreeLeafFrame insertLeafFrame;
+ public IBTreeLeafFrame deleteLeafFrame;
+ public final BTree.BTreeAccessor memBtreeAccessor;
+
+ public LSMTreeOpContext(BTree.BTreeAccessor memBtreeAccessor, ITreeIndexFrameFactory insertLeafFrameFactory,
+ ITreeIndexFrameFactory deleteLeafFrameFactory, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexMetaDataFrame metaFrame, MultiComparator cmp) {
+ super(insertLeafFrameFactory, interiorFrameFactory, metaFrame, cmp);
+
+ this.memBtreeAccessor = memBtreeAccessor;
+ // Overwrite the BTree accessor's op context with our LSMTreeOpContext.
+ this.memBtreeAccessor.setOpContext(this);
+
+ this.insertLeafFrameFactory = insertLeafFrameFactory;
+ this.deleteLeafFrameFactory = deleteLeafFrameFactory;
+ this.insertLeafFrame = (IBTreeLeafFrame) insertLeafFrameFactory.createFrame();
+ this.deleteLeafFrame = (IBTreeLeafFrame) deleteLeafFrameFactory.createFrame();
+
+ if (insertLeafFrame != null) {
+ insertLeafFrame.setMultiComparator(cmp);
+ }
+
+ if (deleteLeafFrame != null) {
+ deleteLeafFrame.setMultiComparator(cmp);
+ }
+
+ reset(op);
+ }
+
+ @Override
+ public void reset(IndexOp newOp) {
+ super.reset(newOp);
+ if(newOp == IndexOp.INSERT) {
+ setInsertMode();
+ }
+ if(newOp == IndexOp.DELETE) {
+ super.reset(IndexOp.INSERT);
+ setDeleteMode();
+ }
+ }
+
+ public void setInsertMode() {
+ this.leafFrame = insertLeafFrame;
+ leafFrameFactory = insertLeafFrameFactory;
+ }
+
+ public void setDeleteMode() {
+ this.leafFrame = deleteLeafFrame;
+ leafFrameFactory = deleteLeafFrameFactory;
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTreeRangeSearchCursor.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTreeRangeSearchCursor.java
new file mode 100644
index 0000000..079dada
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/impls/LSMTreeRangeSearchCursor.java
@@ -0,0 +1,183 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.impls;
+
+import java.util.PriorityQueue;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsmtree.tuples.LSMTypeAwareTupleReference;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+
+public class LSMTreeRangeSearchCursor implements ITreeIndexCursor {
+
+ private BTreeRangeSearchCursor[] rangeCursors;
+ private PriorityQueue<LSMPriorityQueueElement> outputPriorityQueue;
+ private MultiComparator cmp;
+ private LSMPriorityQueueElement outputElement;
+ private LSMPriorityQueueElement reusedElement;
+ private int numberOfTrees;
+ private boolean needPush;
+ private LSMTree lsm;
+
+ public LSMTreeRangeSearchCursor() {
+ outputElement = null;
+ needPush = false;
+ }
+
+ public void initPriorityQueue(int numberOfTrees, LSMPriorityQueueComparator LSMPriorityQueueCmp) throws Exception {
+ this.numberOfTrees = numberOfTrees;
+ outputPriorityQueue = new PriorityQueue<LSMPriorityQueueElement>(numberOfTrees, LSMPriorityQueueCmp);
+ for (int i = 0; i < numberOfTrees; i++) {
+ LSMPriorityQueueElement element;
+ if (rangeCursors[i].hasNext()) {
+ rangeCursors[i].next();
+ element = new LSMPriorityQueueElement(rangeCursors[i].getTuple(), i);
+ outputPriorityQueue.offer(element);
+ }
+ }
+ checkPriorityQueue();
+ }
+
+ public BTreeRangeSearchCursor getCursor(int cursorIndex) {
+ return rangeCursors[cursorIndex];
+ }
+
+ @Override
+ public void reset() {
+ // do nothing
+ }
+
+ @Override
+ public boolean hasNext() throws Exception {
+ checkPriorityQueue();
+ return !outputPriorityQueue.isEmpty();
+ }
+
+ @Override
+ public void next() throws Exception {
+
+ outputElement = outputPriorityQueue.poll();
+ needPush = true;
+ if (outputElement == null) {
+ throw new Exception("The outputPriorityQueue is empty");
+ }
+ }
+
+ @Override
+ public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+
+ lsm = ((LSMTreeCursorInitialState) initialState).getLsm();
+ cmp = ((LSMTreeCursorInitialState) initialState).getCmp();
+
+ rangeCursors = new BTreeRangeSearchCursor[((LSMTreeCursorInitialState) initialState).getNumberOfTrees()];
+
+ for (int i = 0; i < ((LSMTreeCursorInitialState) initialState).getNumberOfTrees(); i++) {
+ rangeCursors[i] = new BTreeRangeSearchCursor((IBTreeLeafFrame) ((LSMTreeCursorInitialState) initialState)
+ .getLeafFrameFactory().createFrame(), false);
+ }
+ }
+
+ @Override
+ public ICachedPage getPage() {
+ // do nothing
+ return null;
+ }
+
+ @Override
+ public void close() throws Exception {
+ lsm.decreaseThreadReferenceCounter();
+ outputPriorityQueue.clear();
+ for (int i = 0; i < numberOfTrees; i++) {
+ rangeCursors[i].close();
+ }
+ rangeCursors = null;
+ }
+
+ @Override
+ public void setBufferCache(IBufferCache bufferCache) {
+ // do nothing
+ }
+
+ @Override
+ public void setFileId(int fileId) {
+ // do nothing
+ }
+
+ @Override
+ public ITupleReference getTuple() {
+ return (ITupleReference) outputElement.getTuple();
+ }
+
+ private void pushIntoPriorityQueue(int cursorIndex) throws Exception {
+
+ if (rangeCursors[cursorIndex].hasNext()) {
+ rangeCursors[cursorIndex].next();
+ reusedElement.reset(rangeCursors[cursorIndex].getTuple(), cursorIndex);
+ outputPriorityQueue.offer(reusedElement);
+ }
+ }
+
+ private void checkPriorityQueue() throws Exception {
+
+ while (!outputPriorityQueue.isEmpty() || needPush == true) {
+ if (!outputPriorityQueue.isEmpty()) {
+ LSMPriorityQueueElement checkElement = outputPriorityQueue.peek();
+ // If there is no previous tuple or the previous tuple can be
+ // ignored
+ if (outputElement == null) {
+ // Test the tuple is a delete tuple or not
+ if (((LSMTypeAwareTupleReference) checkElement.getTuple()).isDelete() == true) {
+ // If the tuple is a delete tuple then pop it and mark
+ // it "needPush"
+ // Cannot push at this time because the tuple may be
+ // modified if "hasNext" is called
+ outputElement = outputPriorityQueue.poll();
+ needPush = true;
+ } else {
+ break;
+ }
+ } else {
+ // Compare the previous tuple and the head tuple in the PQ
+ if (cmp.compare(outputElement.getTuple(), checkElement.getTuple()) == 0) {
+ // If the previous tuple and the head tuple are
+ // identical
+ // then pop the head tuple and push the next tuple from
+ // the tree of head tuple
+
+ // the head element of PQ is useless now
+ reusedElement = outputPriorityQueue.poll();
+ // int treeNum = reusedElement.getTreeNum();
+ pushIntoPriorityQueue(reusedElement.getCursorIndex());
+ } else {
+ // If the previous tuple and the head tuple are
+ // different
+ // the info of previous tuple is useless
+ if (needPush == true) {
+ reusedElement = outputElement;
+ pushIntoPriorityQueue(outputElement.getCursorIndex());
+ needPush = false;
+ }
+ outputElement = null;
+ }
+ }
+ } else {
+ // the priority queue is empty and needPush
+ reusedElement = outputElement;
+ pushIntoPriorityQueue(outputElement.getCursorIndex());
+ needPush = false;
+ outputElement = null;
+ }
+ }
+ }
+
+ @Override
+ public boolean exclusiveLatchNodes() {
+ return false;
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/BTreeBulkLoadRunner.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/BTreeBulkLoadRunner.java
new file mode 100644
index 0000000..d3ae961
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/BTreeBulkLoadRunner.java
@@ -0,0 +1,37 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.perf;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsmtree.datagen.DataGenThread;
+import edu.uci.ics.hyracks.storage.am.lsmtree.datagen.TupleBatch;
+
+public class BTreeBulkLoadRunner extends BTreeRunner {
+
+ protected final float fillFactor;
+
+ public BTreeBulkLoadRunner(int numBatches, int pageSize, int numPages, ITypeTraits[] typeTraits, MultiComparator cmp, float fillFactor)
+ throws HyracksDataException, BTreeException {
+ super(numBatches, pageSize, numPages, typeTraits, cmp);
+ this.fillFactor = fillFactor;
+ }
+
+ @Override
+ public long runExperiment(DataGenThread dataGen, int numThreads) throws Exception {
+ btree.create(btreeFileId);
+ long start = System.currentTimeMillis();
+ IIndexBulkLoadContext bulkLoadCtx = btree.beginBulkLoad(1.0f);
+ for (int i = 0; i < numBatches; i++) {
+ TupleBatch batch = dataGen.tupleBatchQueue.take();
+ for (int j = 0; j < batch.size(); j++) {
+ btree.bulkLoadAddTuple(batch.get(j), bulkLoadCtx);
+ }
+ }
+ btree.endBulkLoad(bulkLoadCtx);
+ long end = System.currentTimeMillis();
+ long time = end - start;
+ return time;
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/BTreePageSizePerf.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/BTreePageSizePerf.java
new file mode 100644
index 0000000..ff02aa1
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/BTreePageSizePerf.java
@@ -0,0 +1,72 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.perf;
+
+import java.util.Enumeration;
+import java.util.logging.Level;
+import java.util.logging.LogManager;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsmtree.datagen.DataGenThread;
+
+public class BTreePageSizePerf {
+ public static void main(String[] args) throws Exception {
+ // Disable logging so we can better see the output times.
+ Enumeration<String> loggers = LogManager.getLogManager().getLoggerNames();
+ while(loggers.hasMoreElements()) {
+ String loggerName = loggers.nextElement();
+ Logger logger = LogManager.getLogManager().getLogger(loggerName);
+ logger.setLevel(Level.OFF);
+ }
+
+ int numTuples = 1000000;
+ int batchSize = 10000;
+ int numBatches = numTuples / batchSize;
+
+ ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE };
+ ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes, 30);
+
+ IBinaryComparator[] cmps = SerdeUtils.serdesToComparators(fieldSerdes, fieldSerdes.length);
+ MultiComparator cmp = new MultiComparator(cmps);
+
+ runExperiment(numBatches, batchSize, 1024, 100000, fieldSerdes, cmp, typeTraits);
+ runExperiment(numBatches, batchSize, 2048, 100000, fieldSerdes, cmp, typeTraits);
+ runExperiment(numBatches, batchSize, 4096, 25000, fieldSerdes, cmp, typeTraits);
+ runExperiment(numBatches, batchSize, 8192, 12500, fieldSerdes, cmp, typeTraits);
+ runExperiment(numBatches, batchSize, 16384, 6250, fieldSerdes, cmp, typeTraits);
+ runExperiment(numBatches, batchSize, 32768, 3125, fieldSerdes, cmp, typeTraits);
+ runExperiment(numBatches, batchSize, 65536, 1564, fieldSerdes, cmp, typeTraits);
+ runExperiment(numBatches, batchSize, 131072, 782, fieldSerdes, cmp, typeTraits);
+ runExperiment(numBatches, batchSize, 262144, 391, fieldSerdes, cmp, typeTraits);
+ }
+
+ private static void runExperiment(int numBatches, int batchSize, int pageSize, int numPages, ISerializerDeserializer[] fieldSerdes, MultiComparator cmp, ITypeTraits[] typeTraits) throws Exception {
+ System.out.println("PAGE SIZE: " + pageSize);
+ System.out.println("NUM PAGES: " + numPages);
+ System.out.println("MEMORY: " + (pageSize * numPages));
+ int repeats = 5;
+ long[] times = new long[repeats];
+ //BTreeRunner runner = new BTreeRunner(numTuples, pageSize, numPages, typeTraits, cmp);
+ InMemoryBTreeRunner runner = new InMemoryBTreeRunner(numBatches, pageSize, numPages, typeTraits, cmp);
+ runner.init();
+ int numThreads = 1;
+ for (int i = 0; i < repeats; i++) {
+ DataGenThread dataGen = new DataGenThread(numBatches, batchSize, 10, numThreads, fieldSerdes, 30, 50, false);
+ dataGen.start();
+ times[i] = runner.runExperiment(dataGen, numThreads);
+ System.out.println("TIME " + i + ": " + times[i] + "ms");
+ }
+ runner.deinit();
+ long avgTime = 0;
+ for (int i = 0; i < repeats; i++) {
+ avgTime += times[i];
+ }
+ avgTime /= repeats;
+ System.out.println("AVG TIME: " + avgTime + "ms");
+ System.out.println("-------------------------------");
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/BTreeRunner.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/BTreeRunner.java
new file mode 100644
index 0000000..46c21e5
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/BTreeRunner.java
@@ -0,0 +1,38 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.perf;
+
+import java.io.File;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
+import edu.uci.ics.hyracks.test.support.TestUtils;
+
+public class BTreeRunner extends InMemoryBTreeRunner {
+ protected static final int MAX_OPEN_FILES = 10;
+ protected static final int HYRACKS_FRAME_SIZE = 128;
+
+ public BTreeRunner(int numTuples, int pageSize, int numPages, ITypeTraits[] typeTraits, MultiComparator cmp) throws HyracksDataException, BTreeException {
+ super(numTuples, pageSize, numPages, typeTraits, cmp);
+ }
+
+ @Override
+ protected void init(int pageSize, int numPages, ITypeTraits[] typeTraits, MultiComparator cmp) throws HyracksDataException, BTreeException {
+ IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
+ TestStorageManagerComponentHolder.init(pageSize, numPages, MAX_OPEN_FILES);
+ bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ FileReference file = new FileReference(new File(fileName));
+ bufferCache.createFile(file);
+ btreeFileId = fmp.lookupFileId(file);
+ bufferCache.openFile(btreeFileId);
+ btree = BTreeUtils
+ .createBTree(bufferCache, btreeFileId, typeTraits, cmp.getComparators(), BTreeLeafFrameType.REGULAR_NSM);
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/ConcurrentSkipListRunner.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/ConcurrentSkipListRunner.java
new file mode 100644
index 0000000..8f90902
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/ConcurrentSkipListRunner.java
@@ -0,0 +1,123 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.perf;
+
+import java.nio.ByteBuffer;
+import java.util.Comparator;
+import java.util.concurrent.ConcurrentSkipListSet;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.lsmtree.datagen.DataGenThread;
+import edu.uci.ics.hyracks.storage.am.lsmtree.datagen.TupleBatch;
+
+public class ConcurrentSkipListRunner implements IExperimentRunner {
+ public class TupleComparator implements Comparator<ITupleReference> {
+ private final MultiComparator cmp;
+
+ public TupleComparator(MultiComparator cmp) {
+ this.cmp = cmp;
+ }
+
+ @Override
+ public int compare(ITupleReference o1, ITupleReference o2) {
+ return cmp.compare(o1, o2);
+ }
+ }
+
+ private final TupleComparator tupleCmp;
+ private final int numBatches;
+ private final int batchSize;
+ private final int tupleSize;
+ private final ITypeTraits[] typeTraits;
+
+ public ConcurrentSkipListRunner(int numBatches, int batchSize, int tupleSize, ITypeTraits[] typeTraits, MultiComparator cmp) {
+ this.numBatches = numBatches;
+ this.tupleSize = tupleSize;
+ this.batchSize = batchSize;
+ this.typeTraits = typeTraits;
+ tupleCmp = new TupleComparator(cmp);
+ }
+
+ @Override
+ public long runExperiment(DataGenThread dataGen, int numThreads) throws InterruptedException {
+ ConcurrentSkipListSet<ITupleReference> skipList = new ConcurrentSkipListSet<ITupleReference>(tupleCmp);
+ SkipListThread[] threads = new SkipListThread[numThreads];
+ int threadNumBatches = numBatches / numThreads;
+ for (int i = 0; i < numThreads; i++) {
+ threads[i] = new SkipListThread(dataGen, skipList, threadNumBatches, batchSize);
+ }
+ // Wait until the tupleBatchQueue is completely full.
+ while (dataGen.tupleBatchQueue.remainingCapacity() != 0) {
+ Thread.sleep(10);
+ }
+
+ long start = System.currentTimeMillis();
+ for (int i = 0; i < numThreads; i++) {
+ threads[i].start();
+ }
+ for (int i = 0; i < numThreads; i++) {
+ threads[i].join();
+ }
+ long end = System.currentTimeMillis();
+ long time = end - start;
+ return time;
+ }
+
+ @Override
+ public void init() throws Exception {
+ }
+
+ @Override
+ public void deinit() throws Exception {
+ }
+
+ public void reset() throws Exception {
+ }
+
+ public class SkipListThread extends Thread {
+ private final DataGenThread dataGen;
+ private final ConcurrentSkipListSet<ITupleReference> skipList;
+ private final int numBatches;
+ public final TypeAwareTupleWriterFactory tupleWriterFactory;
+ public final TypeAwareTupleWriter tupleWriter;
+ public final TypeAwareTupleReference[] tuples;
+ public final ByteBuffer tupleBuf;
+
+ public SkipListThread(DataGenThread dataGen, ConcurrentSkipListSet<ITupleReference> skipList, int numBatches, int batchSize) {
+ this.dataGen = dataGen;
+ this.numBatches = numBatches;
+ this.skipList = skipList;
+ tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
+ tupleWriter = (TypeAwareTupleWriter) tupleWriterFactory.createTupleWriter();
+ int numTuples = numBatches * batchSize;
+ tuples = new TypeAwareTupleReference[numTuples];
+ tupleBuf = ByteBuffer.allocate(numTuples * tupleSize);
+ for (int i = 0; i < numTuples; i++) {
+ tuples[i] = (TypeAwareTupleReference) tupleWriter.createTupleReference();
+ }
+ }
+
+ @Override
+ public void run() {
+ int tupleIndex = 0;
+ try {
+ for (int i = 0; i < numBatches; i++) {
+ TupleBatch batch = dataGen.tupleBatchQueue.take();
+ for (int j = 0; j < batch.size(); j++) {
+ // Copy the tuple to the buffer and set the pre-created tuple ref.
+ tupleWriter.writeTuple(batch.get(j), tupleBuf.array(), tupleIndex * tupleSize);
+ tuples[tupleIndex].resetByTupleOffset(tupleBuf, tupleIndex * tupleSize);
+ skipList.add(tuples[tupleIndex]);
+ tupleIndex++;
+ }
+ }
+ } catch (Exception e) {
+ System.out.println(tupleIndex);
+ e.printStackTrace();
+ }
+ }
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/IExperimentRunner.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/IExperimentRunner.java
new file mode 100644
index 0000000..3533d24
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/IExperimentRunner.java
@@ -0,0 +1,15 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.perf;
+
+import edu.uci.ics.hyracks.storage.am.lsmtree.datagen.DataGenThread;
+
+public interface IExperimentRunner {
+ public static int DEFAULT_MAX_OUTSTANDING = 100000;
+
+ public void init() throws Exception;
+
+ public long runExperiment(DataGenThread dataGen, int numThreads) throws Exception;
+
+ public void reset() throws Exception;
+
+ public void deinit() throws Exception;
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/InMemoryBTreeRunner.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/InMemoryBTreeRunner.java
new file mode 100644
index 0000000..e411856
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/InMemoryBTreeRunner.java
@@ -0,0 +1,125 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.perf;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.lsmtree.datagen.DataGenThread;
+import edu.uci.ics.hyracks.storage.am.lsmtree.datagen.TupleBatch;
+import edu.uci.ics.hyracks.storage.am.lsmtree.freepage.InMemoryBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsmtree.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
+
+public class InMemoryBTreeRunner extends Thread implements IExperimentRunner {
+ protected IBufferCache bufferCache;
+ protected int btreeFileId;
+
+ protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ protected final static String tmpDir = System.getProperty("java.io.tmpdir");
+ protected final static String sep = System.getProperty("file.separator");
+ protected String fileName;
+
+ protected final int numBatches;
+ protected BTree btree;
+
+ public InMemoryBTreeRunner(int numBatches, int pageSize, int numPages, ITypeTraits[] typeTraits, MultiComparator cmp) throws HyracksDataException, BTreeException {
+ this.numBatches = numBatches;
+ fileName = tmpDir + sep + simpleDateFormat.format(new Date());
+ init(pageSize, numPages, typeTraits, cmp);
+ }
+
+ protected void init(int pageSize, int numPages, ITypeTraits[] typeTraits, MultiComparator cmp) throws HyracksDataException, BTreeException {
+ ICacheMemoryAllocator allocator = new HeapBufferAllocator();
+ bufferCache = new InMemoryBufferCache(allocator, pageSize, numPages);
+ // Chose an aribtrary file id.
+ btreeFileId = 0;
+ TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+ IFreePageManager freePageManager = new InMemoryFreePageManager(bufferCache.getNumPages(), metaFrameFactory);
+ btree = new BTree(bufferCache, typeTraits.length, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ }
+
+ @Override
+ public long runExperiment(DataGenThread dataGen, int numThreads) throws Exception {
+ BTreeThread[] threads = new BTreeThread[numThreads];
+ int threadNumBatches = numBatches / numThreads;
+ for (int i = 0; i < numThreads; i++) {
+ threads[i] = new BTreeThread(dataGen, btree, threadNumBatches);
+ }
+ // Wait until the tupleBatchQueue is completely full.
+ while (dataGen.tupleBatchQueue.remainingCapacity() != 0) {
+ Thread.sleep(10);
+ }
+
+ long start = System.currentTimeMillis();
+ for (int i = 0; i < numThreads; i++) {
+ threads[i].start();
+ }
+ for (int i = 0; i < numThreads; i++) {
+ threads[i].join();
+ }
+ long end = System.currentTimeMillis();
+ long time = end - start;
+ return time;
+ }
+
+ @Override
+ public void init() throws Exception {
+ }
+
+ @Override
+ public void deinit() throws Exception {
+ bufferCache.closeFile(btreeFileId);
+ bufferCache.close();
+ }
+
+ @Override
+ public void reset() throws Exception {
+ btree.create(btreeFileId);
+ }
+
+ public class BTreeThread extends Thread {
+ private final DataGenThread dataGen;
+ private final int numBatches;
+ private final ITreeIndexAccessor indexAccessor;
+ public BTreeThread(DataGenThread dataGen, BTree btree, int numBatches) {
+ this.dataGen = dataGen;
+ this.numBatches = numBatches;
+ indexAccessor = btree.createAccessor();
+ }
+
+ @Override
+ public void run() {
+ try {
+ for (int i = 0; i < numBatches; i++) {
+ TupleBatch batch = dataGen.tupleBatchQueue.take();
+ for (int j = 0; j < batch.size(); j++) {
+ try {
+ indexAccessor.insert(batch.get(j));
+ } catch (TreeIndexException e) {
+ }
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/InMemorySortRunner.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/InMemorySortRunner.java
new file mode 100644
index 0000000..90c5c32
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/InMemorySortRunner.java
@@ -0,0 +1,138 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.perf;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.concurrent.ConcurrentSkipListSet;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.lsmtree.datagen.DataGenThread;
+import edu.uci.ics.hyracks.storage.am.lsmtree.datagen.TupleBatch;
+
+public class InMemorySortRunner implements IExperimentRunner {
+ public class TupleComparator implements Comparator<ITupleReference> {
+ private final MultiComparator cmp;
+
+ public TupleComparator(MultiComparator cmp) {
+ this.cmp = cmp;
+ }
+
+ @Override
+ public int compare(ITupleReference o1, ITupleReference o2) {
+ return cmp.compare(o1, o2);
+ }
+ }
+
+ private final TupleComparator tupleCmp;
+ private final int numBatches;
+ private final int batchSize;
+ private final int tupleSize;
+ private final ITypeTraits[] typeTraits;
+
+ private final TypeAwareTupleWriterFactory tupleWriterFactory;
+ private final TypeAwareTupleWriter tupleWriter;
+ private final ArrayList<TypeAwareTupleReference> tuples;
+ private final ByteBuffer tupleBuf;
+
+ public InMemorySortRunner(int numBatches, int batchSize, int tupleSize, ITypeTraits[] typeTraits, MultiComparator cmp) {
+ this.numBatches = numBatches;
+ this.tupleSize = tupleSize;
+ this.batchSize = batchSize;
+ this.typeTraits = typeTraits;
+ tupleCmp = new TupleComparator(cmp);
+ tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
+ tupleWriter = (TypeAwareTupleWriter) tupleWriterFactory.createTupleWriter();
+ int numTuples = numBatches * batchSize;
+ tuples = new ArrayList<TypeAwareTupleReference>();
+ tupleBuf = ByteBuffer.allocate(numTuples * tupleSize);
+ for (int i = 0; i < numTuples; i++) {
+ tuples.add((TypeAwareTupleReference) tupleWriter.createTupleReference());
+ }
+ }
+
+ @Override
+ public long runExperiment(DataGenThread dataGen, int numThreads) throws InterruptedException {
+ // Wait until the tupleBatchQueue is completely full.
+ while (dataGen.tupleBatchQueue.remainingCapacity() != 0) {
+ Thread.sleep(10);
+ }
+
+ long start = System.currentTimeMillis();
+ int tupleIndex = 0;
+ for (int i = 0; i < numBatches; i++) {
+ TupleBatch batch = dataGen.tupleBatchQueue.take();
+ for (int j = 0; j < batch.size(); j++) {
+ // Copy the tuple to the buffer and set the pre-created tuple ref.
+ tupleWriter.writeTuple(batch.get(j), tupleBuf.array(), tupleIndex * tupleSize);
+ tuples.get(tupleIndex).resetByTupleOffset(tupleBuf, tupleIndex * tupleSize);
+ tupleIndex++;
+ }
+ }
+ // Perform the sort.
+ Collections.sort(tuples, tupleCmp);
+ long end = System.currentTimeMillis();
+ long time = end - start;
+ return time;
+ }
+
+ @Override
+ public void init() throws Exception {
+ }
+
+ @Override
+ public void deinit() throws Exception {
+ }
+
+ public void reset() throws Exception {
+ }
+
+ public class SkipListThread extends Thread {
+ private final DataGenThread dataGen;
+ private final ConcurrentSkipListSet<ITupleReference> skipList;
+ private final int numBatches;
+ public final TypeAwareTupleWriterFactory tupleWriterFactory;
+ public final TypeAwareTupleWriter tupleWriter;
+ public final TypeAwareTupleReference[] tuples;
+ public final ByteBuffer tupleBuf;
+
+ public SkipListThread(DataGenThread dataGen, ConcurrentSkipListSet<ITupleReference> skipList, int numBatches, int batchSize) {
+ this.dataGen = dataGen;
+ this.numBatches = numBatches;
+ this.skipList = skipList;
+ tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
+ tupleWriter = (TypeAwareTupleWriter) tupleWriterFactory.createTupleWriter();
+ int numTuples = numBatches * batchSize;
+ tuples = new TypeAwareTupleReference[numTuples];
+ tupleBuf = ByteBuffer.allocate(numTuples * tupleSize);
+ for (int i = 0; i < numTuples; i++) {
+ tuples[i] = (TypeAwareTupleReference) tupleWriter.createTupleReference();
+ }
+ }
+
+ @Override
+ public void run() {
+ int tupleIndex = 0;
+ try {
+ for (int i = 0; i < numBatches; i++) {
+ TupleBatch batch = dataGen.tupleBatchQueue.take();
+ for (int j = 0; j < batch.size(); j++) {
+ // Copy the tuple to the buffer and set the pre-created tuple ref.
+ tupleWriter.writeTuple(batch.get(j), tupleBuf.array(), tupleIndex * tupleSize);
+ tuples[tupleIndex].resetByTupleOffset(tupleBuf, tupleIndex * tupleSize);
+ skipList.add(tuples[tupleIndex]);
+ tupleIndex++;
+ }
+ }
+ } catch (Exception e) {
+ System.out.println(tupleIndex);
+ e.printStackTrace();
+ }
+ }
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/LSMTreeRunner.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/LSMTreeRunner.java
new file mode 100644
index 0000000..b9175ad
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/LSMTreeRunner.java
@@ -0,0 +1,143 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.perf;
+
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsmtree.datagen.DataGenThread;
+import edu.uci.ics.hyracks.storage.am.lsmtree.datagen.TupleBatch;
+import edu.uci.ics.hyracks.storage.am.lsmtree.freepage.InMemoryBufferCacheFactory;
+import edu.uci.ics.hyracks.storage.am.lsmtree.impls.LSMTree;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapManager;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
+import edu.uci.ics.hyracks.test.support.TestUtils;
+
+public class LSMTreeRunner implements IExperimentRunner {
+
+ private static final int MAX_OPEN_FILES = 10000;
+ private static final int HYRACKS_FRAME_SIZE = 128;
+
+ protected IHyracksTaskContext ctx;
+ protected IBufferCache bufferCache;
+ protected int lsmtreeFileId;
+
+ protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ protected final static String tmpDir = System.getProperty("java.io.tmpdir");
+ protected final static String sep = System.getProperty("file.separator");
+ protected final static String classDir = "/lsmtree/";
+ protected String fileName;
+
+ protected final int numBatches;
+ protected final LSMTree lsmtree;
+ protected IBufferCache memBufferCache;
+ private final int onDiskPageSize;
+ private final int onDiskNumPages;
+
+ public LSMTreeRunner(int numBatches, int inMemPageSize, int inMeNumPages, int onDiskPageSize, int onDiskNumPages, ITypeTraits[] typeTraits, MultiComparator cmp) throws HyracksDataException, BTreeException {
+ this.numBatches = numBatches;
+
+ this.onDiskPageSize = onDiskPageSize;
+ this.onDiskNumPages = onDiskNumPages;
+
+ fileName = tmpDir + classDir + sep + simpleDateFormat.format(new Date());
+ ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
+
+
+ TestStorageManagerComponentHolder.init(this.onDiskPageSize, this.onDiskNumPages, MAX_OPEN_FILES);
+ bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ FileReference file = new FileReference(new File(fileName));
+ bufferCache.createFile(file);
+ lsmtreeFileId = fmp.lookupFileId(file);
+ bufferCache.openFile(lsmtreeFileId);
+
+
+ // In Memory
+ InMemoryBufferCacheFactory InMemBufferCacheFactory = new InMemoryBufferCacheFactory(inMemPageSize, inMeNumPages);
+ memBufferCache = InMemBufferCacheFactory.createInMemoryBufferCache();
+
+ lsmtree = LSMTreeUtils.createLSMTree(memBufferCache, bufferCache, lsmtreeFileId, typeTraits, cmp.getComparators(), BTreeLeafFrameType.REGULAR_NSM, (IFileMapManager)fmp);
+ }
+ @Override
+ public void init() throws Exception {
+ }
+
+ @Override
+ public long runExperiment(DataGenThread dataGen, int numThreads) throws Exception {
+ LSMTreeThread[] threads = new LSMTreeThread[numThreads];
+ int threadNumBatches = numBatches / numThreads;
+ for (int i = 0; i < numThreads; i++) {
+ threads[i] = new LSMTreeThread(dataGen, lsmtree, threadNumBatches);
+ }
+ // Wait until the tupleBatchQueue is completely full.
+ while (dataGen.tupleBatchQueue.remainingCapacity() != 0) {
+ Thread.sleep(10);
+ }
+
+ long start = System.currentTimeMillis();
+ for (int i = 0; i < numThreads; i++) {
+ threads[i].start();
+ }
+ for (int i = 0; i < numThreads; i++) {
+ threads[i].join();
+ }
+ long end = System.currentTimeMillis();
+ long time = end - start;
+ return time;
+ }
+
+ @Override
+ public void reset() throws Exception {
+ lsmtree.create(lsmtreeFileId);
+ }
+
+ @Override
+ public void deinit() throws Exception {
+ bufferCache.closeFile(lsmtreeFileId);
+ bufferCache.close();
+ memBufferCache.closeFile(lsmtreeFileId);
+ memBufferCache.close();
+ }
+
+ public class LSMTreeThread extends Thread {
+ private final DataGenThread dataGen;
+ private final LSMTree lsmTree;
+ private final int numBatches;
+ private final ITreeIndexAccessor lsmTreeAccessor;
+ public LSMTreeThread(DataGenThread dataGen, LSMTree lsmTree, int numBatches) {
+ this.dataGen = dataGen;
+ this.lsmTree = lsmTree;
+ this.numBatches = numBatches;
+ lsmTreeAccessor = lsmTree.createAccessor();
+ }
+
+ @Override
+ public void run() {
+ try {
+ for (int i = 0; i < numBatches; i++) {
+ TupleBatch batch = dataGen.tupleBatchQueue.take();
+ for (int j = 0; j < batch.size(); j++) {
+ try {
+ lsmTreeAccessor.insert(batch.get(j));
+ } catch (TreeIndexException e) {
+ }
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/LSMTreeUtils.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/LSMTreeUtils.java
new file mode 100644
index 0000000..853f658
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/LSMTreeUtils.java
@@ -0,0 +1,46 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.perf;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsmtree.freepage.FreePageManagerFactory;
+import edu.uci.ics.hyracks.storage.am.lsmtree.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsmtree.impls.BTreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsmtree.impls.LSMTree;
+import edu.uci.ics.hyracks.storage.am.lsmtree.tuples.LSMEntireTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.lsmtree.tuples.LSMTypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapManager;
+
+public class LSMTreeUtils {
+ public static LSMTree createLSMTree(IBufferCache memCache, IBufferCache bufferCache, int lsmtreeFileId, ITypeTraits[] typeTraits, IBinaryComparator[] cmps, BTreeLeafFrameType leafType, IFileMapManager fileMapManager) throws BTreeException {
+ MultiComparator cmp = new MultiComparator(cmps);
+
+ LSMTypeAwareTupleWriterFactory insertTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
+ LSMTypeAwareTupleWriterFactory deleteTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
+
+ ITreeIndexFrameFactory insertLeafFrameFactory = BTreeUtils.getLeafFrameFactory(insertTupleWriterFactory,leafType);
+ ITreeIndexFrameFactory deleteLeafFrameFactory = BTreeUtils.getLeafFrameFactory(deleteTupleWriterFactory,leafType);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+ IFreePageManager memFreePageManager = new InMemoryFreePageManager(memCache.getNumPages(), metaFrameFactory);
+
+ // For the Flush Mechanism
+ LSMEntireTupleWriterFactory flushTupleWriterFactory = new LSMEntireTupleWriterFactory(typeTraits);
+ ITreeIndexFrameFactory flushLeafFrameFactory = BTreeUtils.getLeafFrameFactory(flushTupleWriterFactory,leafType);
+ FreePageManagerFactory freePageManagerFactory = new FreePageManagerFactory(bufferCache, metaFrameFactory);
+ BTreeFactory bTreeFactory = new BTreeFactory(bufferCache, freePageManagerFactory, cmp, typeTraits.length, interiorFrameFactory, flushLeafFrameFactory);
+
+ LSMTree lsmtree = new LSMTree(memCache, bufferCache, typeTraits.length, cmp, memFreePageManager,
+ interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, bTreeFactory, fileMapManager);
+ return lsmtree;
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/PerfExperiment.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/PerfExperiment.java
new file mode 100644
index 0000000..ee0f1fd
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/perf/PerfExperiment.java
@@ -0,0 +1,158 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.perf;
+
+import java.io.FileOutputStream;
+import java.io.PrintStream;
+import java.util.Enumeration;
+import java.util.logging.Level;
+import java.util.logging.LogManager;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsmtree.datagen.DataGenThread;
+
+public class PerfExperiment {
+ public static void main(String[] args) throws Exception {
+ // Disable logging so we can better see the output times.
+ Enumeration<String> loggers = LogManager.getLogManager().getLoggerNames();
+ while(loggers.hasMoreElements()) {
+ String loggerName = loggers.nextElement();
+ Logger logger = LogManager.getLogManager().getLogger(loggerName);
+ logger.setLevel(Level.OFF);
+ }
+
+ //int numTuples = 1000;
+ //int batchSize = 100;
+ int numTuples = 100000; // 100K
+ //int numTuples = 1000000; // 1M
+ //int numTuples = 2000000; // 2M
+ //int numTuples = 3000000; // 3M
+ //int numTuples = 10000000; // 10M
+ //int numTuples = 20000000; // 20M
+ //int numTuples = 30000000; // 30M
+ //int numTuples = 40000000; // 40M
+ //int numTuples = 60000000; // 60M
+ //int numTuples = 100000000; // 100M
+ //int numTuples = 200000000; // 200M
+ int batchSize = 10000;
+ int numBatches = numTuples / batchSize;
+
+ ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE };
+ ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes, 30);
+
+ FileOutputStream out;
+ PrintStream p;
+ out = new FileOutputStream("/tmp/testOutput.txt");
+ p = new PrintStream( out );
+
+
+ // Add 1 byte for the null flags.
+ // TODO: hide this in some method.
+ int tupleSize = 4 + 30 + 1;
+
+ IBinaryComparator[] cmps = SerdeUtils.serdesToComparators(fieldSerdes, fieldSerdes.length);
+ MultiComparator cmp = new MultiComparator(cmps);
+
+
+ //Test 30M Btree
+ try{
+ p.println ("Start for test 30M BTree");
+ }catch (Exception e){
+ System.err.println ("Error writing to file");
+ }
+
+
+ //int repeats = 1000;
+ int repeats = 1;
+ long[] times = new long[repeats];
+
+ int numThreads = 1;
+ for (int i = 0; i < repeats; i++) {
+ //ConcurrentSkipListRunner runner = new ConcurrentSkipListRunner(numBatches, batchSize, tupleSize, typeTraits, cmp);
+
+ //InMemoryBTreeRunner runner = new InMemoryBTreeRunner(numBatches, 8192, 100000, typeTraits, cmp);
+ //BTreeBulkLoadRunner runner = new BTreeBulkLoadRunner(numBatches, 8192, 100000, typeTraits, cmp, 1.0f);
+
+ //BTreeRunner runner = new BTreeRunner(numBatches, 8192, 100000, typeTraits, cmp);
+ //String btreeName = "071211";
+ //BTreeSearchRunner runner = new BTreeSearchRunner(btreeName, 10, numBatches, 8192, 25000, typeTraits, cmp);
+ LSMTreeRunner runner = new LSMTreeRunner(numBatches, 8192, 100, 8192, 250, typeTraits, cmp);
+ //LSMTreeSearchRunner runner = new LSMTreeSearchRunner(100000, numBatches, 8192, 24750, 8192, 250, typeTraits, cmp);
+ DataGenThread dataGen = new DataGenThread(numBatches, batchSize, 10, numThreads, fieldSerdes, 30, 50, false);
+ dataGen.start();
+ runner.reset();
+ times[i] = runner.runExperiment(dataGen, numThreads);
+ System.out.println("TIME " + i + ": " + times[i] + "ms");
+ try{
+ p.println ("TIME " + i + ": " + times[i] + "ms");
+ }catch (Exception e){
+ System.err.println ("Error writing to file");
+ }
+ runner.deinit();
+ }
+
+ long avgTime = 0;
+ for (int i = 0; i < repeats; i++) {
+ avgTime += times[i];
+ }
+
+ avgTime /= repeats;
+ System.out.println("AVG TIME: " + avgTime + "ms");
+ try{
+ p.println ("AVG TIME: " + avgTime + "ms");
+ }catch (Exception e){
+ System.err.println ("Error writing to file");
+ }
+
+
+
+/*
+ //Test 30M Btree
+ try{
+ p.println ("Start for test 40M LSMTree");
+ }catch (Exception e){
+ System.err.println ("Error writing to file");
+ }
+
+ numTuples = 40000000; // 40M
+ //numTuples = 1000000; // 100K
+ numBatches = numTuples / batchSize;
+ runner = new BTreeRunner(numBatches, 8192, 100000, typeTraits, cmp);
+
+ runner.init();
+ for (int i = 0; i < repeats; i++) {
+ // LSMTreeRunner runner = new LSMTreeRunner(numBatches, 8192, 100000, typeTraits, cmp);
+ DataGenThread dataGen = new DataGenThread(numBatches, batchSize, 10, numThreads, fieldSerdes, 30, 50, false);
+ dataGen.start();
+ runner.reset();
+ times[i] = runner.runExperiment(dataGen, numThreads);
+ System.out.println("TIME " + i + ": " + times[i] + "ms");
+ try{
+ p.println ("TIME " + i + ": " + times[i] + "ms");
+ }catch (Exception e){
+ System.err.println ("Error writing to file");
+ }
+ // runner.deinit();
+ }
+ runner.deinit();
+
+ avgTime = 0;
+ for (int i = 0; i < repeats; i++) {
+ avgTime += times[i];
+ }
+
+ avgTime /= repeats;
+ System.out.println("AVG TIME: " + avgTime + "ms");
+ try{
+ p.println ("AVG TIME: " + avgTime + "ms");
+ }catch (Exception e){
+ System.err.println ("Error writing to file");
+ }
+ */
+ p.close();
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/ILSMTreeTupleReference.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/ILSMTreeTupleReference.java
new file mode 100644
index 0000000..68f2672
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/ILSMTreeTupleReference.java
@@ -0,0 +1,7 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.tuples;
+
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+
+public interface ILSMTreeTupleReference extends ITreeIndexTupleReference {
+ public boolean isDelete();
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMEntireTupleWriter.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMEntireTupleWriter.java
new file mode 100644
index 0000000..e94ef1f
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMEntireTupleWriter.java
@@ -0,0 +1,25 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.tuples;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
+
+public class LSMEntireTupleWriter extends TypeAwareTupleWriter {
+ public LSMEntireTupleWriter(ITypeTraits[] typeTraits){
+ super(typeTraits);
+ }
+ @Override
+ protected int getNullFlagsBytes(ITupleReference tuple) {
+ //+1.0 is for insert/delete tuple checking
+ return (int) Math.ceil(((double) tuple.getFieldCount() + 1.0) / 8.0);
+ }
+
+ @Override
+ public int writeTuple(ITupleReference tuple, byte[] targetBuf, int targetOff) {
+ int tupleSize = this.bytesRequired(tuple);
+ byte[] buf = tuple.getFieldData(0);
+ int tupleStartOff = ((LSMTypeAwareTupleReference)tuple).getTupleStart();
+ System.arraycopy(buf, tupleStartOff, targetBuf, targetOff, tupleSize);
+ return tupleSize;
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMEntireTupleWriterFactory.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMEntireTupleWriterFactory.java
new file mode 100644
index 0000000..6bbc7a6
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMEntireTupleWriterFactory.java
@@ -0,0 +1,20 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.tuples;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+
+public class LSMEntireTupleWriterFactory extends TypeAwareTupleWriterFactory {
+ private static final long serialVersionUID = 1L;
+ private ITypeTraits[] typeTraits;
+
+ public LSMEntireTupleWriterFactory(ITypeTraits[] typeTraits) {
+ super(typeTraits);
+ this.typeTraits = typeTraits;
+ }
+
+ @Override
+ public ITreeIndexTupleWriter createTupleWriter() {
+ return new LSMEntireTupleWriter(typeTraits);
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleReference.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleReference.java
new file mode 100644
index 0000000..a5a3ee4
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleReference.java
@@ -0,0 +1,37 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.tuples;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleReference;
+
+public class LSMTypeAwareTupleReference extends TypeAwareTupleReference implements ILSMTreeTupleReference {
+
+ public LSMTypeAwareTupleReference(ITypeTraits[] typeTraits) {
+ super(typeTraits);
+ }
+
+ @Override
+ protected int getNullFlagsBytes() {
+ //+1.0 is for insert/delete tuple checking
+ return (int) Math.ceil((fieldCount + 1.0) / 8.0);
+ }
+
+ @Override
+ public boolean isDelete() {
+ byte[] temp = buf.array();
+ byte firstByte = temp[tupleStartOff];
+ final byte mask = (byte) (1 << 7);
+ final byte compare = (byte) (1 << 7);
+
+ //check the first bit is 0 or 1
+ if((byte)(firstByte & mask) == compare) {
+ return true;
+ }
+ else {
+ return false;
+ }
+ }
+
+ public int getTupleStart() {
+ return tupleStartOff;
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleReferenceTest.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleReferenceTest.java
new file mode 100644
index 0000000..77fe2f3
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleReferenceTest.java
@@ -0,0 +1,71 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.tuples;
+
+import static org.junit.Assert.fail;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+
+public class LSMTypeAwareTupleReferenceTest {
+
+ @Test
+ public void test01() throws Exception {
+
+ int targetOff = 0;
+ ByteBuffer buf = ByteBuffer.allocate(32);
+
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+
+ LSMTypeAwareTupleWriterFactory insertTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
+ ITreeIndexTupleWriter insertTupleWriter = insertTupleWriterFactory.createTupleWriter();
+ ITreeIndexTupleReference lsmTupleReference = insertTupleWriter.createTupleReference();
+
+ lsmTupleReference.resetByTupleOffset(buf, targetOff);
+ insertTupleWriter.writeTuple(lsmTupleReference, buf, targetOff);
+
+ boolean del = ((LSMTypeAwareTupleReference) lsmTupleReference).isDelete();
+
+ if(del == false) {
+ return;
+ }
+ else {
+ fail("fail to write tuple");
+ }
+ }
+
+ @Test
+ public void test02() throws Exception {
+
+ int targetOff = 0;
+ ByteBuffer buf = ByteBuffer.allocate(32);
+
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+
+ LSMTypeAwareTupleWriterFactory deleteTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
+ ITreeIndexTupleWriter deleteTupleWriter = deleteTupleWriterFactory.createTupleWriter();
+ ITreeIndexTupleReference lsmTupleReference = deleteTupleWriter.createTupleReference();
+
+ lsmTupleReference.resetByTupleOffset(buf, targetOff);
+ deleteTupleWriter.writeTuple(lsmTupleReference, buf, targetOff);
+
+ boolean del = ((LSMTypeAwareTupleReference) lsmTupleReference).isDelete();
+
+ if(del == true) {
+ return;
+ }
+ else {
+ fail("fail to write tuple");
+ }
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleWriter.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleWriter.java
new file mode 100644
index 0000000..d4dd104
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleWriter.java
@@ -0,0 +1,47 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.tuples;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
+
+public class LSMTypeAwareTupleWriter extends TypeAwareTupleWriter {
+ private final boolean isDelete;
+
+ public LSMTypeAwareTupleWriter(ITypeTraits[] typeTraits, boolean isDelete) {
+ super(typeTraits);
+ this.isDelete = isDelete;
+ }
+
+ @Override
+ public ITreeIndexTupleReference createTupleReference() {
+ return new LSMTypeAwareTupleReference(typeTraits);
+ }
+
+ @Override
+ protected int getNullFlagsBytes(int numFields) {
+ //+1.0 is for insert/delete tuple checking
+ return (int) Math.ceil(((double) numFields + 1.0)/ 8.0);
+ }
+
+ @Override
+ protected int getNullFlagsBytes(ITupleReference tuple) {
+ //+1.0 is for insert/delete tuple checking
+ return (int) Math.ceil(((double) tuple.getFieldCount() + 1.0) / 8.0);
+ }
+
+ @Override
+ public int writeTuple(ITupleReference tuple, byte[] targetBuf, int targetOff) {
+ int bytesWritten = super.writeTuple(tuple, targetBuf, targetOff);
+ if(isDelete) {
+ setDeleteBit(targetBuf, targetOff);
+ }
+ return bytesWritten;
+ }
+
+ private void setDeleteBit(byte[] targetBuf, int targetOff) {
+ byte firstByte = targetBuf[targetOff];
+ firstByte = (byte) (firstByte | (1 << 7));
+ targetBuf[targetOff] = firstByte;
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleWriterFactory.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleWriterFactory.java
new file mode 100644
index 0000000..d424a61
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleWriterFactory.java
@@ -0,0 +1,24 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.tuples;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+
+public class LSMTypeAwareTupleWriterFactory extends TypeAwareTupleWriterFactory {
+
+ private static final long serialVersionUID = 1L;
+ private ITypeTraits[] typeTraits;
+ private final boolean isDelete;
+
+ public LSMTypeAwareTupleWriterFactory(ITypeTraits[] typeTraits, boolean isDelete) {
+ super(typeTraits);
+ this.typeTraits = typeTraits;
+ this.isDelete = isDelete;
+ }
+
+ @Override
+ public ITreeIndexTupleWriter createTupleWriter() {
+ return new LSMTypeAwareTupleWriter(typeTraits, isDelete);
+ }
+
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleWriterFactoryTest.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleWriterFactoryTest.java
new file mode 100644
index 0000000..a9cab04
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleWriterFactoryTest.java
@@ -0,0 +1,31 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.tuples;
+
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+
+public class LSMTypeAwareTupleWriterFactoryTest {
+ @Test
+ public void test01() throws Exception {
+
+ // declare fields
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+
+ LSMTypeAwareTupleWriterFactory lsmTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
+ ITreeIndexTupleWriter lsmTupleWriter = lsmTupleWriterFactory.createTupleWriter();
+
+ if(lsmTupleWriter != null) {
+ return;
+ }
+ else {
+ fail("fail to create LSMTypeAwareTupleWriter!");
+ }
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleWriterTest.java b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleWriterTest.java
new file mode 100644
index 0000000..75e76bb
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/tuples/LSMTypeAwareTupleWriterTest.java
@@ -0,0 +1,101 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.tuples;
+
+import static org.junit.Assert.fail;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+
+public class LSMTypeAwareTupleWriterTest {
+
+ // test create tuple reference
+ @Test
+ public void test01() throws Exception {
+
+ // declare fields
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+
+ LSMTypeAwareTupleWriterFactory lsmTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(
+ typeTraits, false);
+ ITreeIndexTupleWriter lsmTupleWriter = lsmTupleWriterFactory.createTupleWriter();
+
+ ITreeIndexTupleReference lsmTupleReference = lsmTupleWriter.createTupleReference();
+
+ if (lsmTupleReference != null) {
+ return;
+ } else {
+ fail("fail to create LSMTypeAwareTupleWriter");
+ }
+ }
+
+ // test insert tuple writer
+ @Test
+ public void test02() throws Exception {
+
+ int targetOff = 0;
+ ByteBuffer buf = ByteBuffer.allocate(32);
+
+ // declare fields
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+
+ LSMTypeAwareTupleWriterFactory insertTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(
+ typeTraits, false);
+ ITreeIndexTupleWriter insertTupleWriter = insertTupleWriterFactory.createTupleWriter();
+ ITreeIndexTupleReference insertTupleReference = insertTupleWriter.createTupleReference();
+
+ insertTupleReference.resetByTupleOffset(buf, targetOff);
+
+ int num = insertTupleWriter.writeTuple(insertTupleReference, buf, targetOff);
+
+ boolean del = ((LSMTypeAwareTupleReference) insertTupleReference).isDelete();
+
+ if (num == 9 && del == false) {
+ return;
+ } else {
+ fail("fail to write tuple");
+ }
+ }
+
+ // test delete tuple writer
+ @Test
+ public void test03() throws Exception {
+
+ int targetOff = 0;
+ ByteBuffer buf = ByteBuffer.allocate(32);
+
+ // declare fields
+ int fieldCount = 3;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[2] = IntegerPointable.TYPE_TRAITS;
+
+ LSMTypeAwareTupleWriterFactory deleteTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(
+ typeTraits, true);
+ ITreeIndexTupleWriter deleteTupleWriter = deleteTupleWriterFactory.createTupleWriter();
+ ITreeIndexTupleReference deleteTupleReference = deleteTupleWriter.createTupleReference();
+
+ deleteTupleReference.resetByTupleOffset(buf, targetOff);
+
+ int num = deleteTupleWriter.writeTuple(deleteTupleReference, buf, targetOff);
+
+ boolean del = ((LSMTypeAwareTupleReference) deleteTupleReference).isDelete();
+
+ if (num == 13 && del == true) {
+ return;
+ } else {
+ fail("fail to write tuple");
+ }
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-common/pom.xml b/hyracks-storage-am-lsmtree-common/pom.xml
new file mode 100644
index 0000000..3a02f53
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-common/pom.xml
@@ -0,0 +1,42 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsmtree-common</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-common</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <type>jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/api/ILSMTree.java b/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/api/ILSMTree.java
new file mode 100644
index 0000000..46cdc2e
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/api/ILSMTree.java
@@ -0,0 +1,9 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.common.api;
+
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+
+public interface ILSMTree extends ITreeIndex {
+ public void merge() throws Exception;
+
+ public void flush() throws Exception;
+}
diff --git a/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/impls/FreePageManagerFactory.java b/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/impls/FreePageManagerFactory.java
new file mode 100644
index 0000000..a0591a6
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/impls/FreePageManagerFactory.java
@@ -0,0 +1,23 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.common.impls;
+
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+
+public class FreePageManagerFactory {
+
+ private final ITreeIndexMetaDataFrameFactory metaDataFrameFactory;
+ private final IBufferCache bufferCache;
+
+ public FreePageManagerFactory(IBufferCache bufferCache, ITreeIndexMetaDataFrameFactory metaDataFrameFactory) {
+ this.metaDataFrameFactory = metaDataFrameFactory;
+ this.bufferCache = bufferCache;
+ }
+
+ public IFreePageManager createFreePageManager(int fileId) {
+ return new LinkedListFreePageManager(bufferCache, fileId, 0, metaDataFrameFactory);
+ }
+
+}
diff --git a/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/impls/InMemoryBufferCache.java b/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/impls/InMemoryBufferCache.java
new file mode 100644
index 0000000..25891dc
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/impls/InMemoryBufferCache.java
@@ -0,0 +1,146 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.common.impls;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCacheInternal;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPageInternal;
+import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+
+public class InMemoryBufferCache implements IBufferCacheInternal {
+
+ private final int pageSize;
+ private final int numPages;
+ protected final CachedPage[] cachedPages;
+
+ //Constructor
+ public InMemoryBufferCache(ICacheMemoryAllocator allocator, int pageSize, int numPages){
+
+ this.pageSize = pageSize;
+ this.numPages = numPages;
+ ByteBuffer[] buffers = allocator.allocate(this.pageSize, this.numPages);
+ cachedPages = new CachedPage[buffers.length];
+ for (int i = 0; i < buffers.length; ++i) {
+ cachedPages[i] = new CachedPage(i, buffers[i]);
+ }
+ }
+
+ @Override
+ public void createFile(FileReference fileRef) throws HyracksDataException {
+ // Do nothing
+ }
+
+ @Override
+ public void openFile(int fileId) throws HyracksDataException {
+ // Do nothing
+ }
+
+ @Override
+ public void closeFile(int fileId) throws HyracksDataException {
+ // Do nothing
+ }
+
+ @Override
+ public void deleteFile(int fileId) throws HyracksDataException {
+ // Do nothing
+ }
+
+ @Override
+ public ICachedPage tryPin(long dpid) throws HyracksDataException {
+ // Just call pin!
+ return null;
+ }
+
+ @Override
+ public ICachedPage pin(long dpid, boolean newPage){
+ return cachedPages[BufferedFileHandle.getPageId(dpid)];
+ }
+
+ @Override
+ public void unpin(ICachedPage page) throws HyracksDataException {
+ //Do Nothing
+ }
+
+ @Override
+ public int getPageSize() {
+ return pageSize;
+ }
+
+ @Override
+ public int getNumPages() {
+ return numPages;
+ }
+
+ @Override
+ public void close() {
+ // Do nothing
+ }
+
+ @Override
+ public ICachedPageInternal getPage(int cpid) {
+ return cachedPages[cpid];
+ }
+
+ private class CachedPage implements ICachedPageInternal {
+ private final int cpid;
+ private final ByteBuffer buffer;
+ private final ReadWriteLock latch;
+
+ public CachedPage(int cpid, ByteBuffer buffer) {
+ this.cpid = cpid;
+ this.buffer = buffer;
+ latch = new ReentrantReadWriteLock(true);
+ }
+
+ @Override
+ public ByteBuffer getBuffer() {
+ return buffer;
+ }
+
+ @Override
+ public Object getReplacementStrategyObject() {
+ //Do nothing
+ return null;
+ }
+
+ @Override
+ public boolean pinIfGoodVictim() {
+ //Do nothing
+ return false;
+ }
+
+ @Override
+ public int getCachedPageId() {
+ return cpid;
+ }
+
+ @Override
+ public void acquireReadLatch() {
+ latch.readLock().lock();
+ }
+
+ private void acquireWriteLatch(boolean markDirty) {
+ latch.writeLock().lock();
+ }
+
+ @Override
+ public void acquireWriteLatch() {
+ acquireWriteLatch(true);
+ }
+
+ @Override
+ public void releaseReadLatch() {
+ latch.readLock().unlock();
+ }
+
+ @Override
+ public void releaseWriteLatch() {
+ latch.writeLock().unlock();
+ }
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/impls/InMemoryBufferCacheFactory.java b/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/impls/InMemoryBufferCacheFactory.java
new file mode 100644
index 0000000..fca8dc4
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/impls/InMemoryBufferCacheFactory.java
@@ -0,0 +1,26 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.common.impls;
+
+import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
+
+public class InMemoryBufferCacheFactory {
+
+ private IBufferCache bufferCache;
+ private final int pageSize;
+ private final int numPages;
+
+ public InMemoryBufferCacheFactory(int pageSize, int numPages) {
+ this.pageSize = pageSize;
+ this.numPages = numPages;
+ bufferCache = null;
+ }
+
+ public synchronized IBufferCache createInMemoryBufferCache() {
+ if (bufferCache == null) {
+ ICacheMemoryAllocator allocator = new HeapBufferAllocator();
+ bufferCache = new InMemoryBufferCache(allocator, pageSize, numPages);
+ }
+ return bufferCache;
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/impls/InMemoryFreePageManager.java b/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/impls/InMemoryFreePageManager.java
new file mode 100644
index 0000000..5bafedf
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/impls/InMemoryFreePageManager.java
@@ -0,0 +1,84 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.common.impls;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
+
+public class InMemoryFreePageManager implements IFreePageManager {
+ private final int maxCapacity;
+ protected int currentCapacity;
+ private final ITreeIndexMetaDataFrameFactory metaDataFrameFactory;
+
+ public InMemoryFreePageManager(int maxCapacity, ITreeIndexMetaDataFrameFactory metaDataFrameFactory) {
+ this.maxCapacity = maxCapacity - 1; // Since the range of CacheArray in
+ // InMemoryBufferCache is 0 ~
+ // maxCapacity-1
+ currentCapacity = 1;
+ this.metaDataFrameFactory = metaDataFrameFactory;
+ }
+
+ public int getCurrentCapacity() {
+ return currentCapacity;
+ }
+
+ @Override
+ public synchronized int getFreePage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException,
+ PageAllocationException {
+
+ if (currentCapacity == maxCapacity) {
+ throw new PageAllocationException("In-mem tree capacity reaches max capacity");
+ }
+ currentCapacity++;
+ return currentCapacity;
+ }
+
+ @Override
+ public void addFreePage(ITreeIndexMetaDataFrame metaFrame, int freePage) throws HyracksDataException {
+ System.out.println("InMemoryFreePageManager.addFreePage()");
+ }
+
+ @Override
+ public int getMaxPage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
+ return currentCapacity;
+ }
+
+ @Override
+ public void init(ITreeIndexMetaDataFrame metaFrame, int currentMaxPage) throws HyracksDataException {
+ currentCapacity = 1;
+ }
+
+ @Override
+ public ITreeIndexMetaDataFrameFactory getMetaDataFrameFactory() {
+ return metaDataFrameFactory;
+ }
+
+ @Override
+ public byte getMetaPageLevelIndicator() {
+ System.out.println("InMemoryFreePageManager.getMetaPageLevelIndicator()");
+ return 0;
+ }
+
+ @Override
+ public byte getFreePageLevelIndicator() {
+ System.out.println("InMemoryFreePageManager.getFreePageLevelIndicator()");
+ return 0;
+ }
+
+ @Override
+ public boolean isMetaPage(ITreeIndexMetaDataFrame metaFrame) {
+ System.out.println("InMemoryFreePageManager.isMetaPage()");
+ return false;
+ }
+
+ @Override
+ public boolean isFreePage(ITreeIndexMetaDataFrame metaFrame) {
+ System.out.println("InMemoryFreePageManager.isFreePage()");
+ return false;
+ }
+
+ public void reset() {
+ currentCapacity = 1;
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/impls/TreeFactory.java b/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/impls/TreeFactory.java
new file mode 100644
index 0000000..7903d18
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/common/impls/TreeFactory.java
@@ -0,0 +1,29 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.common.impls;
+
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+public abstract class TreeFactory {
+
+ protected IBufferCache bufferCache;
+ protected int fieldCount;
+ protected MultiComparator cmp;
+ protected ITreeIndexFrameFactory interiorFrameFactory;
+ protected ITreeIndexFrameFactory leafFrameFactory;
+ protected FreePageManagerFactory freePageManagerFactory;
+
+ public TreeFactory(IBufferCache bufferCache, FreePageManagerFactory freePageManagerFactory, MultiComparator cmp,
+ int fieldCount, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
+ this.bufferCache = bufferCache;
+ this.fieldCount = fieldCount;
+ this.cmp = cmp;
+ this.interiorFrameFactory = interiorFrameFactory;
+ this.leafFrameFactory = leafFrameFactory;
+ this.freePageManagerFactory = freePageManagerFactory;
+ }
+
+ public abstract ITreeIndex createIndexInstance(int fileId);
+
+}
diff --git a/hyracks-storage-am-lsmtree-rtree/pom.xml b/hyracks-storage-am-lsmtree-rtree/pom.xml
new file mode 100644
index 0000000..2c9b94c
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-rtree/pom.xml
@@ -0,0 +1,56 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsmtree-rtree</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsmtree-common</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-btree</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-rtree</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <type>jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/BTreeFactory.java b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/BTreeFactory.java
new file mode 100644
index 0000000..a60cc60
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/BTreeFactory.java
@@ -0,0 +1,24 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.rtree.impls;
+
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsmtree.common.impls.FreePageManagerFactory;
+import edu.uci.ics.hyracks.storage.am.lsmtree.common.impls.TreeFactory;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+public class BTreeFactory extends TreeFactory {
+
+ public BTreeFactory(IBufferCache bufferCache, FreePageManagerFactory freePageManagerFactory, MultiComparator cmp,
+ int fieldCount, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
+ super(bufferCache, freePageManagerFactory, cmp, fieldCount, interiorFrameFactory, leafFrameFactory);
+ }
+
+ @Override
+ public ITreeIndex createIndexInstance(int fileId) {
+ return new BTree(bufferCache, fieldCount, cmp, freePageManagerFactory.createFreePageManager(fileId),
+ interiorFrameFactory, leafFrameFactory);
+ }
+
+}
diff --git a/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMBTreeOpContext.java b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMBTreeOpContext.java
new file mode 100644
index 0000000..f35ee34
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMBTreeOpContext.java
@@ -0,0 +1,30 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.rtree.impls;
+
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+
+public final class LSMBTreeOpContext extends BTreeOpContext {
+
+ public final BTree.BTreeAccessor memBtreeAccessor;
+
+ public LSMBTreeOpContext(BTree.BTreeAccessor memBtreeAccessor, ITreeIndexFrameFactory leafFrameFactory,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexMetaDataFrame metaFrame, MultiComparator cmp) {
+ super(leafFrameFactory, interiorFrameFactory, metaFrame, cmp);
+
+ this.memBtreeAccessor = memBtreeAccessor;
+ // Overwrite the BTree accessor's op context with our LSMBTreeOpContext.
+ this.memBtreeAccessor.setOpContext(this);
+
+ reset(op);
+ }
+
+ @Override
+ public void reset(IndexOp newOp) {
+ super.reset(newOp);
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMRTree.java b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMRTree.java
new file mode 100644
index 0000000..7fd915f
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMRTree.java
@@ -0,0 +1,514 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.rtree.impls;
+
+import java.io.File;
+import java.util.LinkedList;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
+import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsmtree.common.api.ILSMTree;
+import edu.uci.ics.hyracks.storage.am.lsmtree.common.impls.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
+import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.SearchPredicate;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapManager;
+
+public class LSMRTree implements ILSMTree {
+
+ private final IBufferCache bufferCache;
+ private RTree memRTree;
+ private BTree memBTree;
+ private String rtreeFileName;
+ private String btreeFileName;
+ private int rtreeFileId;
+ private int btreeFileId;
+ private boolean created;
+
+ private final IFreePageManager memFreePageManager;
+ private final ITreeIndexFrameFactory rtreeInteriorFrameFactory;
+ private final ITreeIndexFrameFactory btreeInteriorFrameFactory;
+ private final ITreeIndexFrameFactory rtreeLeafFrameFactory;
+ private final ITreeIndexFrameFactory btreeLeafFrameFactory;
+ private final MultiComparator cmp;
+
+ // TODO: change to private, it's public only for LSMTreeSearchTest
+ public LinkedList<ITreeIndex> inDiskRTreeList;
+ public LinkedList<ITreeIndex> inDiskBTreeList;
+ private LinkedList<ITreeIndex> mergedInDiskRTreeList;
+ private LinkedList<ITreeIndex> mergedInDiskBTreeList;
+ private int inDiskTreeCounter;
+ private final RTreeFactory rTreeFactory;
+ private final BTreeFactory bTreeFactory;
+ private final IFileMapManager fileMapManager;
+ private int threadReferenceCounter;
+ private boolean flushFlag;
+
+ public LSMRTree(IBufferCache rtreeMemCache, IBufferCache bufferCache, int fieldCount, MultiComparator cmp,
+ IFreePageManager memFreePageManager, ITreeIndexFrameFactory rtreeInteriorFrameFactory,
+ ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
+ ITreeIndexFrameFactory btreeLeafFrameFactory, RTreeFactory rTreeFactory, BTreeFactory bTreeFactory,
+ IFileMapManager fileMapManager) {
+ this.bufferCache = bufferCache;
+ this.cmp = cmp;
+ this.rtreeInteriorFrameFactory = rtreeInteriorFrameFactory;
+ this.btreeInteriorFrameFactory = btreeInteriorFrameFactory;
+ this.rtreeLeafFrameFactory = rtreeLeafFrameFactory;
+ this.btreeLeafFrameFactory = btreeLeafFrameFactory;
+ this.memFreePageManager = memFreePageManager;
+ this.rTreeFactory = rTreeFactory;
+ this.bTreeFactory = bTreeFactory;
+ this.inDiskRTreeList = new LinkedList<ITreeIndex>();
+ this.inDiskBTreeList = new LinkedList<ITreeIndex>();
+ this.inDiskTreeCounter = 0;
+ this.fileMapManager = fileMapManager;
+ this.threadReferenceCounter = 0;
+ this.created = false;
+ this.flushFlag = false;
+
+ try {
+ this.rtreeFileName = this.fileMapManager.lookupFileName(this.rtreeFileId).toString();
+
+ this.btreeFileName = this.rtreeFileName + "-btree";
+ FileReference file = new FileReference(new File(this.btreeFileName));
+ this.bufferCache.createFile(file);
+ this.btreeFileId = fileMapManager.lookupFileId(file);
+ bufferCache.openFile(btreeFileId);
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ memRTree = new RTree(rtreeMemCache, fieldCount, cmp, memFreePageManager, rtreeInteriorFrameFactory,
+ rtreeLeafFrameFactory);
+ memBTree = new BTree(rtreeMemCache, fieldCount, cmp, memFreePageManager, btreeInteriorFrameFactory,
+ btreeLeafFrameFactory);
+ }
+
+ @Override
+ public ITreeIndexAccessor createAccessor() {
+ return new LSMRTreeAccessor(this);
+ }
+
+ @Override
+ public IIndexBulkLoadContext beginBulkLoad(float fillFactor) throws TreeIndexException, HyracksDataException,
+ PageAllocationException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException,
+ PageAllocationException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void endBulkLoad(IIndexBulkLoadContext ictx) throws HyracksDataException, PageAllocationException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public ITreeIndexFrameFactory getLeafFrameFactory() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public ITreeIndexFrameFactory getInteriorFrameFactory() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public IFreePageManager getFreePageManager() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public int getFieldCount() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public int getRootPageId() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public IndexType getIndexType() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public void create(int indexFileId) throws HyracksDataException {
+ if (created) {
+ return;
+ } else {
+ rtreeFileId = indexFileId;
+ memRTree.create(rtreeFileId);
+ memBTree.create(btreeFileId);
+ created = true;
+ }
+ }
+
+ @Override
+ public void open(int indexFileId) {
+ memRTree.open(rtreeFileId);
+ memBTree.open(btreeFileId);
+ }
+
+ @Override
+ public void close() {
+ memRTree.close();
+ memBTree.close();
+ this.rtreeFileId = -1;
+
+ }
+
+ @Override
+ public void merge() throws Exception {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void flush() throws Exception {
+ inDiskTreeCounter++;
+
+ // scan the RTree
+ ITreeIndexCursor rtreeScanCursor = new RTreeSearchCursor(
+ (IRTreeInteriorFrame) rtreeInteriorFrameFactory.createFrame(),
+ (IRTreeLeafFrame) rtreeLeafFrameFactory.createFrame());
+ SearchPredicate searchPredicate = new SearchPredicate(null, null);
+
+ ITreeIndexAccessor memRTreeAccessor = memRTree.createAccessor();
+ memRTreeAccessor.search(rtreeScanCursor, searchPredicate);
+
+ // Create a new in-Disk RTree
+
+ // Register the RTree information into system.
+ FileReference rtreeFile = new FileReference(new File(getNextFileName(rtreeFileName, inDiskTreeCounter)));
+ // TODO: Delete the file during cleanup.
+ bufferCache.createFile(rtreeFile);
+ int newDiskRTreeId = fileMapManager.lookupFileId(rtreeFile);
+ // TODO: Close the file during cleanup.
+ bufferCache.openFile(newDiskRTreeId);
+
+ // Create new in-Disk RTree.
+ RTree inDiskRTree = (RTree) rTreeFactory.createIndexInstance(newDiskRTreeId);
+ inDiskRTree.create(newDiskRTreeId);
+ // TODO: Close the RTree during cleanup.
+ inDiskRTree.open(newDiskRTreeId);
+
+ // // BulkLoad the tuples from the in-memory tree into the new disk
+ // RTree.
+ IIndexBulkLoadContext rtreeBulkLoadCtx = inDiskRTree.beginBulkLoad(1.0f);
+
+ int i = 0;
+ try {
+ while (rtreeScanCursor.hasNext()) {
+ rtreeScanCursor.next();
+ ITupleReference frameTuple = rtreeScanCursor.getTuple();
+ inDiskRTree.bulkLoadAddTuple(frameTuple, rtreeBulkLoadCtx);
+ i++;
+ }
+ } finally {
+ rtreeScanCursor.close();
+ }
+ inDiskRTree.endBulkLoad(rtreeBulkLoadCtx);
+
+ // scan the BTree
+ ITreeIndexCursor btreeScanCursor = new BTreeRangeSearchCursor(
+ (IBTreeLeafFrame) btreeLeafFrameFactory.createFrame(), false);
+ RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
+ ITreeIndexAccessor memBTreeAccessor = memBTree.createAccessor();
+ memBTreeAccessor.search(btreeScanCursor, nullPred);
+
+ // Create a new in-Disk BTree, which have full fillfactor.
+
+ // Register the BTree information into system.
+ FileReference btreeFile = new FileReference(new File(getNextFileName(btreeFileName, inDiskTreeCounter)));
+ // TODO: Delete the file during cleanup.
+ bufferCache.createFile(btreeFile);
+ int newDiskBTreeId = fileMapManager.lookupFileId(btreeFile);
+ // TODO: Close the file during cleanup.
+ bufferCache.openFile(newDiskBTreeId);
+
+ // Create new in-Disk BTree.
+ BTree inDiskBTree = (BTree) bTreeFactory.createIndexInstance(newDiskBTreeId);
+ inDiskBTree.create(newDiskBTreeId);
+ // TODO: Close the BTree during cleanup.
+ inDiskBTree.open(newDiskBTreeId);
+
+ // BulkLoad the tuples from the in-memory tree into the new disk BTree.
+ IIndexBulkLoadContext btreeBulkLoadCtx = inDiskBTree.beginBulkLoad(1.0f);
+ try {
+ while (btreeScanCursor.hasNext()) {
+ btreeScanCursor.next();
+ ITupleReference frameTuple = btreeScanCursor.getTuple();
+ inDiskBTree.bulkLoadAddTuple(frameTuple, btreeBulkLoadCtx);
+ }
+ } finally {
+ btreeScanCursor.close();
+ }
+ inDiskBTree.endBulkLoad(btreeBulkLoadCtx);
+
+ // After BulkLoading, Clear the in-memTrees
+ resetInMemoryTrees();
+
+ synchronized (inDiskRTreeList) {
+ inDiskRTreeList.addFirst(inDiskRTree);
+ }
+ synchronized (inDiskBTreeList) {
+ inDiskBTreeList.addFirst(inDiskBTree);
+ }
+ }
+
+ private static final String getNextFileName(String fileName, int inDiskTreeCounter) {
+ return fileName + "-" + Integer.toString(inDiskTreeCounter);
+ }
+
+ public void resetInMemoryTrees() throws HyracksDataException {
+ ((InMemoryFreePageManager) memFreePageManager).reset();
+ memRTree.create(rtreeFileId);
+ memBTree.create(btreeFileId);
+ }
+
+ private void decreaseThreadReferenceCounter() throws Exception {
+ synchronized (this) {
+ threadReferenceCounter--;
+ if (flushFlag == true) {
+ if (threadReferenceCounter == 0) {
+ flush();
+ flushFlag = false;
+ return;
+ } else if (threadReferenceCounter < 0) {
+ throw new Error("Thread reference counter is below zero. This indicates a programming error!");
+ }
+ }
+ }
+ }
+
+ private void insert(ITupleReference tuple, LSMTreeOpContext ctx) throws HyracksDataException, TreeIndexException,
+ PageAllocationException {
+ try {
+ boolean continuePerformOp = false;
+ try {
+ while (continuePerformOp == false) {
+ synchronized (this) {
+ if (!flushFlag) {
+ threadReferenceCounter++;
+ continuePerformOp = true;
+ }
+ }
+ }
+ ctx.LSMRTreeOpContext.memRtreeAccessor.insert(tuple);
+ decreaseThreadReferenceCounter();
+ } catch (PageAllocationException e) {
+ synchronized (this) {
+ // If flushFlag is false it means we are the first inserter
+ // to
+ // trigger the flush. If flushFlag is already set to true,
+ // there's no harm in setting it to true again.
+ flushFlag = true;
+ threadReferenceCounter--;
+ if (threadReferenceCounter == 0) {
+ flush();
+ ctx.LSMRTreeOpContext.reset(IndexOp.INSERT);
+ ctx.LSMRTreeOpContext.memRtreeAccessor.insert(tuple);
+ flushFlag = false;
+ return;
+ } else if (threadReferenceCounter < 0) {
+ throw new Error("Thread reference counter is below zero. This indicates a programming error!");
+ }
+ }
+ insert(tuple, ctx);
+ return;
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ private void delete(ITupleReference tuple, LSMTreeOpContext ctx) throws HyracksDataException, TreeIndexException,
+ PageAllocationException {
+ try {
+ boolean continuePerformOp = false;
+ try {
+ while (continuePerformOp == false) {
+ synchronized (this) {
+ if (!flushFlag) {
+ threadReferenceCounter++;
+ continuePerformOp = true;
+ }
+ }
+ }
+ ctx.LSMBTreeOpContext.memBtreeAccessor.insert(tuple);
+ decreaseThreadReferenceCounter();
+ } catch (PageAllocationException e) {
+ synchronized (this) {
+ // If flushFlag is false it means we are the first inserter
+ // to
+ // trigger the flush. If flushFlag is already set to true,
+ // there's no harm in setting it to true again.
+ flushFlag = true;
+ threadReferenceCounter--;
+ if (threadReferenceCounter == 0) {
+ flush();
+ ctx.LSMBTreeOpContext.reset(IndexOp.INSERT);
+ ctx.LSMBTreeOpContext.memBtreeAccessor.insert(tuple);
+ flushFlag = false;
+ return;
+ } else if (threadReferenceCounter < 0) {
+ throw new Error("Thread reference counter is below zero. This indicates a programming error!");
+ }
+ }
+ delete(tuple, ctx);
+ return;
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ private void search(ITreeIndexCursor cursor, ISearchPredicate searchPred, LSMTreeOpContext ctx) throws Exception {
+ // int numberOfInDiskTrees;
+ // ListIterator<InDiskTreeInfo> inDiskTreeInfoListIterator;
+ // boolean continuePerformOp = false;
+ //
+ // ctx.reset(IndexOp.SEARCH);
+ //
+ // while (continuePerformOp == false) {
+ // synchronized (this) {
+ // if (!flushFlag) {
+ // threadReferenceCounter++;
+ // continuePerformOp = true;
+ // }
+ // }
+ // }
+ //
+ // // in-disk
+ // synchronized (inDiskTreeInfoList) {
+ // numberOfInDiskTrees = inDiskTreeInfoList.size();
+ // inDiskTreeInfoListIterator = inDiskTreeInfoList.listIterator();
+ // }
+ //
+ // LSMTreeCursorInitialState initialState = new
+ // LSMTreeCursorInitialState(numberOfInDiskTrees + 1,
+ // insertLeafFrameFactory, cmp, this);
+ // cursor.open(initialState, pred);
+ //
+ // BTree[] onDiskBtrees = new BTree[numberOfInDiskTrees];
+ // ITreeIndexAccessor[] onDiskBtreeAccessors = new
+ // ITreeIndexAccessor[numberOfInDiskTrees];
+ //
+ // for (int i = 0; i < numberOfInDiskTrees; i++) {
+ // // get btree instances for in-disk trees
+ // if (inDiskTreeInfoListIterator.hasNext()) {
+ // onDiskBtrees[i] = ((InDiskTreeInfo)
+ // inDiskTreeInfoListIterator.next()).getBTree();
+ // } else {
+ // throw new HyracksDataException("Cannot find in-disk tree instance");
+ // }
+ // onDiskBtreeAccessors[i] = onDiskBtrees[i].createAccessor();
+ // onDiskBtreeAccessors[i].search(((LSMTreeRangeSearchCursor)
+ // cursor).getCursor(i + 1), pred);
+ // }
+ //
+ // // in-memory
+ // ctx.memBtreeAccessor.search(((LSMTreeRangeSearchCursor)
+ // cursor).getCursor(0), pred);
+ //
+ // LSMPriorityQueueComparator LSMPriorityQueueCmp = new
+ // LSMPriorityQueueComparator(cmp);
+ // ((LSMTreeRangeSearchCursor)
+ // cursor).initPriorityQueue(numberOfInDiskTrees + 1,
+ // LSMPriorityQueueCmp);
+ }
+
+ private LSMTreeOpContext createOpContext() {
+
+ return new LSMTreeOpContext(new LSMRTreeOpContext((RTree.RTreeAccessor) memRTree.createAccessor(),
+ (IRTreeLeafFrame) rtreeLeafFrameFactory.createFrame(),
+ (IRTreeInteriorFrame) rtreeInteriorFrameFactory.createFrame(), memFreePageManager
+ .getMetaDataFrameFactory().createFrame(), 8), new LSMBTreeOpContext(
+ (BTree.BTreeAccessor) memBTree.createAccessor(), btreeLeafFrameFactory, btreeInteriorFrameFactory,
+ memFreePageManager.getMetaDataFrameFactory().createFrame(), cmp));
+ }
+
+ private class LSMRTreeAccessor implements ITreeIndexAccessor {
+ private LSMRTree lsmRTree;
+ private LSMTreeOpContext ctx;
+
+ public LSMRTreeAccessor(LSMRTree lsmRTree) {
+ this.lsmRTree = lsmRTree;
+ this.ctx = lsmRTree.createOpContext();
+
+ }
+
+ @Override
+ public void insert(ITupleReference tuple) throws HyracksDataException, TreeIndexException,
+ PageAllocationException {
+ ctx.LSMRTreeOpContext.reset(IndexOp.INSERT);
+ lsmRTree.insert(tuple, ctx);
+ }
+
+ @Override
+ public void update(ITupleReference tuple) throws HyracksDataException, TreeIndexException,
+ PageAllocationException {
+ throw new UnsupportedOperationException("Update not supported by LSMRTree");
+ }
+
+ @Override
+ public void delete(ITupleReference tuple) throws HyracksDataException, TreeIndexException,
+ PageAllocationException {
+ ctx.LSMBTreeOpContext.reset(IndexOp.INSERT);
+ lsmRTree.delete(tuple, ctx);
+ }
+
+ @Override
+ public void search(ITreeIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException,
+ TreeIndexException, PageAllocationException {
+ ctx.reset(IndexOp.SEARCH);
+ // TODO: fix exception handling throughout LSM tree.
+ try {
+ lsmRTree.search(cursor, searchPred, ctx);
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void diskOrderScan(ITreeIndexCursor cursor) throws HyracksDataException {
+ throw new UnsupportedOperationException("DiskOrderScan not supported by LSMRTree");
+ }
+ }
+
+}
diff --git a/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMRTreeInMemoryBufferCache.java b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMRTreeInMemoryBufferCache.java
new file mode 100644
index 0000000..85c73ae
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMRTreeInMemoryBufferCache.java
@@ -0,0 +1,26 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.rtree.impls;
+
+import edu.uci.ics.hyracks.storage.am.lsmtree.common.impls.InMemoryBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+
+public class LSMRTreeInMemoryBufferCache extends InMemoryBufferCache {
+
+ public LSMRTreeInMemoryBufferCache(ICacheMemoryAllocator allocator, int pageSize, int numPages) {
+ super(allocator, pageSize, numPages);
+ }
+
+ @Override
+ public ICachedPage pin(long dpid, boolean newPage) {
+ int pageId = BufferedFileHandle.getPageId(dpid);
+ int fileId = BufferedFileHandle.getFileId(dpid);
+
+ if (pageId == 0 || pageId == 1) {
+ return cachedPages[pageId + 2 * fileId];
+ } else {
+ return cachedPages[pageId];
+ }
+ }
+
+}
diff --git a/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMRTreeInMemoryBufferCacheFactory.java b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMRTreeInMemoryBufferCacheFactory.java
new file mode 100644
index 0000000..ed652d6
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMRTreeInMemoryBufferCacheFactory.java
@@ -0,0 +1,26 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.rtree.impls;
+
+import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
+
+public class LSMRTreeInMemoryBufferCacheFactory {
+
+ private IBufferCache bufferCache;
+ private final int pageSize;
+ private final int numPages;
+
+ public LSMRTreeInMemoryBufferCacheFactory(int pageSize, int numPages) {
+ this.pageSize = pageSize;
+ this.numPages = numPages;
+ bufferCache = null;
+ }
+
+ public synchronized IBufferCache createInMemoryBufferCache() {
+ if (bufferCache == null) {
+ ICacheMemoryAllocator allocator = new HeapBufferAllocator();
+ bufferCache = new LSMRTreeInMemoryBufferCache(allocator, pageSize, numPages);
+ }
+ return bufferCache;
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMRTreeInMemoryFreePageManager.java b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMRTreeInMemoryFreePageManager.java
new file mode 100644
index 0000000..fe68f38
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMRTreeInMemoryFreePageManager.java
@@ -0,0 +1,23 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.rtree.impls;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.lsmtree.common.impls.InMemoryFreePageManager;
+
+public class LSMRTreeInMemoryFreePageManager extends InMemoryFreePageManager {
+
+ public LSMRTreeInMemoryFreePageManager(int maxCapacity, ITreeIndexMetaDataFrameFactory metaDataFrameFactory) {
+ super(maxCapacity, metaDataFrameFactory);
+ currentCapacity = 3;
+ }
+
+ @Override
+ public void init(ITreeIndexMetaDataFrame metaFrame, int currentMaxPage) throws HyracksDataException {
+ currentCapacity = 3;
+ }
+
+ public void reset() {
+ currentCapacity = 3;
+ }
+}
diff --git a/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMRTreeOpContext.java b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMRTreeOpContext.java
new file mode 100644
index 0000000..a3df90f
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMRTreeOpContext.java
@@ -0,0 +1,31 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.rtree.impls;
+
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
+import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeOpContext;
+
+public final class LSMRTreeOpContext extends RTreeOpContext {
+
+ public final RTree.RTreeAccessor memRtreeAccessor;
+
+ public LSMRTreeOpContext(RTree.RTreeAccessor memRtreeAccessor, IRTreeLeafFrame leafFrame,
+ IRTreeInteriorFrame interiorFrame, ITreeIndexMetaDataFrame metaFrame, int treeHeightHint) {
+
+ super(leafFrame, interiorFrame, metaFrame, treeHeightHint);
+
+ this.memRtreeAccessor = memRtreeAccessor;
+ // Overwrite the RTree accessor's op context with our LSMRTreeOpContext.
+ this.memRtreeAccessor.setOpContext(this);
+
+ reset(op);
+ }
+
+ @Override
+ public void reset(IndexOp newOp) {
+ super.reset(newOp);
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMTreeOpContext.java b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMTreeOpContext.java
new file mode 100644
index 0000000..dcf5247
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMTreeOpContext.java
@@ -0,0 +1,20 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.rtree.impls;
+
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+
+public final class LSMTreeOpContext {
+
+ public LSMRTreeOpContext LSMRTreeOpContext;
+ public LSMBTreeOpContext LSMBTreeOpContext;
+
+ public LSMTreeOpContext(LSMRTreeOpContext LSMRTreeOpContext, LSMBTreeOpContext LSMBTreeOpContext) {
+ this.LSMRTreeOpContext = LSMRTreeOpContext;
+ this.LSMBTreeOpContext = LSMBTreeOpContext;
+ }
+
+ public void reset(IndexOp newOp) {
+ LSMRTreeOpContext.reset(newOp);
+ LSMBTreeOpContext.reset(newOp);
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMTypeAwareTupleWriterFactory.java b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMTypeAwareTupleWriterFactory.java
new file mode 100644
index 0000000..a55248b
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/LSMTypeAwareTupleWriterFactory.java
@@ -0,0 +1,30 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.rtree.impls;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriter;
+
+public class LSMTypeAwareTupleWriterFactory extends TypeAwareTupleWriterFactory {
+
+ private static final long serialVersionUID = 1L;
+ private ITypeTraits[] typeTraits;
+ private final boolean isDelete;
+
+ public LSMTypeAwareTupleWriterFactory(ITypeTraits[] typeTraits, boolean isDelete) {
+ super(typeTraits);
+ this.typeTraits = typeTraits;
+ this.isDelete = isDelete;
+ }
+
+ @Override
+ public ITreeIndexTupleWriter createTupleWriter() {
+ if (isDelete) {
+ return new TypeAwareTupleWriter(typeTraits);
+ } else {
+ return new RTreeTypeAwareTupleWriter(typeTraits);
+ }
+ }
+
+}
diff --git a/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/RTreeFactory.java b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/RTreeFactory.java
new file mode 100644
index 0000000..d5558d7
--- /dev/null
+++ b/hyracks-storage-am-lsmtree-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/impls/RTreeFactory.java
@@ -0,0 +1,24 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.rtree.impls;
+
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsmtree.common.impls.FreePageManagerFactory;
+import edu.uci.ics.hyracks.storage.am.lsmtree.common.impls.TreeFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+public class RTreeFactory extends TreeFactory {
+
+ public RTreeFactory(IBufferCache bufferCache, FreePageManagerFactory freePageManagerFactory, MultiComparator cmp,
+ int fieldCount, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
+ super(bufferCache, freePageManagerFactory, cmp, fieldCount, interiorFrameFactory, leafFrameFactory);
+ }
+
+ @Override
+ public ITreeIndex createIndexInstance(int fileId) {
+ return new RTree(bufferCache, fieldCount, cmp, freePageManagerFactory.createFreePageManager(fileId),
+ interiorFrameFactory, leafFrameFactory);
+ }
+
+}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java
index 2b3065d..242adad 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java
@@ -25,6 +25,8 @@
public int getBestChildPageId();
+ public int getChildPageId(int tupleIndex);
+
public int getChildPageIdIfIntersect(ITupleReference tuple, int tupleIndex,
MultiComparator cmp);
@@ -40,4 +42,6 @@
MultiComparator cmp);
public void enlarge(ITupleReference tuple, MultiComparator cmp);
+
+ boolean checkEnlargement(ITupleReference tuple, MultiComparator cmp);
}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
index f3e6be2..202ae18 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
@@ -169,6 +169,12 @@
}
@Override
+ public int getChildPageId(int tupleIndex) {
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ return buf.getInt(getChildPointerOff(frameTuple));
+ }
+
+ @Override
public int getChildPageIdIfIntersect(ITupleReference tuple, int tupleIndex, MultiComparator cmp) {
frameTuple.setFieldCount(cmp.getKeyFieldCount());
frameTuple.resetByTupleIndex(this, tupleIndex);
@@ -585,6 +591,27 @@
}
@Override
+ public boolean checkEnlargement(ITupleReference tuple, MultiComparator cmp) {
+ int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ int c = cmp.getComparators()[i].compare(frameTuple.getFieldData(i), frameTuple.getFieldStart(i),
+ frameTuple.getFieldLength(i), tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i));
+ if (c > 0) {
+ return true;
+ }
+ c = cmp.getComparators()[j].compare(frameTuple.getFieldData(j), frameTuple.getFieldStart(j),
+ frameTuple.getFieldLength(j), tuple.getFieldData(j), tuple.getFieldStart(j),
+ tuple.getFieldLength(j));
+ if (c < 0) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
public void enlarge(ITupleReference tuple, MultiComparator cmp) {
int maxFieldPos = cmp.getKeyFieldCount() / 2;
for (int i = 0; i < maxFieldPos; i++) {
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
index 5f78dcc..7f06e16 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
@@ -51,7 +51,6 @@
public class RTree implements ITreeIndex {
- private boolean created = false;
private boolean loaded = false;
private final int rootPage = 1; // the root page never changes
@@ -206,10 +205,6 @@
public void create(int fileId) throws HyracksDataException {
treeLatch.writeLock().lock();
try {
- if (created) {
- return;
- }
-
ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
freePageManager.init(metaFrame, rootPage);
@@ -230,8 +225,6 @@
incrementUnpins();
}
currentLevel = 0;
-
- created = true;
} finally {
treeLatch.writeLock().unlock();
}
@@ -251,7 +244,8 @@
.createFrame(), 8);
}
- private void insert(ITupleReference tuple, IIndexOpContext ictx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void insert(ITupleReference tuple, IIndexOpContext ictx) throws HyracksDataException, TreeIndexException,
+ PageAllocationException {
RTreeOpContext ctx = (RTreeOpContext) ictx;
ctx.reset();
ctx.setTuple(tuple);
@@ -289,7 +283,7 @@
incrementUnpins();
}
- public ICachedPage findLeaf(RTreeOpContext ctx) throws HyracksDataException {
+ private ICachedPage findLeaf(RTreeOpContext ctx) throws HyracksDataException {
int pageId = rootPage;
boolean writeLatched = false;
boolean readLatched = false;
@@ -329,7 +323,7 @@
incrementReadLatchesAcquired();
}
}
-
+
if (pageId != rootPage && parentLsn < ctx.interiorFrame.getPageNsn()) {
// Concurrent split detected, go back to parent and
// re-choose
@@ -364,40 +358,58 @@
ctx.interiorFrame.findBestChild(ctx.getTuple(), cmp);
int childPageId = ctx.interiorFrame.getBestChildPageId();
- if (!writeLatched) {
- node.releaseReadLatch();
- readLatched = false;
- incrementReadLatchesReleased();
- // TODO: do we need to un-pin and pin again?
+ // check if enlargement is needed
+ boolean enlarementIsNeeded = ctx.interiorFrame.checkEnlargement(ctx.getTuple(), cmp);
+ if (enlarementIsNeeded) {
+ if (!writeLatched) {
+ node.releaseReadLatch();
+ readLatched = false;
+ incrementReadLatchesReleased();
+ // TODO: do we need to un-pin and pin again?
+ bufferCache.unpin(node);
+ incrementUnpins();
+
+ node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ incrementPins();
+ node.acquireWriteLatch();
+ writeLatched = true;
+ incrementWriteLatchesAcquired();
+ ctx.interiorFrame.setPage(node);
+
+ if (ctx.interiorFrame.getPageLsn() != pageLsn) {
+ // The page was changed while we unlocked it;
+ // thus,
+ // retry (re-choose best child)
+
+ ctx.pathList.moveLast();
+ continue;
+ }
+ }
+ // We don't need to reset the frameTuple because it is
+ // already pointing to the best child
+ ctx.interiorFrame.enlarge(ctx.getTuple(), cmp);
+
+ node.releaseWriteLatch();
+ writeLatched = false;
+ incrementWriteLatchesReleased();
bufferCache.unpin(node);
incrementUnpins();
-
- node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- incrementPins();
- node.acquireWriteLatch();
- writeLatched = true;
- incrementWriteLatchesAcquired();
- ctx.interiorFrame.setPage(node);
-
- if (ctx.interiorFrame.getPageLsn() != pageLsn) {
- // The page was changed while we unlocked it; thus,
- // retry (re-choose best child)
-
- ctx.pathList.moveLast();
- continue;
+ } else {
+ if (readLatched) {
+ node.releaseReadLatch();
+ readLatched = false;
+ incrementReadLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
+ } else if (writeLatched) {
+ node.releaseWriteLatch();
+ writeLatched = false;
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
}
}
- // We don't need to reset the frameTuple because it is
- // already pointing to the best child
- ctx.interiorFrame.enlarge(ctx.getTuple(), cmp);
-
- node.releaseWriteLatch();
- writeLatched = false;
- incrementWriteLatchesReleased();
- bufferCache.unpin(node);
- incrementUnpins();
-
pageId = childPageId;
parentLsn = pageLsn;
} else {
@@ -552,7 +564,8 @@
}
}
- public void updateParentForInsert(RTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void updateParentForInsert(RTreeOpContext ctx) throws HyracksDataException, TreeIndexException,
+ PageAllocationException {
boolean writeLatched = false;
int parentId = ctx.pathList.getLastPageId();
ICachedPage parentNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, parentId), false);
@@ -625,7 +638,7 @@
updateParentForInsert(ctx);
}
- public void findPath(RTreeOpContext ctx) throws HyracksDataException {
+ private void findPath(RTreeOpContext ctx) throws HyracksDataException {
boolean readLatched = false;
int pageId = rootPage;
int parentIndex = -1;
@@ -680,14 +693,14 @@
}
}
- public void fillPath(RTreeOpContext ctx, int pageIndex) {
+ private void fillPath(RTreeOpContext ctx, int pageIndex) {
if (pageIndex != -1) {
fillPath(ctx, ctx.traverseList.getPageIndex(pageIndex));
ctx.pathList.add(ctx.traverseList.getPageId(pageIndex), ctx.traverseList.getPageLsn(pageIndex), -1);
}
}
- public void delete(ITupleReference tuple, RTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
+ private void delete(ITupleReference tuple, RTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
ctx.reset();
ctx.setTuple(tuple);
ctx.splitKey.reset();
@@ -715,7 +728,7 @@
}
}
- public void updateParentForDelete(RTreeOpContext ctx) throws HyracksDataException {
+ private void updateParentForDelete(RTreeOpContext ctx) throws HyracksDataException {
boolean writeLatched = false;
int parentId = ctx.pathList.getLastPageId();
ICachedPage parentNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, parentId), false);
@@ -806,7 +819,7 @@
updateParentForDelete(ctx);
}
- public int findTupleToDelete(RTreeOpContext ctx) throws HyracksDataException {
+ private int findTupleToDelete(RTreeOpContext ctx) throws HyracksDataException {
boolean writeLatched = false;
boolean readLatched = false;
boolean succeed = false;
@@ -921,7 +934,7 @@
return -1;
}
- public void deleteTuple(int pageId, int tupleIndex, RTreeOpContext ctx) throws HyracksDataException {
+ private void deleteTuple(int pageId, int tupleIndex, RTreeOpContext ctx) throws HyracksDataException {
ctx.leafFrame.delete(tupleIndex, cmp);
incrementGlobalNsn();
ctx.leafFrame.setPageLsn(getGlobalNsn());
@@ -933,14 +946,15 @@
}
}
- private void search(ITreeIndexCursor cursor, ISearchPredicate searchPred, RTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
- ctx.reset();
+ private void search(ITreeIndexCursor cursor, ISearchPredicate searchPred, RTreeOpContext ctx)
+ throws HyracksDataException, TreeIndexException {
+ ctx.reset();
ctx.cursor = cursor;
cursor.setBufferCache(bufferCache);
cursor.setFileId(fileId);
ctx.cursorInitialState.setRootPage(rootPage);
- ctx.cursor.open(ctx.cursorInitialState, (SearchPredicate)searchPred);
+ ctx.cursor.open(ctx.cursorInitialState, (SearchPredicate) searchPred);
}
public ITreeIndexFrameFactory getInteriorFrameFactory() {
@@ -969,7 +983,7 @@
public BulkLoadContext(float fillFactor, IRTreeFrame leafFrame, IRTreeFrame interiorFrame,
ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
- indexAccessor = createAccessor();
+ indexAccessor = createAccessor();
}
}
@@ -988,7 +1002,7 @@
@Override
public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException {
try {
- ((BulkLoadContext) ictx).indexAccessor.insert(tuple);
+ ((BulkLoadContext) ictx).indexAccessor.insert(tuple);
} catch (Exception e) {
throw new HyracksDataException("BulkLoad Error");
}
@@ -1036,23 +1050,24 @@
public IndexType getIndexType() {
return IndexType.RTREE;
}
-
+
@Override
- public ITreeIndexAccessor createAccessor() {
- return new RTreeAccessor(this);
- }
-
- private class RTreeAccessor implements ITreeIndexAccessor {
+ public ITreeIndexAccessor createAccessor() {
+ return new RTreeAccessor(this);
+ }
+
+ public class RTreeAccessor implements ITreeIndexAccessor {
private RTree rtree;
private RTreeOpContext ctx;
-
+
public RTreeAccessor(RTree rtree) {
this.rtree = rtree;
this.ctx = rtree.createOpContext();
}
-
+
@Override
- public void insert(ITupleReference tuple) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ public void insert(ITupleReference tuple) throws HyracksDataException, TreeIndexException,
+ PageAllocationException {
ctx.reset(IndexOp.INSERT);
rtree.insert(tuple, ctx);
}
@@ -1081,5 +1096,13 @@
ctx.reset(IndexOp.DISKORDERSCAN);
rtree.diskOrderScan(cursor, ctx);
}
+
+ // TODO: Ideally, this method should not exist. But we need it for
+ // the LSM tree to work correctly, so we can use the LSMOpContext inside
+ // a BTreeAccessor.
+ // Making the appropriate change will involve changing lots of code.
+ public void setOpContext(RTreeOpContext ctx) {
+ this.ctx = ctx;
+ }
}
}
\ No newline at end of file
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java
index c258377..9c60492 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java
@@ -23,7 +23,7 @@
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
-public final class RTreeOpContext implements IIndexOpContext {
+public class RTreeOpContext implements IIndexOpContext {
public final IRTreeInteriorFrame interiorFrame;
public final IRTreeLeafFrame leafFrame;
public IndexOp op;
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
index e65f1d4..ee0cb20 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
@@ -49,9 +49,6 @@
private ITreeIndexTupleReference frameTuple;
private boolean readLatched = false;
- private int pin = 0;
- private int unpin = 0;
-
public RTreeSearchCursor(IRTreeInteriorFrame interiorFrame, IRTreeLeafFrame leafFrame) {
this.interiorFrame = interiorFrame;
this.leafFrame = leafFrame;
@@ -80,12 +77,11 @@
return page;
}
- public boolean fetchNextLeafPage() throws HyracksDataException {
+ private boolean fetchNextLeafPage() throws HyracksDataException {
boolean succeed = false;
if (readLatched) {
page.releaseReadLatch();
bufferCache.unpin(page);
- unpin++;
readLatched = false;
}
@@ -94,7 +90,6 @@
long parentLsn = pathList.getLastPageLsn();
pathList.moveLast();
ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- pin++;
node.acquireReadLatch();
readLatched = true;
try {
@@ -112,12 +107,20 @@
}
if (!isLeaf) {
- for (int i = 0; i < interiorFrame.getTupleCount(); i++) {
- int childPageId = interiorFrame.getChildPageIdIfIntersect(searchKey, i, cmp);
- if (childPageId != -1) {
+ if (searchKey != null) {
+ for (int i = 0; i < interiorFrame.getTupleCount(); i++) {
+ int childPageId = interiorFrame.getChildPageIdIfIntersect(searchKey, i, cmp);
+ if (childPageId != -1) {
+ pathList.add(childPageId, pageLsn, -1);
+ }
+ }
+ } else {
+ for (int i = 0; i < interiorFrame.getTupleCount(); i++) {
+ int childPageId = interiorFrame.getChildPageId(i);
pathList.add(childPageId, pageLsn, -1);
}
}
+
} else {
page = node;
leafFrame.setPage(page);
@@ -131,7 +134,6 @@
node.releaseReadLatch();
readLatched = false;
bufferCache.unpin(node);
- unpin++;
}
}
}
@@ -153,7 +155,13 @@
do {
for (int i = tupleIndex; i < leafFrame.getTupleCount(); i++) {
- if (leafFrame.intersect(searchKey, i, cmp)) {
+ if (searchKey != null) {
+ if (leafFrame.intersect(searchKey, i, cmp)) {
+ frameTuple.resetByTupleIndex(leafFrame, i);
+ tupleIndexInc = i + 1;
+ return true;
+ }
+ } else {
frameTuple.resetByTupleIndex(leafFrame, i);
tupleIndexInc = i + 1;
return true;
@@ -185,17 +193,19 @@
cmp = pred.getLowKeyComparator();
searchKey = pred.getSearchKey();
- int maxFieldPos = cmp.getKeyFieldCount() / 2;
- for (int i = 0; i < maxFieldPos; i++) {
- int j = maxFieldPos + i;
- int c = cmp.getComparators()[i].compare(searchKey.getFieldData(i), searchKey.getFieldStart(i),
- searchKey.getFieldLength(i), searchKey.getFieldData(j), searchKey.getFieldStart(j),
- searchKey.getFieldLength(j));
- if (c > 0) {
- throw new IllegalArgumentException("The low key point has larger coordinates than the high key point.");
+ if (searchKey != null) {
+ int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ int c = cmp.getComparators()[i].compare(searchKey.getFieldData(i), searchKey.getFieldStart(i),
+ searchKey.getFieldLength(i), searchKey.getFieldData(j), searchKey.getFieldStart(j),
+ searchKey.getFieldLength(j));
+ if (c > 0) {
+ throw new IllegalArgumentException("The low key point has larger coordinates than the high key point.");
+ }
}
}
-
+
pathList.add(this.rootPage, -1, -1);
tupleIndex = 0;
fetchNextLeafPage();
diff --git a/hyracks-tests/hyracks-storage-am-lsmtree-rtree-test/pom.xml b/hyracks-tests/hyracks-storage-am-lsmtree-rtree-test/pom.xml
new file mode 100644
index 0000000..737e975
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsmtree-rtree-test/pom.xml
@@ -0,0 +1,55 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsmtree-rtree-test</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-tests</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <type>jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-control-nc</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsmtree-rtree</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-test-support</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-tests/hyracks-storage-am-lsmtree-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/AbstractLSMTreeTest.java b/hyracks-tests/hyracks-storage-am-lsmtree-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/AbstractLSMTreeTest.java
new file mode 100644
index 0000000..58b08ec
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsmtree-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/AbstractLSMTreeTest.java
@@ -0,0 +1,76 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.rtree;
+
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Random;
+import java.util.logging.Logger;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
+import edu.uci.ics.hyracks.test.support.TestUtils;
+
+public abstract class AbstractLSMTreeTest {
+ protected static final Logger LOGGER = Logger.getLogger(AbstractLSMTreeTest.class.getName());
+ public static final long RANDOM_SEED = 50;
+
+ private static final int PAGE_SIZE = 256;
+ private static final int NUM_PAGES = 10;
+ private static final int MAX_OPEN_FILES = 10;
+ private static final int HYRACKS_FRAME_SIZE = 128;
+
+ protected IHyracksTaskContext ctx;
+ protected IBufferCache bufferCache;
+ protected int btreeFileId;
+
+ protected final Random rnd = new Random();
+ protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ protected final static String tmpDir = System.getProperty("java.io.tmpdir");
+ protected final static String sep = System.getProperty("file.separator");
+ protected String fileName;
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ fileName = tmpDir + sep + simpleDateFormat.format(new Date());
+ ctx = TestUtils.create(getHyracksFrameSize());
+ TestStorageManagerComponentHolder.init(getPageSize(), getNumPages(), getMaxOpenFiles());
+ bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ FileReference file = new FileReference(new File(fileName));
+ bufferCache.createFile(file);
+ btreeFileId = fmp.lookupFileId(file);
+ bufferCache.openFile(btreeFileId);
+ rnd.setSeed(RANDOM_SEED);
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ bufferCache.closeFile(btreeFileId);
+ bufferCache.close();
+ File f = new File(fileName);
+ f.deleteOnExit();
+ }
+
+ public int getPageSize() {
+ return PAGE_SIZE;
+ }
+
+ public int getNumPages() {
+ return NUM_PAGES;
+ }
+
+ public int getHyracksFrameSize() {
+ return HYRACKS_FRAME_SIZE;
+ }
+
+ public int getMaxOpenFiles() {
+ return MAX_OPEN_FILES;
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsmtree-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/LSMRTreeTest.java b/hyracks-tests/hyracks-storage-am-lsmtree-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/LSMRTreeTest.java
new file mode 100644
index 0000000..55eed29
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsmtree-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsmtree/rtree/LSMRTreeTest.java
@@ -0,0 +1,257 @@
+package edu.uci.ics.hyracks.storage.am.lsmtree.rtree;
+
+import java.io.DataOutput;
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.Random;
+import java.util.logging.Level;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsmtree.common.impls.FreePageManagerFactory;
+import edu.uci.ics.hyracks.storage.am.lsmtree.rtree.impls.BTreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsmtree.rtree.impls.LSMRTree;
+import edu.uci.ics.hyracks.storage.am.lsmtree.rtree.impls.LSMRTreeInMemoryBufferCacheFactory;
+import edu.uci.ics.hyracks.storage.am.lsmtree.rtree.impls.LSMRTreeInMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsmtree.rtree.impls.LSMTypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.lsmtree.rtree.impls.RTreeFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapManager;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
+import edu.uci.ics.hyracks.test.support.TestUtils;
+
+public class LSMRTreeTest extends AbstractLSMTreeTest {
+
+ private static final int PAGE_SIZE = 256;
+ private static final int NUM_PAGES = 100;
+ private static final int MAX_OPEN_FILES = 100;
+ private static final int HYRACKS_FRAME_SIZE = 128;
+ private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
+
+ // INSERT-DELETE TEST
+ @Test
+ public void Test1() throws Exception {
+ // in disk
+ TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
+ IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ FileReference file = new FileReference(new File(fileName));
+ bufferCache.createFile(file);
+ int fileId = fmp.lookupFileId(file);
+ bufferCache.openFile(fileId);
+
+ // in memory
+ LSMRTreeInMemoryBufferCacheFactory inMemBufferCacheFactory = new LSMRTreeInMemoryBufferCacheFactory(PAGE_SIZE,
+ NUM_PAGES);
+ IBufferCache memBufferCache = inMemBufferCacheFactory.createInMemoryBufferCache();
+
+ // declare keys
+ int keyFieldCount = 4;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
+ cmps[1] = cmps[0];
+ cmps[2] = cmps[0];
+ cmps[3] = cmps[0];
+
+ // declare tuple fields
+ int fieldCount = 7;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = DoublePointable.TYPE_TRAITS;
+ typeTraits[1] = DoublePointable.TYPE_TRAITS;
+ typeTraits[2] = DoublePointable.TYPE_TRAITS;
+ typeTraits[3] = DoublePointable.TYPE_TRAITS;
+ typeTraits[4] = DoublePointable.TYPE_TRAITS;
+ typeTraits[5] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[6] = DoublePointable.TYPE_TRAITS;
+
+ MultiComparator cmp = new MultiComparator(cmps);
+
+ // create value providers
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ cmps.length, DoublePointable.FACTORY);
+
+ LSMTypeAwareTupleWriterFactory rtreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
+ LSMTypeAwareTupleWriterFactory btreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
+
+ ITreeIndexFrameFactory rtreeInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(rtreeTupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexFrameFactory rtreeLeafFrameFactory = new RTreeNSMLeafFrameFactory(rtreeTupleWriterFactory,
+ valueProviderFactories);
+
+ ITreeIndexFrameFactory btreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(btreeTupleWriterFactory);
+ ITreeIndexFrameFactory btreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(btreeTupleWriterFactory);
+
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+
+ IFreePageManager memFreePageManager = new LSMRTreeInMemoryFreePageManager(100, metaFrameFactory);
+
+ FreePageManagerFactory freePageManagerFactory = new FreePageManagerFactory(bufferCache, metaFrameFactory);
+
+ RTreeFactory rTreeFactory = new RTreeFactory(bufferCache, freePageManagerFactory, cmp, fieldCount,
+ rtreeInteriorFrameFactory, rtreeLeafFrameFactory);
+ BTreeFactory bTreeFactory = new BTreeFactory(bufferCache, freePageManagerFactory, cmp, fieldCount,
+ btreeInteriorFrameFactory, btreeLeafFrameFactory);
+
+ LSMRTree lsmrtree = new LSMRTree(memBufferCache, bufferCache, fieldCount, cmp, memFreePageManager,
+ rtreeInteriorFrameFactory, btreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeLeafFrameFactory,
+ rTreeFactory, bTreeFactory, (IFileMapManager) fmp);
+
+ lsmrtree.create(fileId);
+ lsmrtree.open(fileId);
+
+ ByteBuffer frame = ctx.allocateFrame();
+ FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ DataOutput dos = tb.getDataOutput();
+
+ @SuppressWarnings("rawtypes")
+ ISerializerDeserializer[] recDescSers = { DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
+ RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
+
+ IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
+ accessor.reset(frame);
+
+ FrameTupleReference tuple = new FrameTupleReference();
+
+ ITreeIndexAccessor lsmTreeAccessor = lsmrtree.createAccessor();
+
+ Random rnd = new Random();
+ rnd.setSeed(50);
+
+ Random rnd2 = new Random();
+ rnd2.setSeed(50);
+ for (int i = 0; i < 5000; i++) {
+
+ double p1x = rnd.nextDouble();
+ double p1y = rnd.nextDouble();
+ double p2x = rnd.nextDouble();
+ double p2y = rnd.nextDouble();
+
+ double pk1 = rnd2.nextDouble();
+ int pk2 = rnd2.nextInt();
+ double pk3 = rnd2.nextDouble();
+
+ tb.reset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
+ tb.addFieldEndOffset();
+
+ appender.reset(frame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
+
+ tuple.reset(accessor, 0);
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ // if (i % 1000 == 0) {
+ LOGGER.info("INSERTING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
+ // }
+ }
+
+ try {
+ lsmTreeAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ rnd.setSeed(50);
+ for (int i = 0; i < 5000; i++) {
+
+ double p1x = rnd.nextDouble();
+ double p1y = rnd.nextDouble();
+ double p2x = rnd.nextDouble();
+ double p2y = rnd.nextDouble();
+
+ double pk1 = rnd.nextDouble();
+ int pk2 = rnd.nextInt();
+ double pk3 = rnd.nextDouble();
+
+ tb.reset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
+ tb.addFieldEndOffset();
+
+ appender.reset(frame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
+
+ tuple.reset(accessor, 0);
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ // if (i % 1000 == 0) {
+ LOGGER.info("DELETING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
+ // }
+ }
+
+ try {
+ lsmTreeAccessor.delete(tuple);
+ } catch (TreeIndexException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ lsmrtree.close();
+ bufferCache.closeFile(fileId);
+ memBufferCache.close();
+ }
+
+}
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTest.java b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTest.java
index 9fe2c94..e24a086 100644
--- a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTest.java
+++ b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTest.java
@@ -267,7 +267,7 @@
typeTraits[2] = DoublePointable.TYPE_TRAITS;
typeTraits[3] = DoublePointable.TYPE_TRAITS;
typeTraits[4] = DoublePointable.TYPE_TRAITS;
- typeTraits[5] = DoublePointable.TYPE_TRAITS;
+ typeTraits[5] = IntegerPointable.TYPE_TRAITS;
typeTraits[6] = DoublePointable.TYPE_TRAITS;
// create value providers
diff --git a/hyracks-tests/pom.xml b/hyracks-tests/pom.xml
index b120ba82..0e46b3f 100644
--- a/hyracks-tests/pom.xml
+++ b/hyracks-tests/pom.xml
@@ -16,5 +16,7 @@
<module>hyracks-storage-am-btree-test</module>
<module>hyracks-storage-am-invertedindex-test</module>
<module>hyracks-storage-am-rtree-test</module>
+ <module>hyracks-storage-am-lsmtree-btree-test</module>
+ <module>hyracks-storage-am-lsmtree-rtree-test</module>
</modules>
</project>
diff --git a/pom.xml b/pom.xml
index 0ab0646..3b713a5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -94,6 +94,9 @@
<module>hyracks-storage-am-common</module>
<module>hyracks-storage-am-btree</module>
<module>hyracks-storage-am-invertedindex</module>
+ <module>hyracks-storage-am-lsmtree-common</module>
+ <module>hyracks-storage-am-lsmtree-btree</module>
+ <module>hyracks-storage-am-lsmtree-rtree</module>
<module>hyracks-storage-am-rtree</module>
<module>hyracks-test-support</module>
<module>hyracks-tests</module>