Implemented k-buffering for lsm indexes. Also add a fix for issues 589 and 594.
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
index f217419..35c9bcb 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
@@ -15,6 +15,8 @@
package edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow;
+import java.util.List;
+
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
@@ -31,25 +33,25 @@
public class LSMBTreeDataflowHelper extends AbstractLSMIndexDataflowHelper {
public LSMBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- IVirtualBufferCache virtualBufferCache, ILSMMergePolicy mergePolicy,
+ List<IVirtualBufferCache> virtualBufferCaches, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
- this(opDesc, ctx, partition, virtualBufferCache, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
+ this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
opTrackerFactory, ioScheduler, ioOpCallbackProvider);
}
public LSMBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- IVirtualBufferCache virtualBufferCache, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
+ List<IVirtualBufferCache> virtualBufferCaches, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
- super(opDesc, ctx, partition, virtualBufferCache, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
+ super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackProvider);
}
@Override
public ITreeIndex createIndexInstance() throws HyracksDataException {
AbstractTreeIndexOperatorDescriptor treeOpDesc = (AbstractTreeIndexOperatorDescriptor) opDesc;
- return LSMBTreeUtils.createLSMTree(virtualBufferCache, file, opDesc.getStorageManager().getBufferCache(ctx),
+ return LSMBTreeUtils.createLSMTree(virtualBufferCaches, file, opDesc.getStorageManager().getBufferCache(ctx),
opDesc.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
treeOpDesc.getTreeIndexComparatorFactories(), treeOpDesc.getTreeIndexBloomFilterKeyFields(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx), ioScheduler,
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
index d331880..f4be88f 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
@@ -41,7 +41,7 @@
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition) {
return new LSMBTreeDataflowHelper(opDesc, ctx, partition,
- virtualBufferCacheProvider.getVirtualBufferCache(ctx), bloomFilterFalsePositiveRate,
+ virtualBufferCacheProvider.getVirtualBufferCaches(ctx), bloomFilterFalsePositiveRate,
mergePolicyProvider.getMergePolicy(ctx), opTrackerFactory, ioSchedulerProvider.getIOScheduler(ctx),
ioOpCallbackProvider);
}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 342faac..328629a 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -16,6 +16,7 @@
package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
import java.io.File;
+import java.util.ArrayList;
import java.util.List;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -62,9 +63,11 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.IMutableComponentSwitcherCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractMutableLSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
@@ -75,7 +78,7 @@
public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
// In-memory components.
- private final LSMBTreeMutableComponent mutableComponent;
+ private final List<LSMBTreeMutableComponent> mutableComponents;
// For creating BTree's used in flush and merge.
private final LSMBTreeImmutableComponentFactory componentFactory;
@@ -88,19 +91,43 @@
private final ITreeIndexFrameFactory deleteLeafFrameFactory;
private final IBinaryComparatorFactory[] cmpFactories;
- public LSMBTree(IVirtualBufferCache virtualBufferCache, ITreeIndexFrameFactory interiorFrameFactory,
+ public LSMBTree(List<IVirtualBufferCache> virtualBufferCaches, ITreeIndexFrameFactory interiorFrameFactory,
ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory,
ILSMIndexFileManager fileManager, TreeIndexFactory<BTree> diskBTreeFactory,
TreeIndexFactory<BTree> bulkLoadBTreeFactory, BloomFilterFactory bloomFilterFactory,
double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount,
IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
- super(virtualBufferCache, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
+ super(virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallbackProvider);
- mutableComponent = new LSMBTreeMutableComponent(new BTree(virtualBufferCache,
- virtualBufferCache.getFileMapProvider(), new VirtualFreePageManager(virtualBufferCache.getNumPages()),
- interiorFrameFactory, insertLeafFrameFactory, cmpFactories, fieldCount, new FileReference(new File(
- fileManager.getBaseDir() + "_virtual"))), virtualBufferCache);
+ mutableComponents = new ArrayList<LSMBTreeMutableComponent>();
+ final int numMutableComponents = virtualBufferCaches.size();
+ int i = 0;
+ for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
+ LSMBTreeMutableComponent mutableComponent = new LSMBTreeMutableComponent(new BTree(virtualBufferCache,
+ virtualBufferCache.getFileMapProvider(), new VirtualFreePageManager(
+ virtualBufferCache.getNumPages()), interiorFrameFactory, insertLeafFrameFactory,
+ cmpFactories, fieldCount, new FileReference(new File(fileManager.getBaseDir() + "_virtual_" + i))),
+ virtualBufferCache, i == 0 ? true : false);
+
+ mutableComponent.registerOnFlushCallback(new IMutableComponentSwitcherCallback() {
+
+ @Override
+ public void switchComponents() throws HyracksDataException {
+ currentMutableComponentId.set((currentMutableComponentId.get() + 1) % numMutableComponents);
+ mutableComponents.get(currentMutableComponentId.get()).setActive();
+ }
+
+ @Override
+ public void setFlushStatus(boolean isFlushNeeded) {
+ needsFlush[currentMutableComponentId.get()].set(isFlushNeeded);
+ }
+ });
+
+ mutableComponents.add(mutableComponent);
+ ++i;
+ }
+
this.insertLeafFrameFactory = insertLeafFrameFactory;
this.deleteLeafFrameFactory = deleteLeafFrameFactory;
this.cmpFactories = cmpFactories;
@@ -124,10 +151,11 @@
if (isActivated) {
throw new HyracksDataException("Failed to activate the index since it is already activated.");
}
-
- ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).open();
- mutableComponent.getBTree().create();
- mutableComponent.getBTree().activate();
+ for (LSMBTreeMutableComponent mutableComponent : mutableComponents) {
+ ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).open();
+ mutableComponent.getBTree().create();
+ mutableComponent.getBTree().activate();
+ }
List<ILSMComponent> immutableComponents = componentsRef.get();
immutableComponents.clear();
List<LSMComponentFileReferences> validFileReferences;
@@ -176,9 +204,11 @@
btree.deactivate();
bloomFilter.deactivate();
}
- mutableComponent.getBTree().deactivate();
- mutableComponent.getBTree().destroy();
- ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).close();
+ for (LSMBTreeMutableComponent mutableComponent : mutableComponents) {
+ mutableComponent.getBTree().deactivate();
+ mutableComponent.getBTree().destroy();
+ ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).close();
+ }
isActivated = false;
}
@@ -199,7 +229,9 @@
component.getBTree().destroy();
component.getBloomFilter().destroy();
}
- mutableComponent.getBTree().destroy();
+ for (LSMBTreeMutableComponent mutableComponent : mutableComponents) {
+ mutableComponent.getBTree().destroy();
+ }
fileManager.deleteDirs();
}
@@ -210,8 +242,10 @@
}
List<ILSMComponent> immutableComponents = componentsRef.get();
- mutableComponent.getBTree().clear();
- mutableComponent.reset();
+ for (LSMBTreeMutableComponent mutableComponent : mutableComponents) {
+ mutableComponent.getBTree().clear();
+ mutableComponent.reset();
+ }
for (ILSMComponent c : immutableComponents) {
LSMBTreeImmutableComponent component = (LSMBTreeImmutableComponent) c;
component.getBloomFilter().deactivate();
@@ -223,26 +257,33 @@
}
@Override
- public ILSMComponent getMutableComponent() {
- return mutableComponent;
- }
-
- @Override
public void getOperationalComponents(ILSMIndexOperationContext ctx) {
List<ILSMComponent> immutableComponents = componentsRef.get();
List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
operationalComponents.clear();
+ int cmc = currentMutableComponentId.get();
+ ctx.setCurrentMutableComponentId(cmc);
+ int numMutableComponents = mutableComponents.size();
switch (ctx.getOperation()) {
case UPDATE:
case UPSERT:
case PHYSICALDELETE:
case FLUSH:
case DELETE:
- operationalComponents.add(mutableComponent);
+ operationalComponents.add(mutableComponents.get(cmc));
break;
case SEARCH:
case INSERT:
- operationalComponents.add(mutableComponent);
+ for (int i = 0; i < numMutableComponents - 1; i++) {
+ LSMBTreeMutableComponent mutableComponent = mutableComponents.get((cmc + i + 1)
+ % numMutableComponents);
+ if (mutableComponent.isReadable()) {
+ // Make sure newest components are added first
+ operationalComponents.add(0, mutableComponent);
+ }
+ }
+ // The current mutable component is always added
+ operationalComponents.add(0, mutableComponents.get(cmc));
operationalComponents.addAll(immutableComponents);
break;
case MERGE:
@@ -258,27 +299,27 @@
LSMBTreeOpContext ctx = (LSMBTreeOpContext) ictx;
switch (ctx.getOperation()) {
case PHYSICALDELETE:
- ctx.memBTreeAccessor.delete(tuple);
+ ctx.currentMutableBTreeAccessor.delete(tuple);
break;
case INSERT:
insert(tuple, ctx);
break;
default:
- ctx.memBTreeAccessor.upsert(tuple);
+ ctx.currentMutableBTreeAccessor.upsert(tuple);
break;
}
- mutableComponent.setIsModified();
+ mutableComponents.get(currentMutableComponentId.get()).setIsModified();
}
private boolean insert(ITupleReference tuple, LSMBTreeOpContext ctx) throws HyracksDataException, IndexException {
- MultiComparator comparator = MultiComparator.createIgnoreFieldLength(mutableComponent.getBTree()
- .getComparatorFactories());
+ MultiComparator comparator = MultiComparator.createIgnoreFieldLength(mutableComponents
+ .get(currentMutableComponentId.get()).getBTree().getComparatorFactories());
LSMBTreePointSearchCursor searchCursor = new LSMBTreePointSearchCursor(ctx);
- IIndexCursor memCursor = new BTreeRangeSearchCursor(ctx.memBTreeOpCtx.leafFrame, false);
+ IIndexCursor memCursor = new BTreeRangeSearchCursor(ctx.currentMutableBTreeOpCtx.leafFrame, false);
RangePredicate predicate = new RangePredicate(tuple, tuple, true, true, comparator, comparator);
// first check the inmemory component
- ctx.memBTreeAccessor.search(memCursor, predicate);
+ ctx.currentMutableBTreeAccessor.search(memCursor, predicate);
try {
if (memCursor.hasNext()) {
memCursor.next();
@@ -287,7 +328,8 @@
throw new TreeIndexDuplicateKeyException("Failed to insert key since key already exists.");
} else {
memCursor.close();
- ctx.memBTreeAccessor.upsertIfConditionElseInsert(tuple, AntimatterAwareTupleAcceptor.INSTANCE);
+ ctx.currentMutableBTreeAccessor.upsertIfConditionElseInsert(tuple,
+ AntimatterAwareTupleAcceptor.INSTANCE);
return true;
}
}
@@ -301,6 +343,9 @@
// the mutable component?
// the key was not in the inmemory component, so check the disk
// components
+
+ // This is a hack to avoid searching the current active mutable component twice. It is critical to add it back once the search is over.
+ ILSMComponent c = ctx.getComponentHolder().remove(0);
search(ctx, searchCursor, predicate);
try {
if (searchCursor.hasNext()) {
@@ -308,9 +353,11 @@
}
} finally {
searchCursor.close();
+ // Add the current active mutable component back
+ ctx.getComponentHolder().add(0, c);
}
- ctx.memBTreeAccessor.upsertIfConditionElseInsert(tuple, AntimatterAwareTupleAcceptor.INSTANCE);
+ ctx.currentMutableBTreeAccessor.upsertIfConditionElseInsert(tuple, AntimatterAwareTupleAcceptor.INSTANCE);
return true;
}
@@ -319,26 +366,22 @@
throws HyracksDataException, IndexException {
LSMBTreeOpContext ctx = (LSMBTreeOpContext) ictx;
List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
- int numBTrees = operationalComponents.size();
- assert numBTrees > 0;
- boolean includeMutableComponent = operationalComponents.get(0) == mutableComponent;
- LSMBTreeCursorInitialState initialState = new LSMBTreeCursorInitialState(numBTrees, insertLeafFrameFactory,
- ctx.cmp, ctx.bloomFilterCmp, includeMutableComponent, lsmHarness, ctx.memBTreeAccessor, pred,
- ctx.searchCallback, operationalComponents);
+ LSMBTreeCursorInitialState initialState = new LSMBTreeCursorInitialState(insertLeafFrameFactory, ctx.cmp,
+ ctx.bloomFilterCmp, lsmHarness, pred, ctx.searchCallback, operationalComponents);
cursor.open(initialState, pred);
}
@Override
public boolean scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException {
- if (!mutableComponent.isModified()) {
+ ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
+ if (!((AbstractMutableLSMComponent) flushingComponent).isModified()) {
return false;
}
LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
LSMBTreeOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
assert ctx.getComponentHolder().size() == 1;
- ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
opCtx.setOperation(IndexOperation.FLUSH);
opCtx.getComponentHolder().add(flushingComponent);
ILSMIndexAccessorInternal flushAccessor = new LSMBTreeAccessor(lsmHarness, opCtx);
@@ -576,7 +619,7 @@
public LSMBTreeOpContext createOpContext(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
- return new LSMBTreeOpContext(mutableComponent.getBTree(), insertLeafFrameFactory, deleteLeafFrameFactory,
+ return new LSMBTreeOpContext(mutableComponents, insertLeafFrameFactory, deleteLeafFrameFactory,
modificationCallback, searchCallback, componentFactory.getBloomFilterKeyFields().length);
}
@@ -613,47 +656,60 @@
@Override
public ITreeIndexFrameFactory getInteriorFrameFactory() {
- return mutableComponent.getBTree().getInteriorFrameFactory();
+ return mutableComponents.get(currentMutableComponentId.get()).getBTree().getInteriorFrameFactory();
}
@Override
public int getFieldCount() {
- return mutableComponent.getBTree().getFieldCount();
+ return mutableComponents.get(currentMutableComponentId.get()).getBTree().getFieldCount();
}
@Override
public int getFileId() {
- return mutableComponent.getBTree().getFileId();
+ return mutableComponents.get(currentMutableComponentId.get()).getBTree().getFileId();
}
@Override
public IFreePageManager getFreePageManager() {
- return mutableComponent.getBTree().getFreePageManager();
+ return mutableComponents.get(currentMutableComponentId.get()).getBTree().getFreePageManager();
}
@Override
public ITreeIndexFrameFactory getLeafFrameFactory() {
- return mutableComponent.getBTree().getLeafFrameFactory();
+ return mutableComponents.get(currentMutableComponentId.get()).getBTree().getLeafFrameFactory();
}
@Override
public long getMemoryAllocationSize() {
- IBufferCache virtualBufferCache = mutableComponent.getBTree().getBufferCache();
- return virtualBufferCache.getNumPages() * virtualBufferCache.getPageSize();
+ long size = 0;
+ for (LSMBTreeMutableComponent mutableComponent : mutableComponents) {
+ IBufferCache virtualBufferCache = mutableComponent.getBTree().getBufferCache();
+ size += virtualBufferCache.getNumPages() * virtualBufferCache.getPageSize();
+ }
+ return size;
}
@Override
public int getRootPageId() {
- return mutableComponent.getBTree().getRootPageId();
+ return mutableComponents.get(currentMutableComponentId.get()).getBTree().getRootPageId();
}
public boolean isEmptyIndex() throws HyracksDataException {
- return componentsRef.get().isEmpty() && !mutableComponent.isModified();
+ boolean isModified = false;
+ for (LSMBTreeMutableComponent mutableComponent : mutableComponents) {
+ if (mutableComponent.isModified()) {
+ isModified = true;
+ break;
+ }
+ }
+ return componentsRef.get().isEmpty() && !isModified;
}
@Override
public void validate() throws HyracksDataException {
- mutableComponent.getBTree().validate();
+ for (LSMBTreeMutableComponent mutableComponent : mutableComponents) {
+ mutableComponent.getBTree().validate();
+ }
List<ILSMComponent> immutableComponents = componentsRef.get();
for (ILSMComponent c : immutableComponents) {
BTree btree = (BTree) ((LSMBTreeImmutableComponent) c).getBTree();
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeCursorInitialState.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeCursorInitialState.java
index b5c6a45..37f89ff 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeCursorInitialState.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeCursorInitialState.java
@@ -18,7 +18,6 @@
import java.util.List;
import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
@@ -29,39 +28,28 @@
public class LSMBTreeCursorInitialState implements ICursorInitialState {
- private final int numBTrees;
private final ITreeIndexFrameFactory leafFrameFactory;
private MultiComparator cmp;
private final MultiComparator bloomFilterCmp;
- private final boolean includeMemComponent;
private final ILSMHarness lsmHarness;
- private final IIndexAccessor memBtreeAccessor;
private final ISearchPredicate predicate;
private ISearchOperationCallback searchCallback;
private final List<ILSMComponent> operationalComponents;
- public LSMBTreeCursorInitialState(int numBTrees, ITreeIndexFrameFactory leafFrameFactory, MultiComparator cmp,
- MultiComparator bloomFilterCmp, boolean includeMemComponent, ILSMHarness lsmHarness,
- IIndexAccessor memBtreeAccessor, ISearchPredicate predicate, ISearchOperationCallback searchCallback,
- List<ILSMComponent> operationalComponents) {
- this.numBTrees = numBTrees;
+ public LSMBTreeCursorInitialState(ITreeIndexFrameFactory leafFrameFactory, MultiComparator cmp,
+ MultiComparator bloomFilterCmp, ILSMHarness lsmHarness, ISearchPredicate predicate,
+ ISearchOperationCallback searchCallback, List<ILSMComponent> operationalComponents) {
this.leafFrameFactory = leafFrameFactory;
this.cmp = cmp;
this.bloomFilterCmp = bloomFilterCmp;
- this.includeMemComponent = includeMemComponent;
this.lsmHarness = lsmHarness;
this.searchCallback = searchCallback;
- this.memBtreeAccessor = memBtreeAccessor;
this.predicate = predicate;
this.operationalComponents = operationalComponents;
}
- public int getNumBTrees() {
- return numBTrees;
- }
-
public ITreeIndexFrameFactory getLeafFrameFactory() {
return leafFrameFactory;
}
@@ -75,10 +63,6 @@
public void setPage(ICachedPage page) {
}
- public boolean getIncludeMemComponent() {
- return includeMemComponent;
- }
-
public ILSMHarness getLSMHarness() {
return lsmHarness;
}
@@ -97,10 +81,6 @@
return operationalComponents;
}
- public IIndexAccessor getMemBTreeAccessor() {
- return memBtreeAccessor;
- }
-
public ISearchPredicate getSearchPredicate() {
return predicate;
}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeMutableComponent.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeMutableComponent.java
index 6c01b29..c0605e5 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeMutableComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeMutableComponent.java
@@ -24,8 +24,8 @@
private final BTree btree;
- public LSMBTreeMutableComponent(BTree btree, IVirtualBufferCache vbc) {
- super(vbc);
+ public LSMBTreeMutableComponent(BTree btree, IVirtualBufferCache vbc, boolean isActive) {
+ super(vbc, isActive);
this.btree = btree;
}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
index ac62d9f..036c306 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
@@ -37,9 +37,11 @@
public ITreeIndexFrameFactory deleteLeafFrameFactory;
public IBTreeLeafFrame insertLeafFrame;
public IBTreeLeafFrame deleteLeafFrame;
- public final BTree memBTree;
- public BTree.BTreeAccessor memBTreeAccessor;
- public BTreeOpContext memBTreeOpCtx;
+ public final BTree[] mutableBTrees;
+ public BTree.BTreeAccessor[] mutableBTreeAccessors;
+ public BTreeOpContext[] mutableBTreeOpCtxs;
+ public BTree.BTreeAccessor currentMutableBTreeAccessor;
+ public BTreeOpContext currentMutableBTreeOpCtx;
public IndexOperation op;
public final MultiComparator cmp;
public final MultiComparator bloomFilterCmp;
@@ -47,19 +49,30 @@
public final ISearchOperationCallback searchCallback;
private final List<ILSMComponent> componentHolder;
- public LSMBTreeOpContext(BTree memBTree, ITreeIndexFrameFactory insertLeafFrameFactory,
- ITreeIndexFrameFactory deleteLeafFrameFactory, IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback, int numBloomFilterKeyFields) {
- IBinaryComparatorFactory cmpFactories[] = memBTree.getComparatorFactories();
+ public LSMBTreeOpContext(List<LSMBTreeMutableComponent> mutableComponents,
+ ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory,
+ IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback,
+ int numBloomFilterKeyFields) {
+ IBinaryComparatorFactory cmpFactories[] = mutableComponents.get(0).getBTree().getComparatorFactories();
if (cmpFactories[0] != null) {
- this.cmp = MultiComparator.create(memBTree.getComparatorFactories());
+ this.cmp = MultiComparator.create(mutableComponents.get(0).getBTree().getComparatorFactories());
} else {
this.cmp = null;
}
- bloomFilterCmp = MultiComparator.create(memBTree.getComparatorFactories(), 0, numBloomFilterKeyFields);
+ bloomFilterCmp = MultiComparator.create(mutableComponents.get(0).getBTree().getComparatorFactories(), 0,
+ numBloomFilterKeyFields);
- this.memBTree = memBTree;
+ mutableBTrees = new BTree[mutableComponents.size()];
+ mutableBTreeAccessors = new BTree.BTreeAccessor[mutableComponents.size()];
+ mutableBTreeOpCtxs = new BTreeOpContext[mutableComponents.size()];
+ for (int i = 0; i < mutableComponents.size(); i++) {
+ mutableBTrees[i] = mutableComponents.get(i).getBTree();
+ mutableBTreeAccessors[i] = (BTree.BTreeAccessor) mutableBTrees[i].createAccessor(modificationCallback,
+ NoOpOperationCallback.INSTANCE);
+ mutableBTreeOpCtxs[i] = mutableBTreeAccessors[i].getOpContext();
+ }
+
this.insertLeafFrameFactory = insertLeafFrameFactory;
this.deleteLeafFrameFactory = deleteLeafFrameFactory;
this.insertLeafFrame = (IBTreeLeafFrame) insertLeafFrameFactory.createFrame();
@@ -79,48 +92,16 @@
public void setOperation(IndexOperation newOp) {
reset();
this.op = newOp;
- switch (newOp) {
- case SEARCH:
- setMemBTreeAccessor();
- break;
- case DISKORDERSCAN:
- case UPDATE:
- // Attention: It is important to leave the leafFrame and
- // leafFrameFactory of the memBTree as is when doing an update.
- // Update will only be set if a previous attempt to delete or
- // insert failed, so we must preserve the semantics of the
- // previously requested operation.
- setMemBTreeAccessor();
- return;
- case UPSERT:
- case INSERT:
- setInsertMode();
- break;
- case PHYSICALDELETE:
- case DELETE:
- setDeleteMode();
- break;
- }
- }
-
- private void setMemBTreeAccessor() {
- if (memBTreeAccessor == null) {
- memBTreeAccessor = (BTree.BTreeAccessor) memBTree.createAccessor(modificationCallback,
- NoOpOperationCallback.INSTANCE);
- memBTreeOpCtx = memBTreeAccessor.getOpContext();
- }
}
public void setInsertMode() {
- setMemBTreeAccessor();
- memBTreeOpCtx.leafFrame = insertLeafFrame;
- memBTreeOpCtx.leafFrameFactory = insertLeafFrameFactory;
+ currentMutableBTreeOpCtx.leafFrame = insertLeafFrame;
+ currentMutableBTreeOpCtx.leafFrameFactory = insertLeafFrameFactory;
}
public void setDeleteMode() {
- setMemBTreeAccessor();
- memBTreeOpCtx.leafFrame = deleteLeafFrame;
- memBTreeOpCtx.leafFrameFactory = deleteLeafFrameFactory;
+ currentMutableBTreeOpCtx.leafFrame = deleteLeafFrame;
+ currentMutableBTreeOpCtx.leafFrameFactory = deleteLeafFrameFactory;
}
@Override
@@ -146,4 +127,30 @@
public IModificationOperationCallback getModificationCallback() {
return modificationCallback;
}
+
+ @Override
+ public void setCurrentMutableComponentId(int currentMutableComponentId) {
+ currentMutableBTreeAccessor = mutableBTreeAccessors[currentMutableComponentId];
+ currentMutableBTreeOpCtx = mutableBTreeOpCtxs[currentMutableComponentId];
+ switch (op) {
+ case SEARCH:
+ break;
+ case DISKORDERSCAN:
+ case UPDATE:
+ // Attention: It is important to leave the leafFrame and
+ // leafFrameFactory of the mutableBTree as is when doing an update.
+ // Update will only be set if a previous attempt to delete or
+ // insert failed, so we must preserve the semantics of the
+ // previously requested operation.
+ break;
+ case UPSERT:
+ case INSERT:
+ setInsertMode();
+ break;
+ case PHYSICALDELETE:
+ case DELETE:
+ setDeleteMode();
+ break;
+ }
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
index 64e0c28..005629f 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
@@ -15,7 +15,7 @@
package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
-import java.util.ListIterator;
+import java.util.List;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
@@ -32,6 +32,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMTreeTupleReference;
@@ -45,14 +46,14 @@
private final ILSMIndexOperationContext opCtx;
private ISearchOperationCallback searchCallback;
private RangePredicate predicate;
- private IIndexAccessor memBTreeAccessor;
- private boolean includeMemComponent;
+ private boolean includeMutableComponent;
private int numBTrees;
- private IIndexAccessor[] bTreeAccessors;
+ private IIndexAccessor[] btreeAccessors;
private ILSMHarness lsmHarness;
private boolean nextHasBeenCalled;
private boolean foundTuple;
private ITupleReference frameTuple;
+ private List<ILSMComponent> operationalComponents;
public LSMBTreePointSearchCursor(ILSMIndexOperationContext opCtx) {
this.opCtx = opCtx;
@@ -67,7 +68,7 @@
}
boolean reconciled = false;
for (int i = 0; i < numBTrees; ++i) {
- bTreeAccessors[i].search(rangeCursors[i], predicate);
+ btreeAccessors[i].search(rangeCursors[i], predicate);
if (rangeCursors[i].hasNext()) {
rangeCursors[i].next();
// We use the predicate's to lock the key instead of the tuple that we get from cursor to avoid copying the tuple when we do the "unlatch dance"
@@ -83,14 +84,14 @@
return true;
}
}
- if (i == 0 && includeMemComponent) {
+ if (i == 0 && includeMutableComponent) {
// unlatch/unpin
rangeCursors[i].reset();
searchCallback.reconcile(predicate.getLowKey());
reconciled = true;
// retraverse
- memBTreeAccessor.search(rangeCursors[i], predicate);
+ btreeAccessors[0].search(rangeCursors[i], predicate);
searchCallback.complete(predicate.getLowKey());
if (rangeCursors[i].hasNext()) {
rangeCursors[i].next();
@@ -123,14 +124,14 @@
@Override
public void reset() throws HyracksDataException, IndexException {
try {
- if (rangeCursors != null) {
- for (int i = 0; i < rangeCursors.length; ++i) {
- rangeCursors[i].reset();
+ if (rangeCursors != null) {
+ for (int i = 0; i < rangeCursors.length; ++i) {
+ rangeCursors[i].reset();
+ }
}
- }
- rangeCursors = null;
- nextHasBeenCalled = false;
- foundTuple = false;
+ rangeCursors = null;
+ nextHasBeenCalled = false;
+ foundTuple = false;
} finally {
if (lsmHarness != null) {
lsmHarness.endSearch(opCtx);
@@ -141,41 +142,30 @@
@Override
public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
LSMBTreeCursorInitialState lsmInitialState = (LSMBTreeCursorInitialState) initialState;
- includeMemComponent = lsmInitialState.getIncludeMemComponent();
+ operationalComponents = lsmInitialState.getOperationalComponents();
lsmHarness = lsmInitialState.getLSMHarness();
searchCallback = lsmInitialState.getSearchOperationCallback();
- memBTreeAccessor = lsmInitialState.getMemBTreeAccessor();
predicate = (RangePredicate) lsmInitialState.getSearchPredicate();
-
- numBTrees = lsmInitialState.getNumBTrees();
+ numBTrees = operationalComponents.size();
rangeCursors = new IIndexCursor[numBTrees];
- int i = 0;
- if (includeMemComponent) {
- // No need for a bloom filter for the in-memory BTree.
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) lsmInitialState.getLeafFrameFactory().createFrame();
- rangeCursors[i] = new BTreeRangeSearchCursor(leafFrame, false);
- ++i;
- }
- for (; i < numBTrees; ++i) {
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) lsmInitialState.getLeafFrameFactory().createFrame();
- rangeCursors[i] = new BloomFilterAwareBTreePointSearchCursor(leafFrame, false,
- ((LSMBTreeImmutableComponent) lsmInitialState.getOperationalComponents().get(i)).getBloomFilter());
- }
+ btreeAccessors = new IIndexAccessor[numBTrees];
+ includeMutableComponent = false;
- bTreeAccessors = new IIndexAccessor[numBTrees];
- int cursorIx = 0;
- ListIterator<ILSMComponent> btreesIter = lsmInitialState.getOperationalComponents().listIterator();
- if (includeMemComponent) {
- bTreeAccessors[cursorIx] = memBTreeAccessor;
- ++cursorIx;
- btreesIter.next();
- }
-
- while (btreesIter.hasNext()) {
- BTree diskBTree = ((LSMBTreeImmutableComponent) btreesIter.next()).getBTree();
- bTreeAccessors[cursorIx] = diskBTree.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
- cursorIx++;
+ for (int i = 0; i < numBTrees; i++) {
+ ILSMComponent component = operationalComponents.get(i);
+ BTree btree;
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) lsmInitialState.getLeafFrameFactory().createFrame();
+ if (component.getType() == LSMComponentType.MEMORY) {
+ includeMutableComponent = true;
+ // No need for a bloom filter for the in-memory BTree.
+ rangeCursors[i] = new BTreeRangeSearchCursor(leafFrame, false);
+ btree = (BTree) ((LSMBTreeMutableComponent) component).getBTree();
+ } else {
+ rangeCursors[i] = new BloomFilterAwareBTreePointSearchCursor(leafFrame, false,
+ ((LSMBTreeImmutableComponent) component).getBloomFilter());
+ btree = (BTree) ((LSMBTreeImmutableComponent) component).getBTree();
+ }
+ btreeAccessors[i] = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
}
nextHasBeenCalled = false;
foundTuple = false;
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java
index 4bdebae..d832583 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java
@@ -16,7 +16,6 @@
package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
import java.util.Iterator;
-import java.util.ListIterator;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
@@ -35,6 +34,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor;
@@ -44,7 +44,7 @@
private ISearchOperationCallback searchCallback;
private RangePredicate predicate;
- private IIndexAccessor memBTreeAccessor;
+ private IIndexAccessor[] btreeAccessors;
private ArrayTupleBuilder tupleBuilder;
private boolean proceed = true;
@@ -72,25 +72,25 @@
if (!outputPriorityQueue.isEmpty()) {
PriorityQueueElement checkElement = outputPriorityQueue.peek();
if (proceed && !searchCallback.proceed(checkElement.getTuple())) {
- if (includeMemComponent) {
- PriorityQueueElement inMemElement = null;
- boolean inMemElementFound = false;
- // scan the PQ for the in-memory component's element
+ if (includeMutableComponent) {
+ PriorityQueueElement mutableElement = null;
+ boolean mutableElementFound = false;
+ // scan the PQ for the mutable component's element
Iterator<PriorityQueueElement> it = outputPriorityQueue.iterator();
while (it.hasNext()) {
- inMemElement = it.next();
- if (inMemElement.getCursorIndex() == 0) {
- inMemElementFound = true;
+ mutableElement = it.next();
+ if (mutableElement.getCursorIndex() == 0) {
+ mutableElementFound = true;
it.remove();
break;
}
}
- if (inMemElementFound) {
+ if (mutableElementFound) {
// copy the in-mem tuple
if (tupleBuilder == null) {
tupleBuilder = new ArrayTupleBuilder(cmp.getKeyFieldCount());
}
- TupleUtils.copyTuple(tupleBuilder, inMemElement.getTuple(), cmp.getKeyFieldCount());
+ TupleUtils.copyTuple(tupleBuilder, mutableElement.getTuple(), cmp.getKeyFieldCount());
copyTuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
// unlatch/unpin
@@ -105,11 +105,11 @@
}
// retraverse
reusablePred.setLowKey(copyTuple, true);
- memBTreeAccessor.search(rangeCursors[0], reusablePred);
- boolean isNotExhaustedCursor = pushIntoPriorityQueue(inMemElement);
-
+ btreeAccessors[0].search(rangeCursors[0], reusablePred);
+ boolean isNotExhaustedCursor = pushIntoPriorityQueue(mutableElement);
+
if (checkElement.getCursorIndex() == 0) {
- if (!isNotExhaustedCursor || cmp.compare(copyTuple, inMemElement.getTuple()) != 0) {
+ if (!isNotExhaustedCursor || cmp.compare(copyTuple, mutableElement.getTuple()) != 0) {
searchCallback.complete(copyTuple);
searchCallback.cancel(copyTuple);
continue;
@@ -117,7 +117,7 @@
searchCallback.complete(copyTuple);
}
} else {
- // the in-memory cursor is exhausted
+ // the mutable cursor is exhausted
searchCallback.reconcile(checkElement.getTuple());
}
} else {
@@ -174,46 +174,34 @@
IndexException {
LSMBTreeCursorInitialState lsmInitialState = (LSMBTreeCursorInitialState) initialState;
cmp = lsmInitialState.getOriginalKeyComparator();
- includeMemComponent = lsmInitialState.getIncludeMemComponent();
operationalComponents = lsmInitialState.getOperationalComponents();
lsmHarness = lsmInitialState.getLSMHarness();
searchCallback = lsmInitialState.getSearchOperationCallback();
- memBTreeAccessor = lsmInitialState.getMemBTreeAccessor();
predicate = (RangePredicate) lsmInitialState.getSearchPredicate();
reusablePred.setLowKeyComparator(cmp);
reusablePred.setHighKey(predicate.getHighKey(), predicate.isHighKeyInclusive());
reusablePred.setHighKeyComparator(predicate.getHighKeyComparator());
+ includeMutableComponent = false;
- int numBTrees = lsmInitialState.getNumBTrees();
+ int numBTrees = operationalComponents.size();
rangeCursors = new IIndexCursor[numBTrees];
+
+ btreeAccessors = new ITreeIndexAccessor[numBTrees];
for (int i = 0; i < numBTrees; i++) {
+ ILSMComponent component = operationalComponents.get(i);
+ BTree btree;
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) lsmInitialState.getLeafFrameFactory().createFrame();
rangeCursors[i] = new BTreeRangeSearchCursor(leafFrame, false);
+ if (component.getType() == LSMComponentType.MEMORY) {
+ includeMutableComponent = true;
+ btree = (BTree) ((LSMBTreeMutableComponent) component).getBTree();
+ } else {
+ btree = (BTree) ((LSMBTreeImmutableComponent) component).getBTree();
+ }
+ btreeAccessors[i] = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ btreeAccessors[i].search(rangeCursors[i], searchPred);
}
setPriorityQueueComparator();
-
- int cursorIx = 0;
- ListIterator<ILSMComponent> btreesIter = operationalComponents.listIterator();
- if (includeMemComponent) {
- // Open cursor of in-memory BTree at index 0.
- memBTreeAccessor.search(rangeCursors[cursorIx], searchPred);
- // Skip 0 because it is the in-memory BTree.
- ++cursorIx;
- btreesIter.next();
- }
-
- // Open cursors of on-disk BTrees.
- int numDiskComponents = includeMemComponent ? numBTrees - 1 : numBTrees;
- ITreeIndexAccessor[] diskBTreeAccessors = new ITreeIndexAccessor[numDiskComponents];
- int diskBTreeIx = 0;
- while (btreesIter.hasNext()) {
- BTree diskBTree = (BTree) ((LSMBTreeImmutableComponent) btreesIter.next()).getBTree();
- diskBTreeAccessors[diskBTreeIx] = diskBTree.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
- diskBTreeAccessors[diskBTreeIx].search(rangeCursors[cursorIx], searchPred);
- cursorIx++;
- diskBTreeIx++;
- }
initPriorityQueue();
proceed = true;
}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
index 507ec02..f43c8a6 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
@@ -15,6 +15,8 @@
package edu.uci.ics.hyracks.storage.am.lsm.btree.util;
+import java.util.List;
+
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.io.FileReference;
@@ -43,7 +45,7 @@
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class LSMBTreeUtils {
- public static LSMBTree createLSMTree(IVirtualBufferCache virtualBufferCache, FileReference file,
+ public static LSMBTree createLSMTree(List<IVirtualBufferCache> virtualBufferCaches, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
@@ -73,7 +75,7 @@
ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(diskFileMapProvider, file, diskBTreeFactory);
- LSMBTree lsmTree = new LSMBTree(virtualBufferCache, interiorFrameFactory, insertLeafFrameFactory,
+ LSMBTree lsmTree = new LSMBTree(virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory,
deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory,
bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length, cmpFactories, mergePolicy,
opTracker, ioScheduler, ioOpCallbackProvider);
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponent.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponent.java
index 72a3e1f..1d1de53 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponent.java
@@ -18,7 +18,15 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMOperationType;
public interface ILSMComponent {
- public boolean threadEnter(LSMOperationType opType) throws InterruptedException;
+
+ enum LSMComponentType {
+ MEMORY,
+ DISK
+ }
+
+ public boolean threadEnter(LSMOperationType opType) throws InterruptedException, HyracksDataException;
public void threadExit(LSMOperationType opType, boolean failedOperation) throws HyracksDataException;
+
+ public LSMComponentType getType();
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndex.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndex.java
index 105b53a..a92cb42 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndex.java
@@ -32,11 +32,11 @@
* concurrent searches/updates/merges may be ongoing.
*/
public interface ILSMIndex extends IIndex {
-
+
public void deactivate(boolean flushOnExit) throws HyracksDataException;
public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback);
+ ISearchOperationCallback searchCallback) throws HyracksDataException;
public boolean getFlushStatus();
@@ -45,4 +45,6 @@
public ILSMIOOperationScheduler getIOScheduler();
public List<ILSMComponent> getImmutableComponents();
+
+ public int getCurrentMutableComponentId();
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
index 8fb7bdd..41e1425 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
@@ -28,7 +28,7 @@
public interface ILSMIndexInternal extends ILSMIndex {
public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback);
+ ISearchOperationCallback searchCallback) throws HyracksDataException;
public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException;
@@ -58,14 +58,6 @@
*/
public void getOperationalComponents(ILSMIndexOperationContext ctx);
- public List<ILSMComponent> getImmutableComponents();
-
- public ILSMComponent getMutableComponent();
-
public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException;
- public void setFlushStatus(boolean needsFlush);
-
- public boolean isFull();
-
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java
index bbd2ca8..fcd4037 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java
@@ -26,4 +26,6 @@
public ISearchOperationCallback getSearchOperationCallback();
public IModificationOperationCallback getModificationCallback();
+
+ public void setCurrentMutableComponentId(int currentMutableComponentId);
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMOperationTracker.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMOperationTracker.java
index 2659b27..ccde3ba 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMOperationTracker.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMOperationTracker.java
@@ -36,7 +36,7 @@
* then this method does not block and returns false.
* Otherwise, this method returns true, and the operation is considered 'active' in the index.
*/
- public void beforeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
+ public boolean beforeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException;
/**
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/IMutableResetCallback.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/IMutableComponentAdderCallback.java
similarity index 80%
rename from hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/IMutableResetCallback.java
rename to hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/IMutableComponentAdderCallback.java
index ff4a5d0..ba37740 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/IMutableResetCallback.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/IMutableComponentAdderCallback.java
@@ -12,12 +12,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
+package edu.uci.ics.hyracks.storage.am.lsm.common.api;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-public interface IMutableResetCallback {
+public interface IMutableComponentAdderCallback {
- public void reset() throws HyracksDataException;
-
-}
+ public void addComponent() throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/IMutableResetCallback.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/IMutableComponentSwitcherCallback.java
similarity index 75%
copy from hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/IMutableResetCallback.java
copy to hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/IMutableComponentSwitcherCallback.java
index ff4a5d0..8eae3a4 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/IMutableResetCallback.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/IMutableComponentSwitcherCallback.java
@@ -12,12 +12,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
+package edu.uci.ics.hyracks.storage.am.lsm.common.api;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-public interface IMutableResetCallback {
+public interface IMutableComponentSwitcherCallback {
- public void reset() throws HyracksDataException;
+ public void switchComponents() throws HyracksDataException;
-}
+ public void setFlushStatus(boolean isFlushNeeded);
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java
index cd75df0..eff390f 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java
@@ -15,9 +15,10 @@
package edu.uci.ics.hyracks.storage.am.lsm.common.api;
import java.io.Serializable;
+import java.util.List;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
public interface IVirtualBufferCacheProvider extends Serializable {
- public IVirtualBufferCache getVirtualBufferCache(IHyracksTaskContext ctx);
+ public List<IVirtualBufferCache> getVirtualBufferCaches(IHyracksTaskContext ctx);
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java
index e79a394..b1bf6b4 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java
@@ -15,6 +15,8 @@
package edu.uci.ics.hyracks.storage.am.lsm.common.dataflow;
+import java.util.List;
+
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
@@ -30,26 +32,26 @@
protected final double bloomFilterFalsePositiveRate;
- protected final IVirtualBufferCache virtualBufferCache;
+ protected final List<IVirtualBufferCache> virtualBufferCaches;
protected final ILSMMergePolicy mergePolicy;
protected final ILSMIOOperationScheduler ioScheduler;
protected final ILSMOperationTrackerProvider opTrackerFactory;
protected final ILSMIOOperationCallbackProvider ioOpCallbackProvider;
public AbstractLSMIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- IVirtualBufferCache virtualBufferCache, ILSMMergePolicy mergePolicy,
+ List<IVirtualBufferCache> virtualBufferCaches, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
- this(opDesc, ctx, partition, virtualBufferCache, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
+ this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
opTrackerFactory, ioScheduler, ioOpCallbackProvider);
}
public AbstractLSMIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- IVirtualBufferCache virtualBufferCache, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
+ List<IVirtualBufferCache> virtualBufferCaches, double bloomFilterFalsePositiveRate,
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
super(opDesc, ctx, partition);
- this.virtualBufferCache = virtualBufferCache;
+ this.virtualBufferCaches = virtualBufferCaches;
this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
this.mergePolicy = mergePolicy;
this.opTrackerFactory = opTrackerFactory;
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractImmutableLSMComponent.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractImmutableLSMComponent.java
index 376c2e4..59132e6 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractImmutableLSMComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractImmutableLSMComponent.java
@@ -80,6 +80,11 @@
}
}
+ @Override
+ public LSMComponentType getType() {
+ return LSMComponentType.DISK;
+ }
+
protected abstract void destroy() throws HyracksDataException;
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index d9da9e5..86f299f 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -18,6 +18,8 @@
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
@@ -44,7 +46,8 @@
protected final ILSMIOOperationCallbackProvider ioOpCallbackProvider;
// In-memory components.
- protected final IVirtualBufferCache virtualBufferCache;
+ protected final List<IVirtualBufferCache> virtualBufferCaches;
+ protected AtomicInteger currentMutableComponentId;
// On-disk components.
protected final IBufferCache diskBufferCache;
@@ -55,13 +58,13 @@
protected boolean isActivated;
- private boolean needsFlush = false;
+ protected final AtomicBoolean[] needsFlush;
- public AbstractLSMIndex(IVirtualBufferCache virtualBufferCache, IBufferCache diskBufferCache,
+ public AbstractLSMIndex(List<IVirtualBufferCache> virtualBufferCaches, IBufferCache diskBufferCache,
ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
- this.virtualBufferCache = virtualBufferCache;
+ this.virtualBufferCaches = virtualBufferCaches;
this.diskBufferCache = diskBufferCache;
this.diskFileMapProvider = diskFileMapProvider;
this.fileManager = fileManager;
@@ -72,6 +75,11 @@
isActivated = false;
componentsRef = new AtomicReference<List<ILSMComponent>>();
componentsRef.set(new LinkedList<ILSMComponent>());
+ currentMutableComponentId = new AtomicInteger();
+ needsFlush = new AtomicBoolean[virtualBufferCaches.size()];
+ for (int i = 0; i < virtualBufferCaches.size(); i++) {
+ needsFlush[i] = new AtomicBoolean();
+ }
}
protected void forceFlushDirtyPages(ITreeIndex treeIndex) throws HyracksDataException {
@@ -160,13 +168,8 @@
}
@Override
- public void setFlushStatus(boolean needsFlush) {
- this.needsFlush = needsFlush;
- }
-
- @Override
public boolean getFlushStatus() {
- return needsFlush;
+ return needsFlush[currentMutableComponentId.get()].get();
}
@Override
@@ -180,13 +183,13 @@
}
@Override
- public boolean isFull() {
- return virtualBufferCache.isFull();
+ public IBufferCache getBufferCache() {
+ return diskBufferCache;
}
@Override
- public IBufferCache getBufferCache() {
- return diskBufferCache;
+ public int getCurrentMutableComponentId() {
+ return currentMutableComponentId.get();
}
@Override
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMutableLSMComponent.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMutableLSMComponent.java
index 43501f9..05b0a4f 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMutableLSMComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMutableLSMComponent.java
@@ -18,6 +18,8 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.IMutableComponentAdderCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.IMutableComponentSwitcherCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
public abstract class AbstractMutableLSMComponent implements ILSMComponent {
@@ -25,36 +27,52 @@
private int readerCount;
private int writerCount;
private ComponentState state;
- private IMutableResetCallback resetCallback;
private final IVirtualBufferCache vbc;
+ private IMutableComponentAdderCallback adderCallback;
+ private IMutableComponentSwitcherCallback switcherCallback;
+
private final AtomicBoolean isModified;
+ private boolean requestedToBeActive;
private enum ComponentState {
READABLE_WRITABLE,
READABLE_UNWRITABLE,
READABLE_UNWRITABLE_FLUSHING,
- UNREADABLE_UNWRITABLE
+ UNREADABLE_UNWRITABLE,
+ INACTIVE_READABLE_WRITABLE
}
- public AbstractMutableLSMComponent(IVirtualBufferCache vbc) {
+ public AbstractMutableLSMComponent(IVirtualBufferCache vbc, boolean isActive) {
this.vbc = vbc;
readerCount = 0;
writerCount = 0;
- state = ComponentState.READABLE_WRITABLE;
+ if (isActive) {
+ state = ComponentState.READABLE_WRITABLE;
+ } else {
+ state = ComponentState.INACTIVE_READABLE_WRITABLE;
+ }
isModified = new AtomicBoolean();
}
@Override
- public synchronized boolean threadEnter(LSMOperationType opType) throws InterruptedException {
+ public synchronized boolean threadEnter(LSMOperationType opType) throws InterruptedException, HyracksDataException {
switch (opType) {
case FORCE_MODIFICATION:
+ if (state == ComponentState.INACTIVE_READABLE_WRITABLE && requestedToBeActive) {
+ state = ComponentState.READABLE_WRITABLE;
+ requestedToBeActive = false;
+ }
if (state != ComponentState.READABLE_WRITABLE && state != ComponentState.READABLE_UNWRITABLE) {
return false;
}
writerCount++;
break;
case MODIFICATION:
+ if (state == ComponentState.INACTIVE_READABLE_WRITABLE && requestedToBeActive) {
+ state = ComponentState.READABLE_WRITABLE;
+ requestedToBeActive = false;
+ }
if (state != ComponentState.READABLE_WRITABLE) {
return false;
}
@@ -73,9 +91,11 @@
}
state = ComponentState.READABLE_UNWRITABLE_FLUSHING;
+ switcherCallback.setFlushStatus(false);
while (writerCount > 0) {
wait();
}
+ switcherCallback.switchComponents();
readerCount++;
break;
default:
@@ -92,27 +112,36 @@
writerCount--;
if (state == ComponentState.READABLE_WRITABLE && isFull()) {
state = ComponentState.READABLE_UNWRITABLE;
+ switcherCallback.setFlushStatus(true);
}
break;
case SEARCH:
readerCount--;
if (state == ComponentState.UNREADABLE_UNWRITABLE && readerCount == 0) {
reset();
- resetCallback.reset();
- state = ComponentState.READABLE_WRITABLE;
+ adderCallback.addComponent();
+ if (requestedToBeActive == true) {
+ state = ComponentState.READABLE_WRITABLE;
+ requestedToBeActive = false;
+ } else {
+ state = ComponentState.INACTIVE_READABLE_WRITABLE;
+ }
} else if (state == ComponentState.READABLE_WRITABLE && isFull()) {
state = ComponentState.READABLE_UNWRITABLE;
+ switcherCallback.setFlushStatus(true);
}
break;
case FLUSH:
- if (failedOperation) {
- state = isFull() ? ComponentState.READABLE_UNWRITABLE : ComponentState.READABLE_WRITABLE;
- }
readerCount--;
if (readerCount == 0) {
reset();
- resetCallback.reset();
- state = ComponentState.READABLE_WRITABLE;
+ adderCallback.addComponent();
+ if (requestedToBeActive == true) {
+ state = ComponentState.READABLE_WRITABLE;
+ requestedToBeActive = false;
+ } else {
+ state = ComponentState.INACTIVE_READABLE_WRITABLE;
+ }
} else if (state == ComponentState.READABLE_UNWRITABLE_FLUSHING) {
state = ComponentState.UNREADABLE_UNWRITABLE;
}
@@ -123,8 +152,28 @@
notifyAll();
}
- public void registerOnResetCallback(IMutableResetCallback resetCallback) {
- this.resetCallback = resetCallback;
+ @Override
+ public LSMComponentType getType() {
+ return LSMComponentType.MEMORY;
+ }
+
+ public synchronized boolean isReadable() {
+ if (state == ComponentState.UNREADABLE_UNWRITABLE || state == ComponentState.INACTIVE_READABLE_WRITABLE) {
+ return false;
+ }
+ return true;
+ }
+
+ public synchronized void setActive() {
+ requestedToBeActive = true;
+ }
+
+ public void registerOnResetCallback(IMutableComponentAdderCallback adderCallback) {
+ this.adderCallback = adderCallback;
+ }
+
+ public void registerOnFlushCallback(IMutableComponentSwitcherCallback switcherCallback) {
+ this.switcherCallback = switcherCallback;
}
public void setIsModified() {
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index d30866e..364bc47 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -34,6 +34,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.IMutableComponentAdderCallback;
public class LSMHarness implements ILSMHarness {
private static final Logger LOGGER = Logger.getLogger(LSMHarness.class.getName());
@@ -49,9 +50,6 @@
}
private void threadExit(ILSMIndexOperationContext opCtx, LSMOperationType opType) throws HyracksDataException {
- if (!lsmIndex.getFlushStatus() && lsmIndex.isFull()) {
- lsmIndex.setFlushStatus(true);
- }
opTracker.afterOperation(lsmIndex, opType, opCtx.getSearchOperationCallback(), opCtx.getModificationCallback());
}
@@ -64,14 +62,22 @@
lsmIndex.getOperationalComponents(ctx);
List<ILSMComponent> components = ctx.getComponentHolder();
try {
-
- for (ILSMComponent c : components) {
- if (!c.threadEnter(opType)) {
- break;
+ // The purpose of the synchronized block is to make the beforeOperation call and entering the mutable component an atomic operation.
+ synchronized (this) {
+ for (ILSMComponent c : components) {
+ if (!c.threadEnter(opType)) {
+ break;
+ }
+ numEntered++;
}
- numEntered++;
+ entranceSuccessful = numEntered == components.size();
+ if (entranceSuccessful) {
+ if (!opTracker.beforeOperation(lsmIndex, opType, ctx.getSearchOperationCallback(),
+ ctx.getModificationCallback())) {
+ entranceSuccessful = false;
+ }
+ }
}
- entranceSuccessful = numEntered == components.size();
} catch (InterruptedException e) {
entranceSuccessful = false;
throw new HyracksDataException(e);
@@ -90,7 +96,6 @@
return false;
}
}
- opTracker.beforeOperation(lsmIndex, opType, ctx.getSearchOperationCallback(), ctx.getModificationCallback());
return true;
}
@@ -163,14 +168,15 @@
return;
}
- lsmIndex.setFlushStatus(false);
-
if (!lsmIndex.scheduleFlush(ctx, callback)) {
callback.beforeOperation();
callback.afterOperation(null, null);
- AbstractMutableLSMComponent mutableComponent = (AbstractMutableLSMComponent) lsmIndex.getMutableComponent();
- mutableComponent.registerOnResetCallback(new IMutableResetCallback() {
- public void reset() throws HyracksDataException {
+ AbstractMutableLSMComponent mutableComponent = (AbstractMutableLSMComponent) ctx.getComponentHolder()
+ .get(0);
+ mutableComponent.registerOnResetCallback(new IMutableComponentAdderCallback() {
+
+ @Override
+ public void addComponent() throws HyracksDataException {
// do nothing
}
});
@@ -191,9 +197,11 @@
operation.getCallback().afterOperation(null, newComponent);
lsmIndex.markAsValid(newComponent);
- AbstractMutableLSMComponent mutableComponent = (AbstractMutableLSMComponent) lsmIndex.getMutableComponent();
- mutableComponent.registerOnResetCallback(new IMutableResetCallback() {
- public void reset() throws HyracksDataException {
+ AbstractMutableLSMComponent flushingComponent = (AbstractMutableLSMComponent) ctx.getComponentHolder().get(0);
+ flushingComponent.registerOnResetCallback(new IMutableComponentAdderCallback() {
+
+ @Override
+ public void addComponent() throws HyracksDataException {
lsmIndex.addComponent(newComponent);
int numComponents = lsmIndex.getImmutableComponents().size();
try {
@@ -201,7 +209,6 @@
} catch (IndexException e) {
throw new HyracksDataException(e);
}
-
}
});
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
index b6ca21b..ea23091 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
@@ -39,7 +39,7 @@
protected PriorityQueueComparator pqCmp;
protected MultiComparator cmp;
protected boolean needPush;
- protected boolean includeMemComponent;
+ protected boolean includeMutableComponent;
protected ILSMHarness lsmHarness;
protected final ILSMIndexOperationContext opCtx;
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMOperationType.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMOperationType.java
index d932e6d..853b6d0 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMOperationType.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMOperationType.java
@@ -19,6 +19,5 @@
MODIFICATION,
FORCE_MODIFICATION,
FLUSH,
- MERGE,
- NOOP
+ MERGE
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java
index 19ee4b8..752e923 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java
@@ -32,11 +32,12 @@
}
@Override
- public void beforeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
+ public boolean beforeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException {
if (opType == LSMOperationType.MODIFICATION) {
threadRefCount.incrementAndGet();
}
+ return true;
}
@Override
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java
index 1afd9d5..ee8febe 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java
@@ -14,6 +14,8 @@
*/
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.dataflow;
+import java.util.List;
+
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.common.api.IIndex;
@@ -34,18 +36,18 @@
public final class LSMInvertedIndexDataflowHelper extends AbstractLSMIndexDataflowHelper {
public LSMInvertedIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- IVirtualBufferCache virtualBufferCache, ILSMMergePolicy mergePolicy,
+ List<IVirtualBufferCache> virtualBufferCaches, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
- this(opDesc, ctx, partition, virtualBufferCache, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
+ this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
opTrackerFactory, ioScheduler, ioOpCallbackProvider);
}
public LSMInvertedIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- IVirtualBufferCache virtualBufferCache, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
- super(opDesc, ctx, partition, virtualBufferCache, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
+ List<IVirtualBufferCache> virtualBufferCaches, double bloomFilterFalsePositiveRate,
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
+ super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackProvider);
}
@@ -55,7 +57,7 @@
try {
IBufferCache diskBufferCache = opDesc.getStorageManager().getBufferCache(ctx);
IFileMapProvider diskFileMapProvider = opDesc.getStorageManager().getFileMapProvider(ctx);
- LSMInvertedIndex invIndex = InvertedIndexUtils.createLSMInvertedIndex(virtualBufferCache,
+ LSMInvertedIndex invIndex = InvertedIndexUtils.createLSMInvertedIndex(virtualBufferCaches,
diskFileMapProvider, invIndexOpDesc.getInvListsTypeTraits(),
invIndexOpDesc.getInvListsComparatorFactories(), invIndexOpDesc.getTokenTypeTraits(),
invIndexOpDesc.getTokenComparatorFactories(), invIndexOpDesc.getTokenizerFactory(),
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
index c9f81fc..85aa312 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
@@ -41,7 +41,7 @@
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition) {
return new LSMInvertedIndexDataflowHelper(opDesc, ctx, partition,
- virtualBufferCacheProvider.getVirtualBufferCache(ctx), bloomFilterFalsePositiveRate,
+ virtualBufferCacheProvider.getVirtualBufferCaches(ctx), bloomFilterFalsePositiveRate,
mergePolicyProvider.getMergePolicy(ctx), opTrackerFactory, ioSchedulerProvider.getIOScheduler(ctx),
ioOpCallbackProvider);
}
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java
index 0dbd06b..1acb437 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java
@@ -14,6 +14,8 @@
*/
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.dataflow;
+import java.util.List;
+
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.common.api.IIndex;
@@ -34,7 +36,7 @@
public final class PartitionedLSMInvertedIndexDataflowHelper extends AbstractLSMIndexDataflowHelper {
public PartitionedLSMInvertedIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition, IVirtualBufferCache virtualBufferCache, ILSMMergePolicy mergePolicy,
+ int partition, List<IVirtualBufferCache> virtualBufferCache, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
this(opDesc, ctx, partition, virtualBufferCache, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
@@ -42,10 +44,10 @@
}
public PartitionedLSMInvertedIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition, IVirtualBufferCache virtualBufferCache, double bloomFilterFalsePositiveRate,
+ int partition, List<IVirtualBufferCache> virtualBufferCaches, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
- super(opDesc, ctx, partition, virtualBufferCache, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
+ super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackProvider);
}
@@ -56,7 +58,7 @@
IBufferCache diskBufferCache = opDesc.getStorageManager().getBufferCache(ctx);
IFileMapProvider diskFileMapProvider = opDesc.getStorageManager().getFileMapProvider(ctx);
PartitionedLSMInvertedIndex invIndex = InvertedIndexUtils.createPartitionedLSMInvertedIndex(
- virtualBufferCache, diskFileMapProvider, invIndexOpDesc.getInvListsTypeTraits(),
+ virtualBufferCaches, diskFileMapProvider, invIndexOpDesc.getInvListsTypeTraits(),
invIndexOpDesc.getInvListsComparatorFactories(), invIndexOpDesc.getTokenTypeTraits(),
invIndexOpDesc.getTokenComparatorFactories(), invIndexOpDesc.getTokenizerFactory(),
diskBufferCache, file.getFile().getPath(), bloomFilterFalsePositiveRate, mergePolicy,
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
index 67cab6d..63d99aa 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
@@ -41,7 +41,7 @@
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition) {
return new PartitionedLSMInvertedIndexDataflowHelper(opDesc, ctx, partition,
- virtualBufferCacheProvider.getVirtualBufferCache(ctx), bloomFilterFalsePositiveRate,
+ virtualBufferCacheProvider.getVirtualBufferCaches(ctx), bloomFilterFalsePositiveRate,
mergePolicyProvider.getMergePolicy(ctx), opTrackerFactory, ioSchedulerProvider.getIOScheduler(ctx),
ioOpCallbackProvider);
}
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 98df518..5b11f66 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -42,12 +42,13 @@
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IVirtualFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
-import edu.uci.ics.hyracks.storage.am.common.exceptions.TreeIndexDuplicateKeyException;
+import edu.uci.ics.hyracks.storage.am.common.exceptions.TreeIndexNonExistentKeyException;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.tuples.PermutingTupleReference;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
@@ -59,9 +60,11 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.IMutableComponentSwitcherCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractMutableLSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
@@ -80,8 +83,7 @@
public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex {
// In-memory components.
- protected final LSMInvertedIndexMutableComponent mutableComponent;
- protected final IVirtualFreePageManager virtualFreePageManager;
+ private final List<LSMInvertedIndexMutableComponent> mutableComponents;
protected final IBinaryTokenizerFactory tokenizerFactory;
// On-disk components.
@@ -94,31 +96,56 @@
protected final ITypeTraits[] tokenTypeTraits;
protected final IBinaryComparatorFactory[] tokenCmpFactories;
- public LSMInvertedIndex(IVirtualBufferCache virtualBufferCache, OnDiskInvertedIndexFactory diskInvIndexFactory,
- BTreeFactory deletedKeysBTreeFactory, BloomFilterFactory bloomFilterFactory,
- double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager,
- IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits,
+ public LSMInvertedIndex(List<IVirtualBufferCache> virtualBufferCaches,
+ OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory,
+ BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
+ ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackProvider ioOpCallbackProvider) throws IndexException {
- super(virtualBufferCache, diskInvIndexFactory.getBufferCache(), fileManager, diskFileMapProvider,
+ super(virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallbackProvider);
- this.virtualFreePageManager = new VirtualFreePageManager(virtualBufferCache.getNumPages());
+
this.tokenizerFactory = tokenizerFactory;
this.invListTypeTraits = invListTypeTraits;
this.invListCmpFactories = invListCmpFactories;
this.tokenTypeTraits = tokenTypeTraits;
this.tokenCmpFactories = tokenCmpFactories;
- // Create in-memory component.
- InMemoryInvertedIndex memInvIndex = createInMemoryInvertedIndex(virtualBufferCache);
- BTree deleteKeysBTree = BTreeUtils.createBTree(virtualBufferCache, new VirtualFreePageManager(
- virtualBufferCache.getNumPages()), ((IVirtualBufferCache) virtualBufferCache).getFileMapProvider(),
- invListTypeTraits, invListCmpFactories, BTreeLeafFrameType.REGULAR_NSM, new FileReference(new File(
- fileManager.getBaseDir() + "_virtual_del")));
- mutableComponent = new LSMInvertedIndexMutableComponent(memInvIndex, deleteKeysBTree, virtualBufferCache);
+
componentFactory = new LSMInvertedIndexComponentFactory(diskInvIndexFactory, deletedKeysBTreeFactory,
bloomFilterFactory);
+
+ mutableComponents = new ArrayList<LSMInvertedIndexMutableComponent>();
+ final int numMutableComponents = virtualBufferCaches.size();
+ int i = 0;
+ for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
+ InMemoryInvertedIndex memInvIndex = createInMemoryInvertedIndex(virtualBufferCache,
+ new VirtualFreePageManager(virtualBufferCache.getNumPages()), i);
+ BTree deleteKeysBTree = BTreeUtils.createBTree(virtualBufferCache, new VirtualFreePageManager(
+ virtualBufferCache.getNumPages()), ((IVirtualBufferCache) virtualBufferCache).getFileMapProvider(),
+ invListTypeTraits, invListCmpFactories, BTreeLeafFrameType.REGULAR_NSM, new FileReference(new File(
+ fileManager.getBaseDir() + "_virtual_del_" + i)));
+ LSMInvertedIndexMutableComponent mutableComponent = new LSMInvertedIndexMutableComponent(memInvIndex,
+ deleteKeysBTree, virtualBufferCache, i == 0 ? true : false);
+
+ mutableComponent.registerOnFlushCallback(new IMutableComponentSwitcherCallback() {
+
+ @Override
+ public void switchComponents() throws HyracksDataException {
+ currentMutableComponentId.set((currentMutableComponentId.get() + 1) % numMutableComponents);
+ mutableComponents.get(currentMutableComponentId.get()).setActive();
+ }
+
+ @Override
+ public void setFlushStatus(boolean isFlushNeeded) {
+ needsFlush[currentMutableComponentId.get()].set(isFlushNeeded);
+ }
+ });
+
+ mutableComponents.add(mutableComponent);
+ ++i;
+ }
}
@Override
@@ -140,11 +167,14 @@
try {
List<ILSMComponent> immutableComponents = componentsRef.get();
- ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).open();
- mutableComponent.getInvIndex().create();
- mutableComponent.getInvIndex().activate();
- mutableComponent.getDeletedKeysBTree().create();
- mutableComponent.getDeletedKeysBTree().activate();
+ for (LSMInvertedIndexMutableComponent mutableComponent : mutableComponents) {
+ ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).open();
+ mutableComponent.getInvIndex().create();
+ mutableComponent.getInvIndex().activate();
+ mutableComponent.getDeletedKeysBTree().create();
+ mutableComponent.getDeletedKeysBTree().activate();
+ }
+
immutableComponents.clear();
List<LSMComponentFileReferences> validFileReferences = fileManager.cleanupAndGetValidFiles();
for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
@@ -172,9 +202,11 @@
throw new HyracksDataException("Failed to clear the index since it is not activated.");
}
List<ILSMComponent> immutableComponents = componentsRef.get();
- mutableComponent.getInvIndex().clear();
- mutableComponent.getDeletedKeysBTree().clear();
- mutableComponent.reset();
+ for (LSMInvertedIndexMutableComponent mutableComponent : mutableComponents) {
+ mutableComponent.getInvIndex().clear();
+ mutableComponent.getDeletedKeysBTree().clear();
+ mutableComponent.reset();
+ }
for (ILSMComponent c : immutableComponents) {
LSMInvertedIndexImmutableComponent component = (LSMInvertedIndexImmutableComponent) c;
component.getBloomFilter().deactivate();
@@ -214,11 +246,13 @@
component.getInvIndex().deactivate();
component.getDeletedKeysBTree().deactivate();
}
- mutableComponent.getInvIndex().deactivate();
- mutableComponent.getDeletedKeysBTree().deactivate();
- mutableComponent.getInvIndex().destroy();
- mutableComponent.getDeletedKeysBTree().destroy();
- ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).close();
+ for (LSMInvertedIndexMutableComponent mutableComponent : mutableComponents) {
+ mutableComponent.getInvIndex().deactivate();
+ mutableComponent.getDeletedKeysBTree().deactivate();
+ mutableComponent.getInvIndex().destroy();
+ mutableComponent.getDeletedKeysBTree().destroy();
+ ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).close();
+ }
}
@Override
@@ -232,8 +266,10 @@
throw new HyracksDataException("Failed to destroy the index since it is activated.");
}
- mutableComponent.getInvIndex().destroy();
- mutableComponent.getDeletedKeysBTree().destroy();
+ for (LSMInvertedIndexMutableComponent mutableComponent : mutableComponents) {
+ mutableComponent.getInvIndex().destroy();
+ mutableComponent.getDeletedKeysBTree().destroy();
+ }
List<ILSMComponent> immutableComponents = componentsRef.get();
for (ILSMComponent c : immutableComponents) {
LSMInvertedIndexImmutableComponent component = (LSMInvertedIndexImmutableComponent) c;
@@ -245,23 +281,30 @@
}
@Override
- public ILSMComponent getMutableComponent() {
- return mutableComponent;
- }
-
- @Override
public void getOperationalComponents(ILSMIndexOperationContext ctx) {
List<ILSMComponent> immutableComponents = componentsRef.get();
List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
operationalComponents.clear();
+ int cmc = currentMutableComponentId.get();
+ ctx.setCurrentMutableComponentId(cmc);
+ int numMutableComponents = mutableComponents.size();
switch (ctx.getOperation()) {
case FLUSH:
case DELETE:
case INSERT:
- operationalComponents.add(mutableComponent);
+ operationalComponents.add(mutableComponents.get(cmc));
break;
case SEARCH:
- operationalComponents.add(mutableComponent);
+ for (int i = 0; i < numMutableComponents - 1; i++) {
+ LSMInvertedIndexMutableComponent mutableComponent = mutableComponents.get((cmc + i + 1)
+ % numMutableComponents);
+ if (mutableComponent.isReadable()) {
+ // Make sure newest components are added first
+ operationalComponents.add(0, mutableComponent);
+ }
+ }
+ // The current mutable component is always added
+ operationalComponents.add(0, mutableComponents.get(cmc));
operationalComponents.addAll(immutableComponents);
break;
case MERGE:
@@ -300,17 +343,17 @@
switch (ctx.getOperation()) {
case INSERT: {
// Insert into the in-memory inverted index.
- ctx.memInvIndexAccessor.insert(tuple);
+ ctx.currentMutableInvIndexAccessors.insert(tuple);
break;
}
case DELETE: {
// First remove all entries in the in-memory inverted index (if any).
- ctx.memInvIndexAccessor.delete(tuple);
+ ctx.currentMutableInvIndexAccessors.delete(tuple);
// Insert key into the deleted-keys BTree.
ctx.keysOnlyTuple.reset(tuple);
try {
- ctx.deletedKeysBTreeAccessor.insert(ctx.keysOnlyTuple);
- } catch (TreeIndexDuplicateKeyException e) {
+ ctx.currentDeletedKeysBTreeAccessors.insert(ctx.keysOnlyTuple);
+ } catch (TreeIndexNonExistentKeyException e) {
// Key has already been deleted.
}
break;
@@ -319,7 +362,7 @@
throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
}
}
- mutableComponent.setIsModified();
+ mutableComponents.get(currentMutableComponentId.get()).setIsModified();
}
@Override
@@ -328,59 +371,59 @@
List<ILSMComponent> operationalComponents = ictx.getComponentHolder();
int numComponents = operationalComponents.size();
assert numComponents > 0;
- boolean includeMutableComponent = operationalComponents.get(0) == mutableComponent;
+ boolean includeMutableComponent = false;
ArrayList<IIndexAccessor> indexAccessors = new ArrayList<IIndexAccessor>(numComponents);
ArrayList<IIndexAccessor> deletedKeysBTreeAccessors = new ArrayList<IIndexAccessor>(numComponents);
- if (includeMutableComponent) {
- IIndexAccessor invIndexAccessor = mutableComponent.getInvIndex().createAccessor(
- NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- indexAccessors.add(invIndexAccessor);
- IIndexAccessor deletedKeysAccessor = mutableComponent.getDeletedKeysBTree().createAccessor(
- NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- deletedKeysBTreeAccessors.add(deletedKeysAccessor);
- }
- for (int i = includeMutableComponent ? 1 : 0; i < operationalComponents.size(); i++) {
- LSMInvertedIndexImmutableComponent component = (LSMInvertedIndexImmutableComponent) operationalComponents
- .get(i);
- IIndexAccessor invIndexAccessor = component.getInvIndex().createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
- indexAccessors.add(invIndexAccessor);
- IIndexAccessor deletedKeysAccessor = component.getDeletedKeysBTree().createAccessor(
- NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- deletedKeysBTreeAccessors.add(deletedKeysAccessor);
+ for (int i = 0; i < operationalComponents.size(); i++) {
+ ILSMComponent component = operationalComponents.get(i);
+ if (component.getType() == LSMComponentType.MEMORY) {
+ includeMutableComponent = true;
+ IIndexAccessor invIndexAccessor = ((LSMInvertedIndexMutableComponent) component).getInvIndex()
+ .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ indexAccessors.add(invIndexAccessor);
+ IIndexAccessor deletedKeysAccessor = ((LSMInvertedIndexMutableComponent) component)
+ .getDeletedKeysBTree().createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ deletedKeysBTreeAccessors.add(deletedKeysAccessor);
+ } else {
+ IIndexAccessor invIndexAccessor = ((LSMInvertedIndexImmutableComponent) component).getInvIndex()
+ .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ indexAccessors.add(invIndexAccessor);
+ IIndexAccessor deletedKeysAccessor = ((LSMInvertedIndexImmutableComponent) component)
+ .getDeletedKeysBTree().createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ deletedKeysBTreeAccessors.add(deletedKeysAccessor);
+ }
}
ICursorInitialState initState = createCursorInitialState(pred, ictx, includeMutableComponent, indexAccessors,
- deletedKeysBTreeAccessors);
+ deletedKeysBTreeAccessors, operationalComponents);
cursor.open(initState, pred);
}
private ICursorInitialState createCursorInitialState(ISearchPredicate pred, IIndexOperationContext ictx,
boolean includeMutableComponent, ArrayList<IIndexAccessor> indexAccessors,
- ArrayList<IIndexAccessor> deletedKeysBTreeAccessors) {
- List<ILSMComponent> immutableComponents = componentsRef.get();
+ ArrayList<IIndexAccessor> deletedKeysBTreeAccessors, List<ILSMComponent> operationalComponents) {
ICursorInitialState initState = null;
PermutingTupleReference keysOnlyTuple = createKeysOnlyTupleReference();
MultiComparator keyCmp = MultiComparator.createIgnoreFieldLength(invListCmpFactories);
- List<ILSMComponent> operationalComponents = new ArrayList<ILSMComponent>();
- if (includeMutableComponent) {
- operationalComponents.add(mutableComponent);
- }
- operationalComponents.addAll(immutableComponents);
// TODO: This check is not pretty, but it does the job. Come up with something more OO in the future.
// Distinguish between regular searches and range searches (mostly used in merges).
if (pred instanceof InvertedIndexSearchPredicate) {
initState = new LSMInvertedIndexSearchCursorInitialState(keyCmp, keysOnlyTuple, indexAccessors,
- deletedKeysBTreeAccessors, mutableComponent.getDeletedKeysBTree().getLeafFrameFactory(), ictx,
- includeMutableComponent, lsmHarness, operationalComponents);
+ deletedKeysBTreeAccessors, mutableComponents.get(currentMutableComponentId.get())
+ .getDeletedKeysBTree().getLeafFrameFactory(), ictx, includeMutableComponent, lsmHarness,
+ operationalComponents);
} else {
- InMemoryInvertedIndex memInvIndex = (InMemoryInvertedIndex) mutableComponent.getInvIndex();
+ InMemoryInvertedIndex memInvIndex = (InMemoryInvertedIndex) mutableComponents.get(
+ currentMutableComponentId.get()).getInvIndex();
MultiComparator tokensAndKeysCmp = MultiComparator.create(memInvIndex.getBTree().getComparatorFactories());
initState = new LSMInvertedIndexRangeSearchCursorInitialState(tokensAndKeysCmp, keyCmp, keysOnlyTuple,
- mutableComponent.getDeletedKeysBTree().getLeafFrameFactory(), includeMutableComponent, lsmHarness,
- indexAccessors, deletedKeysBTreeAccessors, pred, operationalComponents);
+ mutableComponents.get(currentMutableComponentId.get()).getDeletedKeysBTree().getLeafFrameFactory(),
+ includeMutableComponent, lsmHarness, indexAccessors, deletedKeysBTreeAccessors, pred,
+ operationalComponents);
}
return initState;
}
@@ -401,19 +444,19 @@
@Override
public boolean scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException {
- if (!mutableComponent.isModified()) {
+ ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
+ if (!((AbstractMutableLSMComponent) flushingComponent).isModified()) {
return false;
}
LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
LSMInvertedIndexOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
- ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
opCtx.setOperation(IndexOperation.FLUSH);
opCtx.getComponentHolder().add(flushingComponent);
- ioScheduler.scheduleOperation(new LSMInvertedIndexFlushOperation(new LSMInvertedIndexAccessor(this, lsmHarness,
- fileManager, opCtx), mutableComponent, componentFileRefs.getInsertIndexFileReference(),
- componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(),
- callback));
+ ioScheduler.scheduleOperation(new LSMInvertedIndexFlushOperation(
+ new LSMInvertedIndexAccessor(lsmHarness, opCtx), flushingComponent, componentFileRefs
+ .getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(),
+ componentFileRefs.getBloomFilterFileReference(), callback));
return true;
}
@@ -428,7 +471,8 @@
IInvertedIndex diskInvertedIndex = component.getInvIndex();
// Create a scan cursor on the BTree underlying the in-memory inverted index.
- LSMInvertedIndexMutableComponent flushingComponent = flushOp.getFlushingComponent();
+ LSMInvertedIndexMutableComponent flushingComponent = (LSMInvertedIndexMutableComponent) flushOp
+ .getFlushingComponent();
InMemoryInvertedIndexAccessor memInvIndexAccessor = (InMemoryInvertedIndexAccessor) flushingComponent
.getInvIndex().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
BTreeAccessor memBTreeAccessor = memInvIndexAccessor.getBTreeAccessor();
@@ -521,7 +565,7 @@
String lastFileName = lastInvIndex.getBTree().getFileReference().getFile().getName();
LSMComponentFileReferences relMergeFileRefs = fileManager.getRelMergeFileReference(firstFileName, lastFileName);
- ILSMIndexAccessorInternal accessor = new LSMInvertedIndexAccessor(this, lsmHarness, fileManager, ictx);
+ ILSMIndexAccessorInternal accessor = new LSMInvertedIndexAccessor(lsmHarness, ctx);
ioScheduler.scheduleOperation(new LSMInvertedIndexMergeOperation(accessor, mergingComponents, cursor,
relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(),
relMergeFileRefs.getBloomFilterFileReference(), callback));
@@ -574,7 +618,14 @@
}
public boolean isEmptyIndex() throws HyracksDataException {
- return componentsRef.get().isEmpty() && !mutableComponent.isModified();
+ boolean isModified = false;
+ for (LSMInvertedIndexMutableComponent mutableComponent : mutableComponents) {
+ if (mutableComponent.isModified()) {
+ isModified = true;
+ break;
+ }
+ }
+ return componentsRef.get().isEmpty() && !isModified;
}
public class LSMInvertedIndexBulkLoader implements IIndexBulkLoader {
@@ -637,11 +688,11 @@
}
}
- protected InMemoryInvertedIndex createInMemoryInvertedIndex(IVirtualBufferCache virtualBufferCache)
- throws IndexException {
+ protected InMemoryInvertedIndex createInMemoryInvertedIndex(IVirtualBufferCache virtualBufferCache,
+ IVirtualFreePageManager virtualFreePageManager, int id) throws IndexException {
return InvertedIndexUtils.createInMemoryBTreeInvertedindex(virtualBufferCache, virtualFreePageManager,
invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
- new FileReference(new File(fileManager.getBaseDir() + "_virtual_vocab")));
+ new FileReference(new File(fileManager.getBaseDir() + "_virtual_vocab_" + id)));
}
protected LSMInvertedIndexImmutableComponent createDiskInvIndexComponent(ILSMComponentFactory factory,
@@ -664,15 +715,13 @@
@Override
public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback) {
- return new LSMInvertedIndexAccessor(this, lsmHarness, fileManager, createOpContext(modificationCallback,
- searchCallback));
+ ISearchOperationCallback searchCallback) throws HyracksDataException {
+ return new LSMInvertedIndexAccessor(lsmHarness, createOpContext(modificationCallback, searchCallback));
}
private LSMInvertedIndexOpContext createOpContext(IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback) {
- return new LSMInvertedIndexOpContext(mutableComponent.getInvIndex(), mutableComponent.getDeletedKeysBTree(),
- modificationCallback, searchCallback);
+ ISearchOperationCallback searchCallback) throws HyracksDataException {
+ return new LSMInvertedIndexOpContext(mutableComponents, modificationCallback, searchCallback);
}
@Override
@@ -698,8 +747,12 @@
@Override
public long getMemoryAllocationSize() {
- IBufferCache virtualBufferCache = mutableComponent.getInvIndex().getBufferCache();
- return virtualBufferCache.getNumPages() * virtualBufferCache.getPageSize();
+ long size = 0;
+ for (LSMInvertedIndexMutableComponent mutableComponent : mutableComponents) {
+ IBufferCache virtualBufferCache = mutableComponent.getInvIndex().getBufferCache();
+ size += virtualBufferCache.getNumPages() * virtualBufferCache.getPageSize();
+ }
+ return size;
}
@Override
@@ -747,8 +800,10 @@
@Override
public void validate() throws HyracksDataException {
- mutableComponent.getInvIndex().validate();
- mutableComponent.getDeletedKeysBTree().validate();
+ for (LSMInvertedIndexMutableComponent mutableComponent : mutableComponents) {
+ mutableComponent.getInvIndex().validate();
+ mutableComponent.getDeletedKeysBTree().validate();
+ }
List<ILSMComponent> immutableComponents = componentsRef.get();
for (ILSMComponent c : immutableComponents) {
LSMInvertedIndexImmutableComponent component = (LSMInvertedIndexImmutableComponent) c;
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
index db7529c..e31af9a 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
@@ -25,7 +25,6 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
@@ -33,16 +32,11 @@
public class LSMInvertedIndexAccessor implements ILSMIndexAccessorInternal, IInvertedIndexAccessor {
protected final ILSMHarness lsmHarness;
- protected final ILSMIndexFileManager fileManager;
protected final ILSMIndexOperationContext ctx;
- protected final LSMInvertedIndex invIndex;
- public LSMInvertedIndexAccessor(LSMInvertedIndex invIndex, ILSMHarness lsmHarness,
- ILSMIndexFileManager fileManager, ILSMIndexOperationContext ctx) {
+ public LSMInvertedIndexAccessor(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) {
this.lsmHarness = lsmHarness;
- this.fileManager = fileManager;
this.ctx = ctx;
- this.invIndex = invIndex;
}
@Override
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
index a8973d1..cf5820e 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
@@ -23,22 +23,22 @@
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
public class LSMInvertedIndexFlushOperation implements ILSMIOOperation {
private final ILSMIndexAccessorInternal accessor;
- private final LSMInvertedIndexMutableComponent flushingComponent;
+ private final ILSMComponent flushingComponent;
private final FileReference dictBTreeFlushTarget;
private final FileReference deletedKeysBTreeFlushTarget;
private final FileReference bloomFilterFlushTarget;
private final ILSMIOOperationCallback callback;
- public LSMInvertedIndexFlushOperation(ILSMIndexAccessorInternal accessor,
- LSMInvertedIndexMutableComponent flushingComponent, FileReference dictBTreeFlushTarget,
- FileReference deletedKeysBTreeFlushTarget, FileReference bloomFilterFlushTarget,
- ILSMIOOperationCallback callback) {
+ public LSMInvertedIndexFlushOperation(ILSMIndexAccessorInternal accessor, ILSMComponent flushingComponent,
+ FileReference dictBTreeFlushTarget, FileReference deletedKeysBTreeFlushTarget,
+ FileReference bloomFilterFlushTarget, ILSMIOOperationCallback callback) {
this.accessor = accessor;
this.flushingComponent = flushingComponent;
this.dictBTreeFlushTarget = dictBTreeFlushTarget;
@@ -84,7 +84,7 @@
return bloomFilterFlushTarget;
}
- public LSMInvertedIndexMutableComponent getFlushingComponent() {
+ public ILSMComponent getFlushingComponent() {
return flushingComponent;
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMutableComponent.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMutableComponent.java
index d6c3c5e..72fc3ea 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMutableComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMutableComponent.java
@@ -26,8 +26,9 @@
private final IInvertedIndex invIndex;
private final BTree deletedKeysBTree;
- public LSMInvertedIndexMutableComponent(IInvertedIndex invIndex, BTree deletedKeysBTree, IVirtualBufferCache vbc) {
- super(vbc);
+ public LSMInvertedIndexMutableComponent(IInvertedIndex invIndex, BTree deletedKeysBTree, IVirtualBufferCache vbc,
+ boolean isActive) {
+ super(vbc, isActive);
this.invIndex = invIndex;
this.deletedKeysBTree = deletedKeysBTree;
}
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
index fd83bfb..95df583 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
@@ -19,7 +19,6 @@
import java.util.List;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndex;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
@@ -28,7 +27,6 @@
import edu.uci.ics.hyracks.storage.am.common.tuples.PermutingTupleReference;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
-import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor;
public class LSMInvertedIndexOpContext implements ILSMIndexOperationContext {
@@ -36,8 +34,6 @@
private static final int NUM_DOCUMENT_FIELDS = 1;
private IndexOperation op;
- private final IInvertedIndex memInvIndex;
- private final IIndex memDeletedKeysBTree;
private final List<ILSMComponent> componentHolder;
public final IModificationOperationCallback modificationCallback;
@@ -46,18 +42,40 @@
// Tuple that only has the inverted-index elements (aka keys), projecting away the document fields.
public PermutingTupleReference keysOnlyTuple;
- // Accessor to the in-memory inverted index.
- public IInvertedIndexAccessor memInvIndexAccessor;
- // Accessor to the deleted-keys BTree.
- public IIndexAccessor deletedKeysBTreeAccessor;
+ // Accessor to the in-memory inverted indexes.
+ public IInvertedIndexAccessor[] mutableInvIndexAccessors;
+ // Accessor to the deleted-keys BTrees.
+ public IIndexAccessor[] deletedKeysBTreeAccessors;
- public LSMInvertedIndexOpContext(IInvertedIndex memInvIndex, IIndex memDeletedKeysBTree,
- IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) {
- this.memInvIndex = memInvIndex;
- this.memDeletedKeysBTree = memDeletedKeysBTree;
+ public IInvertedIndexAccessor currentMutableInvIndexAccessors;
+ public IIndexAccessor currentDeletedKeysBTreeAccessors;
+
+ public LSMInvertedIndexOpContext(List<LSMInvertedIndexMutableComponent> mutableComponents,
+ IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback)
+ throws HyracksDataException {
this.componentHolder = new LinkedList<ILSMComponent>();
this.modificationCallback = modificationCallback;
this.searchCallback = searchCallback;
+
+ mutableInvIndexAccessors = new IInvertedIndexAccessor[mutableComponents.size()];
+ deletedKeysBTreeAccessors = new IIndexAccessor[mutableComponents.size()];
+
+ for (int i = 0; i < mutableComponents.size(); i++) {
+ mutableInvIndexAccessors[i] = (IInvertedIndexAccessor) mutableComponents.get(i).getInvIndex()
+ .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ deletedKeysBTreeAccessors[i] = mutableComponents.get(i).getDeletedKeysBTree()
+ .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ }
+
+ assert mutableComponents.size() > 0;
+
+ // Project away the document fields, leaving only the key fields.
+ int numKeyFields = mutableComponents.get(0).getInvIndex().getInvListTypeTraits().length;
+ int[] keyFieldPermutation = new int[numKeyFields];
+ for (int i = 0; i < numKeyFields; i++) {
+ keyFieldPermutation[i] = NUM_DOCUMENT_FIELDS + i;
+ }
+ keysOnlyTuple = new PermutingTupleReference(keyFieldPermutation);
}
@Override
@@ -69,26 +87,6 @@
// TODO: Ignore opcallback for now.
public void setOperation(IndexOperation newOp) throws HyracksDataException {
reset();
- switch (newOp) {
- case INSERT:
- case DELETE:
- case PHYSICALDELETE: {
- if (deletedKeysBTreeAccessor == null) {
- memInvIndexAccessor = (IInvertedIndexAccessor) memInvIndex.createAccessor(
- NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- deletedKeysBTreeAccessor = memDeletedKeysBTree.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
- // Project away the document fields, leaving only the key fields.
- int numKeyFields = memInvIndex.getInvListTypeTraits().length;
- int[] keyFieldPermutation = new int[numKeyFields];
- for (int i = 0; i < numKeyFields; i++) {
- keyFieldPermutation[i] = NUM_DOCUMENT_FIELDS + i;
- }
- keysOnlyTuple = new PermutingTupleReference(keyFieldPermutation);
- }
- break;
- }
- }
op = newOp;
}
@@ -111,4 +109,10 @@
public IModificationOperationCallback getModificationCallback() {
return modificationCallback;
}
+
+ @Override
+ public void setCurrentMutableComponentId(int currentMutableComponentId) {
+ currentMutableInvIndexAccessors = mutableInvIndexAccessors[currentMutableComponentId];
+ currentDeletedKeysBTreeAccessors = deletedKeysBTreeAccessors[currentMutableComponentId];
+ }
}
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
index 820b516..d7911b1 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
@@ -63,7 +63,7 @@
}
lsmHarness = lsmInitState.getLSMHarness();
operationalComponents = lsmInitState.getOperationalComponents();
- includeMemComponent = lsmInitState.getIncludeMemComponent();
+ includeMutableComponent = lsmInitState.getIncludeMemComponent();
// For searching the deleted-keys BTrees.
this.keysOnlyTuple = lsmInitState.getKeysOnlyTuple();
@@ -72,7 +72,7 @@
if (!deletedKeysBTreeAccessors.isEmpty()) {
deletedKeysBTreeCursors = new IIndexCursor[deletedKeysBTreeAccessors.size()];
int i = 0;
- if (includeMemComponent) {
+ if (includeMutableComponent) {
// No need for a bloom filter for the in-memory BTree.
deletedKeysBTreeCursors[i] = deletedKeysBTreeAccessors.get(i).createSearchCursor();
++i;
@@ -113,7 +113,7 @@
}
return false;
}
-
+
@Override
protected void checkPriorityQueue() throws HyracksDataException, IndexException {
while (!outputPriorityQueue.isEmpty() || needPush == true) {
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
index 185321b..5b0c48d 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
@@ -16,11 +16,13 @@
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls;
import java.io.File;
+import java.util.List;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IVirtualFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -37,7 +39,7 @@
public class PartitionedLSMInvertedIndex extends LSMInvertedIndex {
- public PartitionedLSMInvertedIndex(IVirtualBufferCache virtualBufferCache,
+ public PartitionedLSMInvertedIndex(List<IVirtualBufferCache> virtualBufferCaches,
OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory,
BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits,
@@ -45,17 +47,18 @@
IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackProvider ioOpCallbackProvider) throws IndexException {
- super(virtualBufferCache, diskInvIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory,
+ super(virtualBufferCaches, diskInvIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory,
bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories,
tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler,
ioOpCallbackProvider);
}
- protected InMemoryInvertedIndex createInMemoryInvertedIndex(IVirtualBufferCache virtualBufferCache)
- throws IndexException {
+ @Override
+ protected InMemoryInvertedIndex createInMemoryInvertedIndex(IVirtualBufferCache virtualBufferCache,
+ IVirtualFreePageManager virtualFreePageManager, int id) throws IndexException {
return InvertedIndexUtils.createPartitionedInMemoryBTreeInvertedindex(virtualBufferCache,
virtualFreePageManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories,
- tokenizerFactory, new FileReference(new File(fileManager.getBaseDir() + "_virtual_vocab")));
+ tokenizerFactory, new FileReference(new File(fileManager.getBaseDir() + "_virtual_vocab_" + id)));
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
index 9d85f56..ed02ba1 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
@@ -16,6 +16,7 @@
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.util;
import java.io.File;
+import java.util.List;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
@@ -115,7 +116,7 @@
return deletedKeysBTreeFactory;
}
- public static LSMInvertedIndex createLSMInvertedIndex(IVirtualBufferCache virtualBufferCache,
+ public static LSMInvertedIndex createLSMInvertedIndex(List<IVirtualBufferCache> virtualBufferCaches,
IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
@@ -143,20 +144,21 @@
diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, fileManager);
- LSMInvertedIndex invIndex = new LSMInvertedIndex(virtualBufferCache, invIndexFactory, deletedKeysBTreeFactory,
+ LSMInvertedIndex invIndex = new LSMInvertedIndex(virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory,
bloomFilterFactory, bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits,
invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker,
ioScheduler, ioOpCallbackProvider);
return invIndex;
}
- public static PartitionedLSMInvertedIndex createPartitionedLSMInvertedIndex(IVirtualBufferCache virtualBufferCache,
- IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits,
- IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
- IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
- IBufferCache diskBufferCache, String onDiskDir, double bloomFilterFalsePositiveRate,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackProvider ioOpCallbackProvider) throws IndexException {
+ public static PartitionedLSMInvertedIndex createPartitionedLSMInvertedIndex(
+ List<IVirtualBufferCache> virtualBufferCaches, IFileMapProvider diskFileMapProvider,
+ ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
+ ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
+ IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String onDiskDir,
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider)
+ throws IndexException {
BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(diskFileMapProvider, invListTypeTraits,
invListCmpFactories, diskBufferCache);
@@ -178,7 +180,7 @@
diskBufferCache, diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories,
tokenTypeTraits, tokenCmpFactories, fileManager);
- PartitionedLSMInvertedIndex invIndex = new PartitionedLSMInvertedIndex(virtualBufferCache, invIndexFactory,
+ PartitionedLSMInvertedIndex invIndex = new PartitionedLSMInvertedIndex(virtualBufferCaches, invIndexFactory,
deletedKeysBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, fileManager,
diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories,
tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallbackProvider);
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
index 9fc8cea..5d57aae 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
@@ -15,6 +15,8 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.dataflow;
+import java.util.List;
+
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
@@ -44,24 +46,24 @@
protected final ILinearizeComparatorFactory linearizeCmpFactory;
public AbstractLSMRTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- IVirtualBufferCache virtualBufferCache, IBinaryComparatorFactory[] btreeComparatorFactories,
+ List<IVirtualBufferCache> virtualBufferCaches, IBinaryComparatorFactory[] btreeComparatorFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
ILinearizeComparatorFactory linearizeCmpFactory) {
- this(opDesc, ctx, partition, virtualBufferCache, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE,
+ this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE,
btreeComparatorFactories, valueProviderFactories, rtreePolicyType, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackProvider, linearizeCmpFactory);
}
public AbstractLSMRTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- IVirtualBufferCache virtualBufferCache, double bloomFilterFalsePositiveRate,
+ List<IVirtualBufferCache> virtualBufferCaches, double bloomFilterFalsePositiveRate,
IBinaryComparatorFactory[] btreeComparatorFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
ILinearizeComparatorFactory linearizeCmpFactory) {
- super(opDesc, ctx, partition, virtualBufferCache, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
+ super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackProvider);
this.btreeComparatorFactories = btreeComparatorFactories;
this.valueProviderFactories = valueProviderFactories;
@@ -72,14 +74,14 @@
@Override
public ITreeIndex createIndexInstance() throws HyracksDataException {
AbstractTreeIndexOperatorDescriptor treeOpDesc = (AbstractTreeIndexOperatorDescriptor) opDesc;
- return createLSMTree(virtualBufferCache, file, opDesc.getStorageManager().getBufferCache(ctx), opDesc
+ return createLSMTree(virtualBufferCaches, file, opDesc.getStorageManager().getBufferCache(ctx), opDesc
.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
treeOpDesc.getTreeIndexComparatorFactories(), btreeComparatorFactories,
opTrackerFactory.getOperationTracker(ctx), valueProviderFactories, rtreePolicyType, linearizeCmpFactory);
}
- protected abstract ITreeIndex createLSMTree(IVirtualBufferCache virtualBufferCache, FileReference file,
+ protected abstract ITreeIndex createLSMTree(List<IVirtualBufferCache> virtualBufferCaches, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
ILSMOperationTracker opTracker, IPrimitiveValueProviderFactory[] valueProviderFactories,
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
index b47bc02..bc4bca3 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
@@ -15,6 +15,8 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.dataflow;
+import java.util.List;
+
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
@@ -39,36 +41,36 @@
public class LSMRTreeDataflowHelper extends AbstractLSMRTreeDataflowHelper {
public LSMRTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- IVirtualBufferCache virtualBufferCache, IBinaryComparatorFactory[] btreeComparatorFactories,
+ List<IVirtualBufferCache> virtualBufferCaches, IBinaryComparatorFactory[] btreeComparatorFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
ILinearizeComparatorFactory linearizeCmpFactory) {
- super(opDesc, ctx, partition, virtualBufferCache, btreeComparatorFactories, valueProviderFactories,
+ super(opDesc, ctx, partition, virtualBufferCaches, btreeComparatorFactories, valueProviderFactories,
rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackProvider, linearizeCmpFactory);
}
public LSMRTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- IVirtualBufferCache virtualBufferCache, double bloomFilterFalsePositiveRate,
+ List<IVirtualBufferCache> virtualBufferCaches, double bloomFilterFalsePositiveRate,
IBinaryComparatorFactory[] btreeComparatorFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
ILinearizeComparatorFactory linearizeCmpFactory) {
- super(opDesc, ctx, partition, virtualBufferCache, bloomFilterFalsePositiveRate, btreeComparatorFactories,
+ super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, btreeComparatorFactories,
valueProviderFactories, rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler,
ioOpCallbackProvider, linearizeCmpFactory);
}
@Override
- protected ITreeIndex createLSMTree(IVirtualBufferCache virtualBufferCache, FileReference file,
+ protected ITreeIndex createLSMTree(List<IVirtualBufferCache> virtualBufferCaches, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
ILSMOperationTracker opTracker, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory)
throws HyracksDataException {
try {
- return LSMRTreeUtils.createLSMTree(virtualBufferCache, file, diskBufferCache, diskFileMapProvider,
+ return LSMRTreeUtils.createLSMTree(virtualBufferCaches, file, diskBufferCache, diskFileMapProvider,
typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallbackProvider,
linearizeCmpFactory);
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
index 5d35bc5..99b4219 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
@@ -56,7 +56,7 @@
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition) {
return new LSMRTreeDataflowHelper(opDesc, ctx, partition,
- virtualBufferCacheProvider.getVirtualBufferCache(ctx), bloomFilterFalsePositiveRate,
+ virtualBufferCacheProvider.getVirtualBufferCaches(ctx), bloomFilterFalsePositiveRate,
btreeComparatorFactories, valueProviderFactories, rtreePolicyType,
mergePolicyProvider.getMergePolicy(ctx), opTrackerFactory, ioSchedulerProvider.getIOScheduler(ctx),
ioOpCallbackProvider, linearizeCmpFactory);
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
index ea1b1e2..72f3480 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
@@ -15,6 +15,8 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.dataflow;
+import java.util.List;
+
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
@@ -38,24 +40,24 @@
public class LSMRTreeWithAntiMatterTuplesDataflowHelper extends AbstractLSMRTreeDataflowHelper {
public LSMRTreeWithAntiMatterTuplesDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition, IVirtualBufferCache virtualBufferCache, IBinaryComparatorFactory[] btreeComparatorFactories,
+ int partition, List<IVirtualBufferCache> virtualBufferCaches, IBinaryComparatorFactory[] btreeComparatorFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
ILinearizeComparatorFactory linearizeCmpFactory) {
- super(opDesc, ctx, partition, virtualBufferCache, btreeComparatorFactories, valueProviderFactories,
+ super(opDesc, ctx, partition, virtualBufferCaches, btreeComparatorFactories, valueProviderFactories,
rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackProvider, linearizeCmpFactory);
}
@Override
- protected ITreeIndex createLSMTree(IVirtualBufferCache virtualBufferCache, FileReference file,
+ protected ITreeIndex createLSMTree(List<IVirtualBufferCache> virtualBufferCaches, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
ILSMOperationTracker opTracker, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory)
throws HyracksDataException {
try {
- return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(virtualBufferCache, file, diskBufferCache,
+ return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(virtualBufferCaches, file, diskBufferCache,
diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories,
rtreePolicyType, mergePolicy, opTracker, ioScheduler, ioOpCallbackProvider, linearizeCmpFactory);
} catch (TreeIndexException e) {
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java
index 5fca08f..00268b3 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java
@@ -63,7 +63,7 @@
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition) {
return new LSMRTreeWithAntiMatterTuplesDataflowHelper(opDesc, ctx, partition,
- virtualBufferCacheProvider.getVirtualBufferCache(ctx), btreeComparatorFactories,
+ virtualBufferCacheProvider.getVirtualBufferCaches(ctx), btreeComparatorFactories,
valueProviderFactories, rtreePolicyType, mergePolicyProvider.getMergePolicy(ctx), opTrackerProvider,
ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackProvider, linearizeCmpFactory);
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index 31a2773..5ee6cb0 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -16,6 +16,7 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
import java.io.File;
+import java.util.ArrayList;
import java.util.List;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -24,17 +25,16 @@
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.IVirtualFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
-import edu.uci.ics.hyracks.storage.am.common.exceptions.TreeIndexDuplicateKeyException;
import edu.uci.ics.hyracks.storage.am.common.exceptions.TreeIndexNonExistentKeyException;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
@@ -47,6 +47,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.IMutableComponentSwitcherCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
@@ -56,7 +57,6 @@
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree.RTreeAccessor;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@@ -67,9 +67,7 @@
protected final IBinaryComparatorFactory[] linearizerArray;
// In-memory components.
- protected final LSMRTreeMutableComponent mutableComponent;
- protected final IVirtualBufferCache virtualBufferCache;
- protected final IVirtualFreePageManager virtualFreePageManager;
+ protected final List<LSMRTreeMutableComponent> mutableComponents;
protected TreeTupleSorter rTreeTupleSorter;
@@ -86,27 +84,53 @@
protected final ITreeIndexFrameFactory rtreeLeafFrameFactory;
protected final ITreeIndexFrameFactory btreeLeafFrameFactory;
- public AbstractLSMRTree(IVirtualBufferCache virtualBufferCache, ITreeIndexFrameFactory rtreeInteriorFrameFactory,
- ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory,
- ITreeIndexFrameFactory btreeLeafFrameFactory, ILSMIndexFileManager fileManager,
- TreeIndexFactory<RTree> diskRTreeFactory, ILSMComponentFactory componentFactory,
- IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories,
- IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
- int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, double bloomFilterFalsePositiveRate,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
- super(virtualBufferCache, diskRTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
+ public AbstractLSMRTree(List<IVirtualBufferCache> virtualBufferCaches,
+ ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
+ ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
+ ILSMIndexFileManager fileManager, TreeIndexFactory<RTree> diskRTreeFactory,
+ ILSMComponentFactory componentFactory, IFileMapProvider diskFileMapProvider, int fieldCount,
+ IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
+ ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
+ super(virtualBufferCaches, diskRTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallbackProvider);
- virtualFreePageManager = new VirtualFreePageManager(virtualBufferCache.getNumPages());
- RTree memRTree = new RTree(virtualBufferCache, ((IVirtualBufferCache) virtualBufferCache).getFileMapProvider(),
- virtualFreePageManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories,
- fieldCount, new FileReference(new File(fileManager.getBaseDir() + "_virtual_r")));
- BTree memBTree = new BTree(virtualBufferCache, ((IVirtualBufferCache) virtualBufferCache).getFileMapProvider(),
- new VirtualFreePageManager(virtualBufferCache.getNumPages()), btreeInteriorFrameFactory,
- btreeLeafFrameFactory, btreeCmpFactories, fieldCount, new FileReference(new File(
- fileManager.getBaseDir() + "_virtual_b")));
- mutableComponent = new LSMRTreeMutableComponent(memRTree, memBTree, virtualBufferCache);
- this.virtualBufferCache = virtualBufferCache;
+
+ mutableComponents = new ArrayList<LSMRTreeMutableComponent>();
+ final int numMutableComponents = virtualBufferCaches.size();
+ int i = 0;
+ for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
+ RTree memRTree = new RTree(virtualBufferCache,
+ ((IVirtualBufferCache) virtualBufferCache).getFileMapProvider(), new VirtualFreePageManager(
+ virtualBufferCache.getNumPages()), rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
+ rtreeCmpFactories, fieldCount, new FileReference(new File(fileManager.getBaseDir() + "_virtual_r_"
+ + i)));
+ BTree memBTree = new BTree(virtualBufferCache,
+ ((IVirtualBufferCache) virtualBufferCache).getFileMapProvider(), new VirtualFreePageManager(
+ virtualBufferCache.getNumPages()), btreeInteriorFrameFactory, btreeLeafFrameFactory,
+ btreeCmpFactories, fieldCount, new FileReference(new File(fileManager.getBaseDir() + "_virtual_b_"
+ + i)));
+ LSMRTreeMutableComponent mutableComponent = new LSMRTreeMutableComponent(memRTree, memBTree,
+ virtualBufferCache, i == 0 ? true : false);
+
+ mutableComponent.registerOnFlushCallback(new IMutableComponentSwitcherCallback() {
+
+ @Override
+ public void switchComponents() throws HyracksDataException {
+ currentMutableComponentId.set((currentMutableComponentId.get() + 1) % numMutableComponents);
+ mutableComponents.get(currentMutableComponentId.get()).setActive();
+ }
+
+ @Override
+ public void setFlushStatus(boolean isFlushNeeded) {
+ needsFlush[currentMutableComponentId.get()].set(isFlushNeeded);
+ }
+ });
+
+ mutableComponents.add(mutableComponent);
+ ++i;
+ }
+
this.rtreeInteriorFrameFactory = rtreeInteriorFrameFactory;
this.rtreeLeafFrameFactory = rtreeLeafFrameFactory;
this.btreeInteriorFrameFactory = btreeInteriorFrameFactory;
@@ -137,11 +161,13 @@
throw new HyracksDataException("Failed to activate the index since it is already activated.");
}
- ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).open();
- mutableComponent.getRTree().create();
- mutableComponent.getBTree().create();
- mutableComponent.getRTree().activate();
- mutableComponent.getBTree().activate();
+ for (LSMRTreeMutableComponent mutableComponent : mutableComponents) {
+ ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).open();
+ mutableComponent.getRTree().create();
+ mutableComponent.getBTree().create();
+ mutableComponent.getRTree().activate();
+ mutableComponent.getBTree().activate();
+ }
}
@Override
@@ -163,11 +189,13 @@
}
}
- mutableComponent.getRTree().deactivate();
- mutableComponent.getBTree().deactivate();
- mutableComponent.getRTree().destroy();
- mutableComponent.getBTree().destroy();
- ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).close();
+ for (LSMRTreeMutableComponent mutableComponent : mutableComponents) {
+ mutableComponent.getRTree().deactivate();
+ mutableComponent.getBTree().deactivate();
+ mutableComponent.getRTree().destroy();
+ mutableComponent.getBTree().destroy();
+ ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).close();
+ }
}
@Override
@@ -183,14 +211,11 @@
throw new HyracksDataException("Failed to clear the index since it is not activated.");
}
- mutableComponent.getRTree().clear();
- mutableComponent.getBTree().clear();
- mutableComponent.reset();
- }
-
- @Override
- public ILSMComponent getMutableComponent() {
- return mutableComponent;
+ for (LSMRTreeMutableComponent mutableComponent : mutableComponents) {
+ mutableComponent.getRTree().clear();
+ mutableComponent.getBTree().clear();
+ mutableComponent.reset();
+ }
}
@Override
@@ -198,14 +223,26 @@
List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
operationalComponents.clear();
List<ILSMComponent> immutableComponents = componentsRef.get();
+ int cmc = currentMutableComponentId.get();
+ ctx.setCurrentMutableComponentId(cmc);
+ int numMutableComponents = mutableComponents.size();
switch (ctx.getOperation()) {
case INSERT:
case DELETE:
case FLUSH:
- operationalComponents.add(mutableComponent);
+ operationalComponents.add(mutableComponents.get(cmc));
break;
case SEARCH:
- operationalComponents.add(mutableComponent);
+ for (int i = 0; i < numMutableComponents - 1; i++) {
+ LSMRTreeMutableComponent mutableComponent = mutableComponents.get((cmc + i + 1)
+ % numMutableComponents);
+ if (mutableComponent.isReadable()) {
+ // Make sure newest components are added first
+ operationalComponents.add(0, mutableComponent);
+ }
+ }
+ // The current mutable component is always added
+ operationalComponents.add(0, mutableComponents.get(cmc));
operationalComponents.addAll(immutableComponents);
break;
case MERGE:
@@ -216,6 +253,19 @@
}
}
+ @Override
+ public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred)
+ throws HyracksDataException, IndexException {
+ LSMRTreeOpContext ctx = (LSMRTreeOpContext) ictx;
+ List<ILSMComponent> operationalComponents = ictx.getComponentHolder();
+
+ LSMRTreeCursorInitialState initialState = new LSMRTreeCursorInitialState(rtreeLeafFrameFactory,
+ rtreeInteriorFrameFactory, btreeLeafFrameFactory, ctx.getBTreeMultiComparator(), lsmHarness,
+ comparatorFields, linearizerArray, ctx.searchCallback, operationalComponents);
+
+ cursor.open(initialState, pred);
+ }
+
protected LSMComponentFileReferences getMergeTargetFileName(List<ILSMComponent> mergingDiskComponents)
throws HyracksDataException {
RTree firstTree = ((LSMRTreeImmutableComponent) mergingDiskComponents.get(0)).getRTree();
@@ -253,32 +303,32 @@
@Override
public ITreeIndexFrameFactory getLeafFrameFactory() {
- return mutableComponent.getRTree().getLeafFrameFactory();
+ return mutableComponents.get(currentMutableComponentId.get()).getRTree().getLeafFrameFactory();
}
@Override
public ITreeIndexFrameFactory getInteriorFrameFactory() {
- return mutableComponent.getRTree().getInteriorFrameFactory();
+ return mutableComponents.get(currentMutableComponentId.get()).getRTree().getInteriorFrameFactory();
}
@Override
public IFreePageManager getFreePageManager() {
- return mutableComponent.getRTree().getFreePageManager();
+ return mutableComponents.get(currentMutableComponentId.get()).getRTree().getFreePageManager();
}
@Override
public int getFieldCount() {
- return mutableComponent.getRTree().getFieldCount();
+ return mutableComponents.get(currentMutableComponentId.get()).getRTree().getFieldCount();
}
@Override
public int getRootPageId() {
- return mutableComponent.getRTree().getRootPageId();
+ return mutableComponents.get(currentMutableComponentId.get()).getRTree().getRootPageId();
}
@Override
public int getFileId() {
- return mutableComponent.getRTree().getFileId();
+ return mutableComponents.get(currentMutableComponentId.get()).getRTree().getFileId();
}
@Override
@@ -298,8 +348,8 @@
// added RTree tuple.
RangePredicate btreeRangePredicate = new RangePredicate(tuple, tuple, true, true,
ctx.getBTreeMultiComparator(), ctx.getBTreeMultiComparator());
- ITreeIndexCursor cursor = ctx.memBTreeAccessor.createSearchCursor();
- ctx.memBTreeAccessor.search(cursor, btreeRangePredicate);
+ ITreeIndexCursor cursor = ctx.currentMutableBTreeAccessor.createSearchCursor();
+ ctx.currentMutableBTreeAccessor.search(cursor, btreeRangePredicate);
boolean foundTupleInMemoryBTree = false;
try {
if (cursor.hasNext()) {
@@ -310,7 +360,7 @@
}
if (foundTupleInMemoryBTree) {
try {
- ctx.memBTreeAccessor.delete(tuple);
+ ctx.currentMutableBTreeAccessor.delete(tuple);
} catch (TreeIndexNonExistentKeyException e) {
// Tuple has been deleted in the meantime. Do nothing.
// This normally shouldn't happen if we are dealing with
@@ -319,31 +369,25 @@
// insert between them.
}
} else {
- ctx.memRTreeAccessor.insert(tuple);
+ ctx.currentMutableRTreeAccessor.insert(tuple);
}
} else {
try {
- ctx.memBTreeAccessor.insert(tuple);
- } catch (TreeIndexDuplicateKeyException e) {
+ ctx.currentMutableBTreeAccessor.insert(tuple);
+ } catch (TreeIndexNonExistentKeyException e) {
// Do nothing, because one delete tuple is enough to indicate
// that all the corresponding insert tuples are deleted
}
}
- mutableComponent.setIsModified();
+ mutableComponents.get(currentMutableComponentId.get()).setIsModified();
}
protected LSMRTreeOpContext createOpContext(IModificationOperationCallback modCallback) {
- RTreeAccessor rtreeAccessor = (RTree.RTreeAccessor) mutableComponent.getRTree().createAccessor(
- NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- BTreeAccessor btreeAccessor = (BTree.BTreeAccessor) mutableComponent.getBTree().createAccessor(
- NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
-
- return new LSMRTreeOpContext(rtreeAccessor, (IRTreeLeafFrame) rtreeLeafFrameFactory.createFrame(),
- (IRTreeInteriorFrame) rtreeInteriorFrameFactory.createFrame(), virtualFreePageManager
- .getMetaDataFrameFactory().createFrame(), 4, btreeAccessor, btreeLeafFrameFactory,
- btreeInteriorFrameFactory, virtualFreePageManager.getMetaDataFrameFactory().createFrame(),
- rtreeCmpFactories, btreeCmpFactories, modCallback, NoOpOperationCallback.INSTANCE);
+ return new LSMRTreeOpContext(mutableComponents, (IRTreeLeafFrame) rtreeLeafFrameFactory.createFrame(),
+ (IRTreeInteriorFrame) rtreeInteriorFrameFactory.createFrame(), btreeLeafFrameFactory,
+ btreeInteriorFrameFactory, rtreeCmpFactories, btreeCmpFactories, modCallback,
+ NoOpOperationCallback.INSTANCE);
}
@Override
@@ -352,7 +396,14 @@
}
public boolean isEmptyIndex() throws HyracksDataException {
- return componentsRef.get().isEmpty() && !mutableComponent.isModified();
+ boolean isModified = false;
+ for (LSMRTreeMutableComponent mutableComponent : mutableComponents) {
+ if (mutableComponent.isModified()) {
+ isModified = true;
+ break;
+ }
+ }
+ return componentsRef.get().isEmpty() && !isModified;
}
@Override
@@ -362,8 +413,12 @@
@Override
public long getMemoryAllocationSize() {
- IBufferCache virtualBufferCache = mutableComponent.getRTree().getBufferCache();
- return virtualBufferCache.getNumPages() * virtualBufferCache.getPageSize();
+ long size = 0;
+ for (LSMRTreeMutableComponent mutableComponent : mutableComponents) {
+ IBufferCache virtualBufferCache = mutableComponent.getRTree().getBufferCache();
+ size += virtualBufferCache.getNumPages() * virtualBufferCache.getPageSize();
+ }
+ return size;
}
@Override
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 0b71373..ef91e7a 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -16,7 +16,6 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
import java.util.List;
-import java.util.ListIterator;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
@@ -55,6 +54,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractMutableLSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
@@ -66,7 +66,7 @@
public class LSMRTree extends AbstractLSMRTree {
- public LSMRTree(IVirtualBufferCache virtualBufferCache, ITreeIndexFrameFactory rtreeInteriorFrameFactory,
+ public LSMRTree(List<IVirtualBufferCache> virtualBufferCaches, ITreeIndexFrameFactory rtreeInteriorFrameFactory,
ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory,
ITreeIndexFrameFactory btreeLeafFrameFactory, ILSMIndexFileManager fileNameManager,
TreeIndexFactory<RTree> diskRTreeFactory, TreeIndexFactory<BTree> diskBTreeFactory,
@@ -76,7 +76,7 @@
int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMMergePolicy mergePolicy,
ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
- super(virtualBufferCache, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory,
+ super(virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory,
btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, new LSMRTreeComponentFactory(
diskRTreeFactory, diskBTreeFactory, bloomFilterFactory), diskFileMapProvider, fieldCount,
rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields, linearizerArray,
@@ -168,51 +168,14 @@
}
@Override
- public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred)
- throws HyracksDataException, IndexException {
- LSMRTreeOpContext ctx = (LSMRTreeOpContext) ictx;
- List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
- boolean includeMutableComponent = operationalComponents.get(0) == mutableComponent;
- int numTrees = operationalComponents.size();
-
- ListIterator<ILSMComponent> diskComponentIter = operationalComponents.listIterator();
- ITreeIndexAccessor[] rTreeAccessors = new ITreeIndexAccessor[numTrees];
- ITreeIndexAccessor[] bTreeAccessors = new ITreeIndexAccessor[numTrees];
- int diskComponentIx = 0;
- if (includeMutableComponent) {
- rTreeAccessors[0] = ctx.memRTreeAccessor;
- bTreeAccessors[0] = ctx.memBTreeAccessor;
- diskComponentIx++;
- diskComponentIter.next();
- }
-
- while (diskComponentIter.hasNext()) {
- LSMRTreeImmutableComponent component = (LSMRTreeImmutableComponent) diskComponentIter.next();
- RTree diskRTree = component.getRTree();
- BTree diskBTree = component.getBTree();
- rTreeAccessors[diskComponentIx] = diskRTree.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
- bTreeAccessors[diskComponentIx] = diskBTree.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
- diskComponentIx++;
- }
-
- LSMRTreeCursorInitialState initialState = new LSMRTreeCursorInitialState(numTrees, rtreeLeafFrameFactory,
- rtreeInteriorFrameFactory, btreeLeafFrameFactory, ctx.getBTreeMultiComparator(), rTreeAccessors,
- bTreeAccessors, includeMutableComponent, lsmHarness, comparatorFields, linearizerArray,
- ctx.searchCallback, operationalComponents);
- cursor.open(initialState, pred);
- }
-
- @Override
public boolean scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException {
- if (!mutableComponent.isModified()) {
+ ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
+ if (!((AbstractMutableLSMComponent) flushingComponent).isModified()) {
return false;
}
LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
ILSMIndexOperationContext rctx = createOpContext(NoOpOperationCallback.INSTANCE);
- LSMRTreeMutableComponent flushingComponent = (LSMRTreeMutableComponent) ctx.getComponentHolder().get(0);
rctx.setOperation(IndexOperation.FLUSH);
rctx.getComponentHolder().addAll(ctx.getComponentHolder());
LSMRTreeAccessor accessor = new LSMRTreeAccessor(lsmHarness, rctx);
@@ -255,6 +218,9 @@
while (rtreeScanCursor.hasNext()) {
isEmpty = false;
rtreeScanCursor.next();
+ if (rtreeScanCursor.getPageId() == 0) {
+ System.out.println();
+ }
rTreeTupleSorter.insertTupleEntry(rtreeScanCursor.getPageId(), rtreeScanCursor.getTupleOffset());
}
} finally {
@@ -329,9 +295,6 @@
@Override
public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException, IndexException {
- // Renaming order is critical because we use assume ordering when we
- // read the file names when we open the tree.
- // The RTree should be renamed before the BTree.
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
ILSMIndexOperationContext rctx = createOpContext(NoOpOperationCallback.INSTANCE);
rctx.getComponentHolder().addAll(mergingComponents);
@@ -389,7 +352,7 @@
public MultiComparator getMultiComparator() {
LSMRTreeOpContext concreteCtx = (LSMRTreeOpContext) ctx;
- return concreteCtx.rtreeOpContext.cmp;
+ return concreteCtx.currentRTreeOpContext.cmp;
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
index 0cd578a..279b777 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
@@ -12,146 +12,155 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
-import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
-import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
-import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BloomFilterAwareBTreePointSearchCursor;
-import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
-import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeSearchCursor;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.SearchPredicate;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
-
-public abstract class LSMRTreeAbstractCursor implements ITreeIndexCursor {
-
- protected RTreeSearchCursor[] rtreeCursors;
- protected boolean open = false;
- protected ITreeIndexCursor[] btreeCursors;
- protected ITreeIndexAccessor[] rTreeAccessors;
- protected ITreeIndexAccessor[] bTreeAccessors;
- private MultiComparator btreeCmp;
- protected int numberOfTrees;
- protected SearchPredicate rtreeSearchPredicate;
- protected RangePredicate btreeRangePredicate;
- protected ITupleReference frameTuple;
- protected boolean includeMemRTree;
- protected ILSMHarness lsmHarness;
- protected boolean foundNext;
- protected final ILSMIndexOperationContext opCtx;
-
- protected List<ILSMComponent> operationalComponents;
-
- public LSMRTreeAbstractCursor(ILSMIndexOperationContext opCtx) {
- super();
- this.opCtx = opCtx;
- }
-
- public RTreeSearchCursor getCursor(int cursorIndex) {
- return rtreeCursors[cursorIndex];
- }
-
- @Override
- public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
- LSMRTreeCursorInitialState lsmInitialState = (LSMRTreeCursorInitialState) initialState;
- btreeCmp = lsmInitialState.getBTreeCmp();
- includeMemRTree = lsmInitialState.getIncludeMemComponent();
- operationalComponents = lsmInitialState.getOperationalComponents();
- lsmHarness = lsmInitialState.getLSMHarness();
- numberOfTrees = lsmInitialState.getNumberOfTrees();
- rTreeAccessors = lsmInitialState.getRTreeAccessors();
- bTreeAccessors = lsmInitialState.getBTreeAccessors();
-
- rtreeCursors = new RTreeSearchCursor[numberOfTrees];
- btreeCursors = new ITreeIndexCursor[numberOfTrees];
-
- int i = 0;
- if (includeMemRTree) {
- rtreeCursors[i] = new RTreeSearchCursor((IRTreeInteriorFrame) lsmInitialState
- .getRTreeInteriorFrameFactory().createFrame(), (IRTreeLeafFrame) lsmInitialState
- .getRTreeLeafFrameFactory().createFrame());
-
- // No need for a bloom filter for the in-memory BTree.
- btreeCursors[i] = new BTreeRangeSearchCursor((IBTreeLeafFrame) lsmInitialState.getBTreeLeafFrameFactory()
- .createFrame(), false);
- ++i;
- }
- for (; i < numberOfTrees; i++) {
- rtreeCursors[i] = new RTreeSearchCursor((IRTreeInteriorFrame) lsmInitialState
- .getRTreeInteriorFrameFactory().createFrame(), (IRTreeLeafFrame) lsmInitialState
- .getRTreeLeafFrameFactory().createFrame());
-
- btreeCursors[i] = new BloomFilterAwareBTreePointSearchCursor((IBTreeLeafFrame) lsmInitialState
- .getBTreeLeafFrameFactory().createFrame(), false,
- ((LSMRTreeImmutableComponent) operationalComponents.get(i)).getBloomFilter());
- }
-
- rtreeSearchPredicate = (SearchPredicate) searchPred;
- btreeRangePredicate = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp);
-
- open = true;
- }
-
- @Override
- public ICachedPage getPage() {
- // do nothing
- return null;
- }
-
- @Override
- public void close() throws HyracksDataException {
- if (!open) {
- return;
- }
-
- try {
- if (rtreeCursors != null && btreeCursors != null) {
- for (int i = 0; i < numberOfTrees; i++) {
- rtreeCursors[i].close();
- btreeCursors[i].close();
- }
- }
- rtreeCursors = null;
- btreeCursors = null;
- } finally {
- lsmHarness.endSearch(opCtx);
- }
-
- open = false;
- }
-
- @Override
- public void setBufferCache(IBufferCache bufferCache) {
- // do nothing
- }
-
- @Override
- public void setFileId(int fileId) {
- // do nothing
- }
-
- @Override
- public ITupleReference getTuple() {
- return frameTuple;
- }
-
- @Override
- public boolean exclusiveLatchNodes() {
- return false;
- }
-
+package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BloomFilterAwareBTreePointSearchCursor;
+import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
+import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.SearchPredicate;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+
+public abstract class LSMRTreeAbstractCursor implements ITreeIndexCursor {
+
+ protected boolean open;
+ protected RTreeSearchCursor[] rtreeCursors;
+ protected ITreeIndexCursor[] btreeCursors;
+ protected ITreeIndexAccessor[] rtreeAccessors;
+ protected ITreeIndexAccessor[] btreeAccessors;
+ private MultiComparator btreeCmp;
+ protected int numberOfTrees;
+ protected SearchPredicate rtreeSearchPredicate;
+ protected RangePredicate btreeRangePredicate;
+ protected ITupleReference frameTuple;
+ protected boolean includeMutableComponent;
+ protected ILSMHarness lsmHarness;
+ protected boolean foundNext;
+ protected final ILSMIndexOperationContext opCtx;
+
+ protected List<ILSMComponent> operationalComponents;
+
+ public LSMRTreeAbstractCursor(ILSMIndexOperationContext opCtx) {
+ super();
+ this.opCtx = opCtx;
+ }
+
+ public RTreeSearchCursor getCursor(int cursorIndex) {
+ return rtreeCursors[cursorIndex];
+ }
+
+ @Override
+ public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ LSMRTreeCursorInitialState lsmInitialState = (LSMRTreeCursorInitialState) initialState;
+ btreeCmp = lsmInitialState.getBTreeCmp();
+
+ operationalComponents = lsmInitialState.getOperationalComponents();
+ lsmHarness = lsmInitialState.getLSMHarness();
+ numberOfTrees = operationalComponents.size();
+
+ rtreeCursors = new RTreeSearchCursor[numberOfTrees];
+ btreeCursors = new ITreeIndexCursor[numberOfTrees];
+ rtreeAccessors = new ITreeIndexAccessor[numberOfTrees];
+ btreeAccessors = new ITreeIndexAccessor[numberOfTrees];
+
+ includeMutableComponent = false;
+ for (int i = 0; i < numberOfTrees; i++) {
+ ILSMComponent component = operationalComponents.get(i);
+ RTree rtree;
+ BTree btree;
+ if (component.getType() == LSMComponentType.MEMORY) {
+ includeMutableComponent = true;
+ // No need for a bloom filter for the in-memory BTree.
+ btreeCursors[i] = new BTreeRangeSearchCursor((IBTreeLeafFrame) lsmInitialState
+ .getBTreeLeafFrameFactory().createFrame(), false);
+ rtree = (RTree) ((LSMRTreeMutableComponent) component).getRTree();
+ btree = (BTree) ((LSMRTreeMutableComponent) component).getBTree();
+ } else {
+ btreeCursors[i] = new BloomFilterAwareBTreePointSearchCursor((IBTreeLeafFrame) lsmInitialState
+ .getBTreeLeafFrameFactory().createFrame(), false,
+ ((LSMRTreeImmutableComponent) operationalComponents.get(i)).getBloomFilter());
+ rtree = (RTree) ((LSMRTreeImmutableComponent) component).getRTree();
+ btree = (BTree) ((LSMRTreeImmutableComponent) component).getBTree();
+ }
+ rtreeCursors[i] = new RTreeSearchCursor((IRTreeInteriorFrame) lsmInitialState
+ .getRTreeInteriorFrameFactory().createFrame(), (IRTreeLeafFrame) lsmInitialState
+ .getRTreeLeafFrameFactory().createFrame());
+ rtreeAccessors[i] = rtree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ btreeAccessors[i] = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ }
+
+ rtreeSearchPredicate = (SearchPredicate) searchPred;
+ btreeRangePredicate = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp);
+
+ open = true;
+ }
+
+ @Override
+ public ICachedPage getPage() {
+ // do nothing
+ return null;
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ if (!open) {
+ return;
+ }
+
+ try {
+ if (rtreeCursors != null && btreeCursors != null) {
+ for (int i = 0; i < numberOfTrees; i++) {
+ rtreeCursors[i].close();
+ btreeCursors[i].close();
+ }
+ }
+ rtreeCursors = null;
+ btreeCursors = null;
+ } finally {
+ lsmHarness.endSearch(opCtx);
+ }
+
+ open = false;
+ }
+
+ @Override
+ public void setBufferCache(IBufferCache bufferCache) {
+ // do nothing
+ }
+
+ @Override
+ public void setFileId(int fileId) {
+ // do nothing
+ }
+
+ @Override
+ public ITupleReference getTuple() {
+ return frameTuple;
+ }
+
+ @Override
+ public boolean exclusiveLatchNodes() {
+ return false;
+ }
+
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeCursorInitialState.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeCursorInitialState.java
index 21230b0..27545b91 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeCursorInitialState.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeCursorInitialState.java
@@ -20,7 +20,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
@@ -29,35 +28,26 @@
public class LSMRTreeCursorInitialState implements ICursorInitialState {
- private final int numberOfTrees;
private final ITreeIndexFrameFactory rtreeInteriorFrameFactory;
private final ITreeIndexFrameFactory rtreeLeafFrameFactory;
private final ITreeIndexFrameFactory btreeLeafFrameFactory;
private final MultiComparator btreeCmp;
private final MultiComparator hilbertCmp;
- private final ITreeIndexAccessor[] rTreeAccessors;
- private final ITreeIndexAccessor[] bTreeAccessors;
- private final boolean includeMemRTree;
private final ILSMHarness lsmHarness;
private final int[] comparatorFields;
private ISearchOperationCallback searchCallback;
private final List<ILSMComponent> operationalComponents;
- public LSMRTreeCursorInitialState(int numberOfTrees, ITreeIndexFrameFactory rtreeLeafFrameFactory,
+ public LSMRTreeCursorInitialState(ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
- MultiComparator btreeCmp, ITreeIndexAccessor[] rTreeAccessors, ITreeIndexAccessor[] bTreeAccessors,
- boolean includeMemRTree, ILSMHarness lsmHarness, int[] comparatorFields,
+ MultiComparator btreeCmp, ILSMHarness lsmHarness, int[] comparatorFields,
IBinaryComparatorFactory[] linearizerArray, ISearchOperationCallback searchCallback,
List<ILSMComponent> operationalComponents) {
- this.numberOfTrees = numberOfTrees;
this.rtreeLeafFrameFactory = rtreeLeafFrameFactory;
this.rtreeInteriorFrameFactory = rtreeInteriorFrameFactory;
this.btreeLeafFrameFactory = btreeLeafFrameFactory;
this.btreeCmp = btreeCmp;
- this.rTreeAccessors = rTreeAccessors;
- this.bTreeAccessors = bTreeAccessors;
- this.includeMemRTree = includeMemRTree;
this.lsmHarness = lsmHarness;
this.comparatorFields = comparatorFields;
this.hilbertCmp = MultiComparator.create(linearizerArray);
@@ -73,10 +63,6 @@
return comparatorFields;
}
- public int getNumberOfTrees() {
- return numberOfTrees;
- }
-
public ITreeIndexFrameFactory getRTreeInteriorFrameFactory() {
return rtreeInteriorFrameFactory;
}
@@ -106,18 +92,6 @@
return operationalComponents;
}
- public ITreeIndexAccessor[] getRTreeAccessors() {
- return rTreeAccessors;
- }
-
- public ITreeIndexAccessor[] getBTreeAccessors() {
- return bTreeAccessors;
- }
-
- public boolean getIncludeMemComponent() {
- return includeMemRTree;
- }
-
public ILSMHarness getLSMHarness() {
return lsmHarness;
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMutableComponent.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMutableComponent.java
index ffe583c..e960ee0 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMutableComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMutableComponent.java
@@ -26,8 +26,8 @@
private final RTree rtree;
private final BTree btree;
- public LSMRTreeMutableComponent(RTree rtree, BTree btree, IVirtualBufferCache vbc) {
- super(vbc);
+ public LSMRTreeMutableComponent(RTree rtree, BTree btree, IVirtualBufferCache vbc, boolean isActive) {
+ super(vbc, isActive);
this.rtree = rtree;
this.btree = btree;
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
index 8607a05..28b1572 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
@@ -24,7 +24,6 @@
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
@@ -37,43 +36,68 @@
public final class LSMRTreeOpContext implements ILSMIndexOperationContext {
- public RTreeOpContext rtreeOpContext;
- public BTreeOpContext btreeOpContext;
- public final RTree.RTreeAccessor memRTreeAccessor;
- public final BTree.BTreeAccessor memBTreeAccessor;
+ public RTree.RTreeAccessor[] mutableRTreeAccessors;
+ public RTree.RTreeAccessor currentMutableRTreeAccessor;
+ public BTree.BTreeAccessor[] mutableBTreeAccessors;
+ public BTree.BTreeAccessor currentMutableBTreeAccessor;
+
+ public RTreeOpContext[] rtreeOpContexts;
+ public BTreeOpContext[] btreeOpContexts;
+ public RTreeOpContext currentRTreeOpContext;
+ public BTreeOpContext currentBTreeOpContext;
+
private IndexOperation op;
public final List<ILSMComponent> componentHolder;
public final IModificationOperationCallback modificationCallback;
public final ISearchOperationCallback searchCallback;
- public LSMRTreeOpContext(RTree.RTreeAccessor memRtreeAccessor, IRTreeLeafFrame rtreeLeafFrame,
- IRTreeInteriorFrame rtreeInteriorFrame, ITreeIndexMetaDataFrame rtreeMetaFrame, int rTreeHeightHint,
- BTree.BTreeAccessor memBtreeAccessor, ITreeIndexFrameFactory btreeLeafFrameFactory,
- ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexMetaDataFrame btreeMetaFrame,
- IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
- IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) {
- this.memRTreeAccessor = memRtreeAccessor;
- this.memBTreeAccessor = memBtreeAccessor;
+ public LSMRTreeOpContext(List<LSMRTreeMutableComponent> mutableComponents, IRTreeLeafFrame rtreeLeafFrame,
+ IRTreeInteriorFrame rtreeInteriorFrame, ITreeIndexFrameFactory btreeLeafFrameFactory,
+ ITreeIndexFrameFactory btreeInteriorFrameFactory, IBinaryComparatorFactory[] rtreeCmpFactories,
+ IBinaryComparatorFactory[] btreeCmpFactories, IModificationOperationCallback modificationCallback,
+ ISearchOperationCallback searchCallback) {
+ mutableRTreeAccessors = new RTree.RTreeAccessor[mutableComponents.size()];
+ mutableBTreeAccessors = new BTree.BTreeAccessor[mutableComponents.size()];
+ rtreeOpContexts = new RTreeOpContext[mutableComponents.size()];
+ btreeOpContexts = new BTreeOpContext[mutableComponents.size()];
+
+ for (int i = 0; i < mutableComponents.size(); i++) {
+ mutableRTreeAccessors[i] = (RTree.RTreeAccessor) mutableComponents.get(i).getRTree()
+ .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ mutableBTreeAccessors[i] = (BTree.BTreeAccessor) mutableComponents.get(i).getBTree()
+ .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+
+ rtreeOpContexts[i] = mutableRTreeAccessors[i].getOpContext();
+ btreeOpContexts[i] = mutableBTreeAccessors[i].getOpContext();
+ }
+
+ assert mutableComponents.size() > 0;
+ currentRTreeOpContext = rtreeOpContexts[0];
+ currentBTreeOpContext = btreeOpContexts[0];
this.componentHolder = new LinkedList<ILSMComponent>();
this.modificationCallback = modificationCallback;
this.searchCallback = searchCallback;
- this.rtreeOpContext = new RTreeOpContext(rtreeLeafFrame, rtreeInteriorFrame, rtreeMetaFrame, rtreeCmpFactories,
- rTreeHeightHint, NoOpOperationCallback.INSTANCE);
- this.btreeOpContext = new BTreeOpContext(memBtreeAccessor, btreeLeafFrameFactory, btreeInteriorFrameFactory,
- btreeMetaFrame, btreeCmpFactories, NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
}
public void setOperation(IndexOperation newOp) {
reset();
- if (newOp == IndexOperation.INSERT) {
- rtreeOpContext.setOperation(newOp);
- } else if (newOp == IndexOperation.DELETE) {
- btreeOpContext.setOperation(IndexOperation.INSERT);
- }
this.op = newOp;
}
@Override
+ public void setCurrentMutableComponentId(int currentMutableComponentId) {
+ currentMutableRTreeAccessor = mutableRTreeAccessors[currentMutableComponentId];
+ currentMutableBTreeAccessor = mutableBTreeAccessors[currentMutableComponentId];
+ currentRTreeOpContext = rtreeOpContexts[currentMutableComponentId];
+ currentBTreeOpContext = btreeOpContexts[currentMutableComponentId];
+ if (op == IndexOperation.INSERT) {
+ currentRTreeOpContext.setOperation(op);
+ } else if (op == IndexOperation.DELETE) {
+ currentBTreeOpContext.setOperation(IndexOperation.INSERT);
+ }
+ }
+
+ @Override
public void reset() {
componentHolder.clear();
}
@@ -84,7 +108,7 @@
}
public MultiComparator getBTreeMultiComparator() {
- return btreeOpContext.cmp;
+ return currentBTreeOpContext.cmp;
}
@Override
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java
index 9b0869c..30dd467 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java
@@ -61,7 +61,7 @@
if (currentCursor < numberOfTrees) {
rtreeCursors[currentCursor].reset();
try {
- rTreeAccessors[currentCursor].search(rtreeCursors[currentCursor], rtreeSearchPredicate);
+ rtreeAccessors[currentCursor].search(rtreeCursors[currentCursor], rtreeSearchPredicate);
} catch (IndexException e) {
throw new HyracksDataException(e);
}
@@ -80,14 +80,10 @@
boolean killerTupleFound = false;
for (int i = 0; i <= currentCursor; i++) {
- try {
- btreeCursors[i].reset();
- btreeRangePredicate.setHighKey(currentTuple, true);
- btreeRangePredicate.setLowKey(currentTuple, true);
- bTreeAccessors[i].search(btreeCursors[i], btreeRangePredicate);
- } catch (IndexException e) {
- throw new HyracksDataException(e);
- }
+ btreeCursors[i].reset();
+ btreeRangePredicate.setHighKey(currentTuple, true);
+ btreeRangePredicate.setLowKey(currentTuple, true);
+ btreeAccessors[i].search(btreeCursors[i], btreeRangePredicate);
try {
if (btreeCursors[i].hasNext()) {
killerTupleFound = true;
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java
index 31dfe27..b93eb1b 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java
@@ -44,7 +44,7 @@
for (int i = 0; i < numberOfTrees; i++) {
rtreeCursors[i].reset();
try {
- rTreeAccessors[i].search(rtreeCursors[i], rtreeSearchPredicate);
+ rtreeAccessors[i].search(rtreeCursors[i], rtreeSearchPredicate);
} catch (IndexException e) {
throw new HyracksDataException(e);
}
@@ -103,7 +103,7 @@
btreeCursors[i].reset();
btreeRangePredicate.setHighKey(frameTuple, true);
btreeRangePredicate.setLowKey(frameTuple, true);
- bTreeAccessors[i].search(btreeCursors[i], btreeRangePredicate);
+ btreeAccessors[i].search(btreeCursors[i], btreeRangePredicate);
} catch (IndexException e) {
throw new HyracksDataException(e);
}
@@ -138,7 +138,7 @@
for (int i = 0; i < numberOfTrees; i++) {
rtreeCursors[i].reset();
try {
- rTreeAccessors[i].search(rtreeCursors[i], rtreeSearchPredicate);
+ rtreeAccessors[i].search(rtreeCursors[i], rtreeSearchPredicate);
} catch (IndexException e) {
throw new HyracksDataException(e);
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index 0be497a..6ccd681 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -16,7 +16,6 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
import java.util.List;
-import java.util.ListIterator;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
@@ -25,7 +24,6 @@
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
@@ -50,6 +48,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractMutableLSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
@@ -67,7 +66,7 @@
// because it should have a different tuple writer in it's leaf frames.
private final ILSMComponentFactory bulkLoaComponentFactory;
- public LSMRTreeWithAntiMatterTuples(IVirtualBufferCache virtualBufferCache,
+ public LSMRTreeWithAntiMatterTuples(List<IVirtualBufferCache> virtualBufferCaches,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
ILSMIndexFileManager fileManager, TreeIndexFactory<RTree> diskRTreeFactory,
@@ -76,7 +75,7 @@
ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
- super(virtualBufferCache, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory,
+ super(virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory,
btreeLeafFrameFactory, fileManager, diskRTreeFactory, new LSMRTreeWithAntiMatterTuplesComponentFactory(
diskRTreeFactory), diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories,
linearizer, comparatorFields, linearizerArray, 0, mergePolicy, opTracker, ioScheduler,
@@ -149,59 +148,14 @@
}
@Override
- public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred)
- throws HyracksDataException, IndexException {
- LSMRTreeOpContext ctx = (LSMRTreeOpContext) ictx;
- List<ILSMComponent> operationalComponents = ictx.getComponentHolder();
- boolean includeMutableComponent = operationalComponents.get(0) == mutableComponent;
- LSMRTreeWithAntiMatterTuplesSearchCursor lsmTreeCursor = (LSMRTreeWithAntiMatterTuplesSearchCursor) cursor;
- int numDiskRComponents = operationalComponents.size();
-
- LSMRTreeCursorInitialState initialState;
- ITreeIndexAccessor[] bTreeAccessors = null;
- if (includeMutableComponent) {
- // Only in-memory BTree
- bTreeAccessors = new ITreeIndexAccessor[1];
- bTreeAccessors[0] = ctx.memBTreeAccessor;
- }
-
- initialState = new LSMRTreeCursorInitialState(numDiskRComponents, rtreeLeafFrameFactory,
- rtreeInteriorFrameFactory, btreeLeafFrameFactory, ctx.getBTreeMultiComparator(), null, bTreeAccessors,
- includeMutableComponent, lsmHarness, comparatorFields, linearizerArray, ctx.searchCallback,
- operationalComponents);
-
- lsmTreeCursor.open(initialState, pred);
-
- ListIterator<ILSMComponent> diskComponentsIter = operationalComponents.listIterator();
- int diskComponentIx = 0;
- if (includeMutableComponent) {
- // Open cursor of in-memory RTree
- ctx.memRTreeAccessor.search(lsmTreeCursor.getMemRTreeCursor(), pred);
- diskComponentIx++;
- diskComponentsIter.next();
- }
-
- // Open cursors of on-disk RTrees.
- ITreeIndexAccessor[] diskRTreeAccessors = new ITreeIndexAccessor[numDiskRComponents];
- while (diskComponentsIter.hasNext()) {
- RTree diskRTree = (RTree) ((LSMRTreeImmutableComponent) diskComponentsIter.next()).getRTree();
- diskRTreeAccessors[diskComponentIx] = diskRTree.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
- diskRTreeAccessors[diskComponentIx].search(lsmTreeCursor.getCursor(diskComponentIx), pred);
- diskComponentIx++;
- }
- lsmTreeCursor.initPriorityQueue();
- }
-
- @Override
public boolean scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException {
- if (!mutableComponent.isModified()) {
+ ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
+ if (!((AbstractMutableLSMComponent) flushingComponent).isModified()) {
return false;
}
LSMRTreeOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE);
LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference();
- ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
opCtx.setOperation(IndexOperation.FLUSH);
opCtx.getComponentHolder().add(flushingComponent);
ILSMIndexAccessorInternal accessor = new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness, opCtx);
@@ -358,7 +312,7 @@
public MultiComparator getMultiComparator() {
LSMRTreeOpContext concreteCtx = (LSMRTreeOpContext) ctx;
- return concreteCtx.rtreeOpContext.cmp;
+ return concreteCtx.currentRTreeOpContext.cmp;
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java
index c937709..fbf0445 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java
@@ -18,6 +18,7 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
@@ -25,74 +26,133 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.SearchPredicate;
public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCursor {
- private RTreeSearchCursor memRTreeCursor;
- private BTreeRangeSearchCursor memBTreeCursor;
+ private ITreeIndexAccessor[] mutableRTreeAccessors;
+ private ITreeIndexAccessor[] btreeAccessors;
+ private RTreeSearchCursor[] mutableRTreeCursors;
+ private ITreeIndexCursor[] btreeCursors;
private RangePredicate btreeRangePredicate;
- private ITreeIndexAccessor memBTreeAccessor;
private boolean foundNext;
private ITupleReference frameTuple;
private int[] comparatorFields;
private MultiComparator btreeCmp;
+ private int currentCursor;
+ private SearchPredicate rtreeSearchPredicate;
+ private int numMutableComponents;
public LSMRTreeWithAntiMatterTuplesSearchCursor(ILSMIndexOperationContext opCtx) {
super(opCtx);
+ currentCursor = 0;
}
@Override
- public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException,
+ IndexException {
LSMRTreeCursorInitialState lsmInitialState = (LSMRTreeCursorInitialState) initialState;
cmp = lsmInitialState.getHilbertCmp();
btreeCmp = lsmInitialState.getBTreeCmp();
- int numDiskRTrees = lsmInitialState.getNumberOfTrees();
- rangeCursors = new RTreeSearchCursor[numDiskRTrees];
- for (int i = 0; i < numDiskRTrees; i++) {
- rangeCursors[i] = new RTreeSearchCursor((IRTreeInteriorFrame) lsmInitialState
- .getRTreeInteriorFrameFactory().createFrame(), (IRTreeLeafFrame) lsmInitialState
- .getRTreeLeafFrameFactory().createFrame());
- }
- includeMemComponent = lsmInitialState.getIncludeMemComponent();
- operationalComponents = lsmInitialState.getOperationalComponents();
- if (includeMemComponent) {
- memRTreeCursor = new RTreeSearchCursor((IRTreeInteriorFrame) lsmInitialState.getRTreeInteriorFrameFactory()
- .createFrame(), (IRTreeLeafFrame) lsmInitialState.getRTreeLeafFrameFactory().createFrame());
- memBTreeCursor = new BTreeRangeSearchCursor((IBTreeLeafFrame) lsmInitialState.getBTreeLeafFrameFactory()
- .createFrame(), false);
- memBTreeAccessor = lsmInitialState.getBTreeAccessors()[0];
- btreeRangePredicate = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp);
- }
lsmHarness = lsmInitialState.getLSMHarness();
comparatorFields = lsmInitialState.getComparatorFields();
+ operationalComponents = lsmInitialState.getOperationalComponents();
+ rtreeSearchPredicate = (SearchPredicate) searchPred;
+
+ includeMutableComponent = false;
+ numMutableComponents = 0;
+ int numImmutableComponents = 0;
+ for (ILSMComponent component : operationalComponents) {
+ if (component.getType() == LSMComponentType.MEMORY) {
+ includeMutableComponent = true;
+ numMutableComponents++;
+ } else {
+ numImmutableComponents++;
+ }
+ }
+ if (includeMutableComponent) {
+ btreeRangePredicate = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp);
+ }
+
+ mutableRTreeCursors = new RTreeSearchCursor[numMutableComponents];
+ mutableRTreeAccessors = new ITreeIndexAccessor[numMutableComponents];
+ btreeCursors = new BTreeRangeSearchCursor[numMutableComponents];
+ btreeAccessors = new ITreeIndexAccessor[numMutableComponents];
+ for (int i = 0; i < numMutableComponents; i++) {
+ ILSMComponent component = operationalComponents.get(i);
+ RTree rtree = (RTree) ((LSMRTreeMutableComponent) component).getRTree();
+ BTree btree = (BTree) ((LSMRTreeMutableComponent) component).getBTree();
+ mutableRTreeCursors[i] = new RTreeSearchCursor((IRTreeInteriorFrame) lsmInitialState
+ .getRTreeInteriorFrameFactory().createFrame(), (IRTreeLeafFrame) lsmInitialState
+ .getRTreeLeafFrameFactory().createFrame());
+ btreeCursors[i] = new BTreeRangeSearchCursor((IBTreeLeafFrame) lsmInitialState.getBTreeLeafFrameFactory()
+ .createFrame(), false);
+ btreeAccessors[i] = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ mutableRTreeAccessors[i] = rtree.createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ }
+
+ rangeCursors = new RTreeSearchCursor[numImmutableComponents];
+ ITreeIndexAccessor[] immutableRTreeAccessors = new ITreeIndexAccessor[numImmutableComponents];
+ int j = 0;
+ for (int i = numMutableComponents; i < operationalComponents.size(); i++) {
+ ILSMComponent component = operationalComponents.get(i);
+ rangeCursors[j] = new RTreeSearchCursor((IRTreeInteriorFrame) lsmInitialState
+ .getRTreeInteriorFrameFactory().createFrame(), (IRTreeLeafFrame) lsmInitialState
+ .getRTreeLeafFrameFactory().createFrame());
+ RTree rtree = (RTree) ((LSMRTreeImmutableComponent) component).getRTree();
+ immutableRTreeAccessors[j] = rtree.createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ immutableRTreeAccessors[j].search(rangeCursors[j], searchPred);
+ j++;
+ }
+ searchNextCursor();
setPriorityQueueComparator();
+ initPriorityQueue();
+ }
+
+ private void searchNextCursor() throws HyracksDataException, IndexException {
+ if (currentCursor < numMutableComponents) {
+ mutableRTreeCursors[currentCursor].reset();
+ mutableRTreeAccessors[currentCursor].search(mutableRTreeCursors[currentCursor], rtreeSearchPredicate);
+ }
}
@Override
public boolean hasNext() throws HyracksDataException, IndexException {
- if (includeMemComponent) {
+ if (includeMutableComponent) {
if (foundNext) {
return true;
}
- while (memRTreeCursor.hasNext()) {
- memRTreeCursor.next();
- ITupleReference memRTreeTuple = memRTreeCursor.getTuple();
- if (searchMemBTree(memRTreeTuple)) {
- foundNext = true;
- frameTuple = memRTreeTuple;
- return true;
+
+ while (currentCursor < numMutableComponents) {
+ while (mutableRTreeCursors[currentCursor].hasNext()) {
+ mutableRTreeCursors[currentCursor].next();
+ ITupleReference currentTuple = mutableRTreeCursors[currentCursor].getTuple();
+ if (searchMemBTrees(currentTuple, currentCursor)) {
+ foundNext = true;
+ frameTuple = currentTuple;
+ return true;
+ }
}
+ mutableRTreeCursors[currentCursor].close();
+ currentCursor++;
+ searchNextCursor();
}
while (super.hasNext()) {
super.next();
ITupleReference diskRTreeTuple = super.getTuple();
- if (searchMemBTree(diskRTreeTuple)) {
+ if (searchMemBTrees(diskRTreeTuple, numMutableComponents - 1)) {
foundNext = true;
frameTuple = diskRTreeTuple;
return true;
@@ -107,7 +167,7 @@
@Override
public void next() throws HyracksDataException {
- if (includeMemComponent) {
+ if (includeMutableComponent) {
foundNext = false;
} else {
super.next();
@@ -117,7 +177,7 @@
@Override
public ITupleReference getTuple() {
- if (includeMemComponent) {
+ if (includeMutableComponent) {
return frameTuple;
} else {
return super.getTuple();
@@ -127,48 +187,49 @@
@Override
public void reset() throws HyracksDataException, IndexException {
- if (includeMemComponent) {
- memRTreeCursor.reset();
- memBTreeCursor.reset();
+ if (includeMutableComponent) {
+ for (int i = 0; i < numMutableComponents; i++) {
+ mutableRTreeCursors[i].reset();
+ btreeCursors[i].reset();
+ }
}
+ currentCursor = 0;
super.reset();
}
@Override
public void close() throws HyracksDataException {
- if (includeMemComponent) {
- memRTreeCursor.close();
- memBTreeCursor.close();
+ if (includeMutableComponent) {
+ for (int i = 0; i < numMutableComponents; i++) {
+ mutableRTreeCursors[i].close();
+ btreeCursors[i].close();
+ }
}
+ currentCursor = 0;
super.close();
}
- public ITreeIndexCursor getMemRTreeCursor() {
- return memRTreeCursor;
- }
-
@Override
protected int compare(MultiComparator cmp, ITupleReference tupleA, ITupleReference tupleB) {
return cmp.selectiveFieldCompare(tupleA, tupleB, comparatorFields);
}
- private boolean searchMemBTree(ITupleReference tuple) throws HyracksDataException {
- try {
+ private boolean searchMemBTrees(ITupleReference tuple, int lastBTreeToSearch) throws HyracksDataException,
+ IndexException {
+ for (int i = 0; i <= lastBTreeToSearch; i++) {
+ btreeCursors[i].reset();
btreeRangePredicate.setHighKey(tuple, true);
btreeRangePredicate.setLowKey(tuple, true);
- memBTreeAccessor.search(memBTreeCursor, btreeRangePredicate);
- } catch (IndexException e) {
- throw new HyracksDataException(e);
- }
- try {
- if (memBTreeCursor.hasNext()) {
- return false;
- } else {
- return true;
+ btreeAccessors[i].search(btreeCursors[i], btreeRangePredicate);
+ try {
+ if (btreeCursors[i].hasNext()) {
+ return false;
+ }
+ } finally {
+ btreeCursors[i].close();
}
- } finally {
- memBTreeCursor.close();
}
+ return true;
}
@Override
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
index 50e5d09..b16d2b6 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
@@ -15,6 +15,8 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.utils;
+import java.util.List;
+
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
@@ -58,7 +60,7 @@
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class LSMRTreeUtils {
- public static LSMRTree createLSMTree(IVirtualBufferCache virtualBufferCache, FileReference file,
+ public static LSMRTree createLSMTree(List<IVirtualBufferCache> virtualBufferCaches, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
@@ -99,7 +101,7 @@
ILSMIndexFileManager fileNameManager = new LSMRTreeFileManager(diskFileMapProvider, file, diskRTreeFactory,
diskBTreeFactory);
- LSMRTree lsmTree = new LSMRTree(virtualBufferCache, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
+ LSMRTree lsmTree = new LSMRTree(virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, diskBTreeFactory,
bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length,
rtreeCmpFactories, btreeCmpFactories, linearizeCmpFactory, comparatorFields, linearizerArray,
@@ -108,7 +110,7 @@
}
public static LSMRTreeWithAntiMatterTuples createLSMTreeWithAntiMatterTuples(
- IVirtualBufferCache virtualBufferCache, FileReference file, IBufferCache diskBufferCache,
+ List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
@@ -151,7 +153,7 @@
ILSMIndexFileManager fileNameManager = new LSMRTreeWithAntiMatterTuplesFileManager(diskFileMapProvider, file,
diskRTreeFactory);
- LSMRTreeWithAntiMatterTuples lsmTree = new LSMRTreeWithAntiMatterTuples(virtualBufferCache,
+ LSMRTreeWithAntiMatterTuples lsmTree = new LSMRTreeWithAntiMatterTuples(virtualBufferCaches,
rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory,
fileNameManager, diskRTreeFactory, bulkLoadRTreeFactory, diskFileMapProvider, typeTraits.length,
rtreeCmpFactories, btreeCmpFactories, linearizerCmpFactory, comparatorFields, linearizerArray,
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
index 528c004..7bb3583 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
@@ -147,7 +147,7 @@
private RTreeOpContext createOpContext(IModificationOperationCallback modificationCallback) {
return new RTreeOpContext((IRTreeLeafFrame) leafFrameFactory.createFrame(),
(IRTreeInteriorFrame) interiorFrameFactory.createFrame(), freePageManager.getMetaDataFrameFactory()
- .createFrame(), cmpFactories, 8, modificationCallback);
+ .createFrame(), cmpFactories, modificationCallback);
}
private void insert(ITupleReference tuple, IIndexOperationContext ictx) throws HyracksDataException,
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java
index da73f85..a9171c3 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java
@@ -31,6 +31,7 @@
public class RTreeOpContext implements IIndexOperationContext {
private static final int INITIAL_TRAVERSE_LIST_SIZE = 100;
+ private static final int INITIAL_HEIGHT = 8;
public final MultiComparator cmp;
public final IRTreeInteriorFrame interiorFrame;
public final IRTreeLeafFrame leafFrame;
@@ -51,20 +52,20 @@
public final IModificationOperationCallback modificationCallback;
public RTreeOpContext(IRTreeLeafFrame leafFrame, IRTreeInteriorFrame interiorFrame,
- ITreeIndexMetaDataFrame metaFrame, IBinaryComparatorFactory[] cmpFactories, int treeHeightHint,
+ ITreeIndexMetaDataFrame metaFrame, IBinaryComparatorFactory[] cmpFactories,
IModificationOperationCallback modificationCallback) {
-
- if (cmpFactories[0] != null) {
+
+ if (cmpFactories[0] != null) {
this.cmp = MultiComparator.create(cmpFactories);
} else {
this.cmp = null;
}
-
+
this.interiorFrame = interiorFrame;
this.leafFrame = leafFrame;
this.metaFrame = metaFrame;
this.modificationCallback = modificationCallback;
- pathList = new PathList(treeHeightHint, treeHeightHint);
+ pathList = new PathList(INITIAL_HEIGHT, INITIAL_HEIGHT);
NSNUpdates = new ArrayList<ICachedPage>();
LSNUpdates = new ArrayList<ICachedPage>();
}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java
index 693dc47..52f1a9f 100644
--- a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java
@@ -29,6 +29,7 @@
public static final int LSM_RTREE_BULKLOAD_ROUNDS = 5;
public static final int LSM_RTREE_MAX_TREES_TO_MERGE = 3;
public static final boolean LSM_RTREE_TEST_RSTAR_POLICY = false;
+ public static final int LSM_RTREE_NUM_MUTABLE_COMPONENTS = 2;
// Test params for BTree, LSMBTree.
public static final int BTREE_NUM_TUPLES_TO_INSERT = 100;
@@ -39,17 +40,18 @@
// Test params for LSMBTree only.
public static final int LSM_BTREE_BULKLOAD_ROUNDS = 5;
public static final int LSM_BTREE_MAX_TREES_TO_MERGE = 10;
+ public static final int LSM_BTREE_NUM_MUTABLE_COMPONENTS = 2;
// Mem configuration for RTree.
public static final int RTREE_PAGE_SIZE = 512;
public static final int RTREE_NUM_PAGES = 1000;
- public static final int RTREE_MAX_OPEN_FILES = 10;
+ public static final int RTREE_MAX_OPEN_FILES = Integer.MAX_VALUE;
public static final int RTREE_HYRACKS_FRAME_SIZE = 128;
// Mem configuration for LSMRTree and LSMRTreeWithAntiMatterTuples.
public static final int LSM_RTREE_DISK_PAGE_SIZE = 512;
public static final int LSM_RTREE_DISK_NUM_PAGES = 1000;
- public static final int LSM_RTREE_DISK_MAX_OPEN_FILES = 2000;
+ public static final int LSM_RTREE_DISK_MAX_OPEN_FILES = Integer.MAX_VALUE;
public static final int LSM_RTREE_MEM_PAGE_SIZE = 512;
public static final int LSM_RTREE_MEM_NUM_PAGES = 1000;
public static final int LSM_RTREE_HYRACKS_FRAME_SIZE = 128;
@@ -58,13 +60,13 @@
// Mem configuration for BTree.
public static final int BTREE_PAGE_SIZE = 256;
public static final int BTREE_NUM_PAGES = 100;
- public static final int BTREE_MAX_OPEN_FILES = 10;
+ public static final int BTREE_MAX_OPEN_FILES = Integer.MAX_VALUE;
public static final int BTREE_HYRACKS_FRAME_SIZE = 128;
// Mem configuration for LSMBTree.
public static final int LSM_BTREE_DISK_PAGE_SIZE = 256;
public static final int LSM_BTREE_DISK_NUM_PAGES = 1000;
- public static final int LSM_BTREE_DISK_MAX_OPEN_FILES = 200;
+ public static final int LSM_BTREE_DISK_MAX_OPEN_FILES = Integer.MAX_VALUE;
public static final int LSM_BTREE_MEM_PAGE_SIZE = 256;
public static final int LSM_BTREE_MEM_NUM_PAGES = 100;
public static final int LSM_BTREE_HYRACKS_FRAME_SIZE = 128;
@@ -73,11 +75,13 @@
// Mem configuration for Inverted Index.
public static final int LSM_INVINDEX_DISK_PAGE_SIZE = 1024;
public static final int LSM_INVINDEX_DISK_NUM_PAGES = 1000;
- public static final int LSM_INVINDEX_DISK_MAX_OPEN_FILES = 1000;
+ public static final int LSM_INVINDEX_DISK_MAX_OPEN_FILES = Integer.MAX_VALUE;
public static final int LSM_INVINDEX_MEM_PAGE_SIZE = 1024;
public static final int LSM_INVINDEX_MEM_NUM_PAGES = 100;
public static final int LSM_INVINDEX_HYRACKS_FRAME_SIZE = 32768;
public static final double LSM_INVINDEX_BLOOMFILTER_FALSE_POSITIVE_RATE = 0.01;
+ public static final int LSM_INVINDEX_NUM_MUTABLE_COMPONENTS = 2;
+
// Test parameters.
public static final int LSM_INVINDEX_NUM_DOCS_TO_INSERT = 100;
// Used for full-fledged search test.
@@ -100,7 +104,7 @@
// Mem configuration for BloomFilter.
public static final int BLOOM_FILTER_PAGE_SIZE = 256;
public static final int BLOOM_FILTER_NUM_PAGES = 1000;
- public static final int BLOOM_FILTER_MAX_OPEN_FILES = 10;
+ public static final int BLOOM_FILTER_MAX_OPEN_FILES = Integer.MAX_VALUE;
public static final int BLOOM_FILTER_HYRACKS_FRAME_SIZE = 128;
}
@@ -129,13 +133,13 @@
// Mem configuration for RTree.
public static final int RTREE_PAGE_SIZE = 512;
public static final int RTREE_NUM_PAGES = 1000;
-public static final int RTREE_MAX_OPEN_FILES = 10;
+public static final int RTREE_MAX_OPEN_FILES = Integer.MAX_VALUE;
public static final int RTREE_HYRACKS_FRAME_SIZE = 128;
// Mem configuration for LSMRTree and LSMRTreeWithAntiMatterTuples.
public static final int LSM_RTREE_DISK_PAGE_SIZE = 256;
public static final int LSM_RTREE_DISK_NUM_PAGES = 1000;
-public static final int LSM_RTREE_DISK_MAX_OPEN_FILES = 2000;
+public static final int LSM_RTREE_DISK_MAX_OPEN_FILES = Integer.MAX_VALUE;
public static final int LSM_RTREE_MEM_PAGE_SIZE = 256;
public static final int LSM_RTREE_MEM_NUM_PAGES = 1000;
public static final int LSM_RTREE_HYRACKS_FRAME_SIZE = 128;
@@ -143,13 +147,13 @@
// Mem configuration for BTree.
public static final int BTREE_PAGE_SIZE = 256;
public static final int BTREE_NUM_PAGES = 100;
-public static final int BTREE_MAX_OPEN_FILES = 10;
+public static final int BTREE_MAX_OPEN_FILES = Integer.MAX_VALUE;
public static final int BTREE_HYRACKS_FRAME_SIZE = 128;
// Mem configuration for LSMBTree.
public static final int LSM_BTREE_DISK_PAGE_SIZE = 256;
public static final int LSM_BTREE_DISK_NUM_PAGES = 1000;
-public static final int LSM_BTREE_DISK_MAX_OPEN_FILES = 200;
+public static final int LSM_BTREE_DISK_MAX_OPEN_FILES = Integer.MAX_VALUE;
public static final int LSM_BTREE_MEM_PAGE_SIZE = 256;
public static final int LSM_BTREE_MEM_NUM_PAGES = 100;
public static final int LSM_BTREE_HYRACKS_FRAME_SIZE = 128;
@@ -157,13 +161,13 @@
// Mem configuration for Inverted Index.
public static final int INVINDEX_PAGE_SIZE = 32768;
public static final int INVINDEX_NUM_PAGES = 100;
-public static final int INVINDEX_MAX_OPEN_FILES = 10;
+public static final int INVINDEX_MAX_OPEN_FILES = Integer.MAX_VALUE;
public static final int INVINDEX_HYRACKS_FRAME_SIZE = 32768;
// Mem configuration for Inverted Index.
public static final int LSM_INVINDEX_DISK_PAGE_SIZE = 1024;
public static final int LSM_INVINDEX_DISK_NUM_PAGES = 1000;
-public static final int LSM_INVINDEX_DISK_MAX_OPEN_FILES = 1000;
+public static final int LSM_INVINDEX_DISK_MAX_OPEN_FILES = Integer.MAX_VALUE;
public static final int LSM_INVINDEX_MEM_PAGE_SIZE = 1024;
public static final int LSM_INVINDEX_MEM_NUM_PAGES = 100;
public static final int LSM_INVINDEX_HYRACKS_FRAME_SIZE = 32768;
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java
index 44ab0d9..705a976 100644
--- a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java
@@ -134,12 +134,6 @@
TupleUtils.createIntegerTuple(tb, tuple, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x),
Math.max(p1y, p2y), pk1, pk2);
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("Inserting " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
- + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y) + ", " + pk1 + ", " + pk2);
- }
- }
try {
indexAccessor.insert(tuple);
} catch (TreeIndexException e) {
@@ -494,13 +488,6 @@
TupleUtils.createDoubleTuple(tb, tuple, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.min(p1z, p2z),
Math.max(p1x, p2x), Math.max(p1y, p2y), Math.max(p1z, p2z), pk);
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("Inserting " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
- + Math.min(p1z, p2z) + " " + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y) + " "
- + Math.max(p1z, p2z) + ", " + pk);
- }
- }
try {
indexAccessor.insert(tuple);
} catch (TreeIndexException e) {
@@ -608,11 +595,6 @@
TupleUtils.createIntegerTuple(tb, tuple, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x),
Math.max(p1y, p2y), pk);
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("Inserting " + i);
- }
- }
try {
indexAccessor.insert(tuple);
} catch (TreeIndexException e) {
@@ -626,11 +608,6 @@
int delDone = 0;
for (int i = 0; i < numInserts; i++) {
TupleUtils.createIntegerTuple(tb, tuple, p1xs[i], p1ys[i], p2xs[i], p2ys[i], pks[i]);
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("Deleting " + i);
- }
- }
try {
indexAccessor.delete(tuple);
delDone++;
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestVirtualBufferCacheProvider.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestVirtualBufferCacheProvider.java
index 9db21cb..afbb86b 100644
--- a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestVirtualBufferCacheProvider.java
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestVirtualBufferCacheProvider.java
@@ -14,6 +14,9 @@
*/
package edu.uci.ics.hyracks.test.support;
+import java.util.ArrayList;
+import java.util.List;
+
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCacheProvider;
@@ -33,7 +36,12 @@
}
@Override
- public IVirtualBufferCache getVirtualBufferCache(IHyracksTaskContext ctx) {
- return new VirtualBufferCache(new HeapBufferAllocator(), pageSize, numPages);
+ public List<IVirtualBufferCache> getVirtualBufferCaches(IHyracksTaskContext ctx) {
+ List<IVirtualBufferCache> vbcs = new ArrayList<IVirtualBufferCache>();
+ for (int i = 0; i < 2; i++) {
+ IVirtualBufferCache vbc = new VirtualBufferCache(new HeapBufferAllocator(), pageSize, numPages / 2);
+ vbcs.add(vbc);
+ }
+ return vbcs;
}
}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
index e0a2a46..753be5d 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
@@ -51,7 +51,7 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
- return LSMBTreeTestContext.create(harness.getVirtualBufferCache(), harness.getFileReference(),
+ return LSMBTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
index 122b788..646343a 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
@@ -51,7 +51,7 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
- return LSMBTreeTestContext.create(harness.getVirtualBufferCache(), harness.getFileReference(),
+ return LSMBTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
index 60e0492..b0617a1 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
@@ -34,7 +34,7 @@
@Override
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
int[] bloomFilterKeyFields) throws TreeIndexException {
- return LSMBTreeUtils.createLSMTree(harness.getVirtualBufferCache(), harness.getFileReference(),
+ return LSMBTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, cmpFactories,
bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
index 76a0206..bce69a2 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
@@ -51,7 +51,7 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
- return LSMBTreeTestContext.create(harness.getVirtualBufferCache(), harness.getFileReference(),
+ return LSMBTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
index 639fd47..0f903e5 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
@@ -53,7 +53,7 @@
@Override
public void setup() throws Exception {
harness.setUp();
- testCtx = LSMBTreeTestContext.create(harness.getVirtualBufferCache(), harness.getFileReference(),
+ testCtx = LSMBTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, fieldSerdes.length,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
index e3c86df..ab46f3f 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
@@ -50,7 +50,7 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
- return LSMBTreeTestContext.create(harness.getVirtualBufferCache(), harness.getFileReference(),
+ return LSMBTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
index 77b65ca..cc3389f 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
@@ -42,7 +42,7 @@
@Override
protected void createIndexInstance() throws Exception {
- index = LSMBTreeUtils.createLSMTree(harness.getVirtualBufferCache(), harness.getFileReference(),
+ index = LSMBTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
SerdeUtils.serdesToTypeTraits(keySerdes),
SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), bloomFilterKeyFields,
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
index a7c8b81..d7d29a8 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
@@ -52,7 +52,7 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
- return LSMBTreeTestContext.create(harness.getVirtualBufferCache(), harness.getFileReference(),
+ return LSMBTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
index e15e80a..ce9ae0e 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
@@ -49,7 +49,7 @@
@Override
protected void createIndexInstance() throws Exception {
- index = LSMBTreeUtils.createLSMTree(harness.getVirtualBufferCache(), harness.getFileReference(),
+ index = LSMBTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
SerdeUtils.serdesToTypeTraits(keySerdes),
SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), bloomFilterKeyFields,
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
index db221a2..6f8ef84 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
@@ -51,7 +51,7 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
- return LSMBTreeTestContext.create(harness.getVirtualBufferCache(), harness.getFileReference(),
+ return LSMBTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
index 9dfa713..1ae201f 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
@@ -50,7 +50,7 @@
@Override
protected ITreeIndex createIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
int[] bloomFilterKeyFields) throws TreeIndexException {
- return LSMBTreeUtils.createLSMTree(harness.getVirtualBufferCache(), harness.getFileReference(),
+ return LSMBTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, cmpFactories,
bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
index fc6642c..3591f78a 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
@@ -17,7 +17,9 @@
import java.io.File;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Date;
+import java.util.List;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -88,10 +90,15 @@
ioDeviceId = 0;
IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- IVirtualBufferCache virtualBufferCache = new VirtualBufferCache(new HeapBufferAllocator(), inMemPageSize,
- inMemNumPages);
+ List<IVirtualBufferCache> virtualBufferCaches = new ArrayList<IVirtualBufferCache>();
+ for (int i = 0; i < 2; i++) {
+ IVirtualBufferCache virtualBufferCache = new VirtualBufferCache(new HeapBufferAllocator(), inMemPageSize,
+ inMemNumPages);
+ virtualBufferCaches.add(virtualBufferCache);
+ }
+
this.ioScheduler = SynchronousScheduler.INSTANCE;
- lsmtree = LSMBTreeUtils.createLSMTree(virtualBufferCache, file, bufferCache, fmp, typeTraits, cmpFactories,
+ lsmtree = LSMBTreeUtils.createLSMTree(virtualBufferCaches, file, bufferCache, fmp, typeTraits, cmpFactories,
bloomFilterKeyFields, bloomFilterFalsePositiveRate, NoMergePolicy.INSTANCE,
new ThreadCountingTracker(), ioScheduler, NoOpIOOperationCallback.INSTANCE);
}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
index ab78ee61..39e7184 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
@@ -16,6 +16,7 @@
package edu.uci.ics.hyracks.storage.am.lsm.btree.util;
import java.util.Collection;
+import java.util.List;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
@@ -62,7 +63,7 @@
upsertCheckTuple(checkTuple, checkTuples);
}
- public static LSMBTreeTestContext create(IVirtualBufferCache virtualBufferCache, FileReference file,
+ public static LSMBTreeTestContext create(List<IVirtualBufferCache> virtualBufferCaches, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ISerializerDeserializer[] fieldSerdes,
int numKeyFields, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
@@ -73,7 +74,7 @@
for (int i = 0; i < numKeyFields; ++i) {
bloomFilterKeyFields[i] = i;
}
- LSMBTree lsmTree = LSMBTreeUtils.createLSMTree(virtualBufferCache, file, diskBufferCache, diskFileMapProvider,
+ LSMBTree lsmTree = LSMBTreeUtils.createLSMTree(virtualBufferCaches, file, diskBufferCache, diskFileMapProvider,
typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
ioScheduler, ioOpCallbackProvider);
LSMBTreeTestContext testCtx = new LSMBTreeTestContext(fieldSerdes, lsmTree);
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
index e643cf9..216861f 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
@@ -18,7 +18,9 @@
import java.io.File;
import java.io.FilenameFilter;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Date;
+import java.util.List;
import java.util.Random;
import java.util.logging.Logger;
@@ -60,12 +62,13 @@
protected final int memNumPages;
protected final int hyracksFrameSize;
protected final double bloomFilterFalsePositiveRate;
+ protected final int numMutableComponents;
protected IOManager ioManager;
protected int ioDeviceId;
protected IBufferCache diskBufferCache;
protected IFileMapProvider diskFileMapProvider;
- protected IVirtualBufferCache virtualBufferCache;
+ protected List<IVirtualBufferCache> virtualBufferCaches;
protected IHyracksTaskContext ctx;
protected ILSMIOOperationScheduler ioScheduler;
protected ILSMMergePolicy mergePolicy;
@@ -90,20 +93,7 @@
this.mergePolicy = NoMergePolicy.INSTANCE;
this.opTracker = new ThreadCountingTracker();
this.ioOpCallbackProvider = NoOpIOOperationCallback.INSTANCE;
- }
-
- public LSMBTreeTestHarness(int diskPageSize, int diskNumPages, int diskMaxOpenFiles, int memPageSize,
- int memNumPages, int hyracksFrameSize, double bloomFilterFalsePositiveRate) {
- this.diskPageSize = diskPageSize;
- this.diskNumPages = diskNumPages;
- this.diskMaxOpenFiles = diskMaxOpenFiles;
- this.memPageSize = memPageSize;
- this.memNumPages = memNumPages;
- this.hyracksFrameSize = hyracksFrameSize;
- this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
- this.ioScheduler = SynchronousScheduler.INSTANCE;
- this.mergePolicy = NoMergePolicy.INSTANCE;
- this.opTracker = new ThreadCountingTracker();
+ this.numMutableComponents = AccessMethodTestsConfig.LSM_BTREE_NUM_MUTABLE_COMPONENTS;
}
public void setUp() throws HyracksException {
@@ -116,7 +106,12 @@
TestStorageManagerComponentHolder.init(diskPageSize, diskNumPages, diskMaxOpenFiles);
diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
diskFileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- virtualBufferCache = new VirtualBufferCache(new HeapBufferAllocator(), memPageSize, memNumPages);
+ virtualBufferCaches = new ArrayList<IVirtualBufferCache>();
+ for (int i = 0; i < numMutableComponents; i++) {
+ IVirtualBufferCache virtualBufferCache = new VirtualBufferCache(new HeapBufferAllocator(), memPageSize,
+ memNumPages / numMutableComponents);
+ virtualBufferCaches.add(virtualBufferCache);
+ }
rnd.setSeed(RANDOM_SEED);
}
@@ -179,8 +174,8 @@
return diskFileMapProvider;
}
- public IVirtualBufferCache getVirtualBufferCache() {
- return virtualBufferCache;
+ public List<IVirtualBufferCache> getVirtualBufferCaches() {
+ return virtualBufferCaches;
}
public double getBoomFilterFalsePositiveRate() {
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
index 7276464..99087f0 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
@@ -18,7 +18,9 @@
import java.io.File;
import java.io.FilenameFilter;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Date;
+import java.util.List;
import java.util.Random;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
@@ -27,15 +29,12 @@
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.control.nc.io.IOManager;
-import edu.uci.ics.hyracks.storage.am.common.api.IVirtualFreePageManager;
import edu.uci.ics.hyracks.storage.am.config.AccessMethodTestsConfig;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.MultitenantVirtualBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.SynchronousScheduler;
@@ -58,13 +57,13 @@
protected final int memNumPages;
protected final int hyracksFrameSize;
protected final double bloomFilterFalsePositiveRate;
+ protected final int numMutableComponents;
protected IOManager ioManager;
protected int ioDeviceId;
protected IBufferCache diskBufferCache;
protected IFileMapProvider diskFileMapProvider;
- protected IVirtualBufferCache virtualBufferCache;
- protected IVirtualFreePageManager virtualFreePageManager;
+ protected List<IVirtualBufferCache> virtualBufferCaches;
protected IHyracksTaskContext ctx;
protected ILSMIOOperationScheduler ioScheduler;
protected ILSMMergePolicy mergePolicy;
@@ -91,20 +90,7 @@
this.mergePolicy = NoMergePolicy.INSTANCE;
this.opTracker = new ThreadCountingTracker();
this.ioOpCallbackProvider = NoOpIOOperationCallback.INSTANCE;
- }
-
- public LSMInvertedIndexTestHarness(int diskPageSize, int diskNumPages, int diskMaxOpenFiles, int memPageSize,
- int memNumPages, int hyracksFrameSize, double bloomFilterFalsePositiveRate) {
- this.diskPageSize = diskPageSize;
- this.diskNumPages = diskNumPages;
- this.diskMaxOpenFiles = diskMaxOpenFiles;
- this.memPageSize = memPageSize;
- this.memNumPages = memNumPages;
- this.hyracksFrameSize = hyracksFrameSize;
- this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
- this.ioScheduler = SynchronousScheduler.INSTANCE;
- this.mergePolicy = NoMergePolicy.INSTANCE;
- this.opTracker = new ThreadCountingTracker();
+ this.numMutableComponents = AccessMethodTestsConfig.LSM_INVINDEX_NUM_MUTABLE_COMPONENTS;
}
public void setUp() throws HyracksException {
@@ -116,10 +102,12 @@
TestStorageManagerComponentHolder.init(diskPageSize, diskNumPages, diskMaxOpenFiles);
diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
diskFileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- virtualBufferCache = new MultitenantVirtualBufferCache(new VirtualBufferCache(new HeapBufferAllocator(),
- memPageSize, memNumPages));
- virtualBufferCache.open();
- virtualFreePageManager = new VirtualFreePageManager(memNumPages);
+ virtualBufferCaches = new ArrayList<IVirtualBufferCache>();
+ for (int i = 0; i < numMutableComponents; i++) {
+ IVirtualBufferCache virtualBufferCache = new VirtualBufferCache(new HeapBufferAllocator(), memPageSize,
+ memNumPages / numMutableComponents);
+ virtualBufferCaches.add(virtualBufferCache);
+ }
rnd.setSeed(RANDOM_SEED);
invIndexFileRef = ioManager.getIODevices().get(0).createFileReference(onDiskDir + invIndexFileName);
}
@@ -141,7 +129,6 @@
}
}
dir.delete();
- virtualBufferCache.close();
}
public FileReference getInvListsFileRef() {
@@ -188,18 +175,14 @@
return diskFileMapProvider;
}
- public IVirtualBufferCache getVirtualBufferCache() {
- return virtualBufferCache;
+ public List<IVirtualBufferCache> getVirtualBufferCaches() {
+ return virtualBufferCaches;
}
public double getBoomFilterFalsePositiveRate() {
return bloomFilterFalsePositiveRate;
}
- public IVirtualFreePageManager getVirtualFreePageManager() {
- return virtualFreePageManager;
- }
-
public IHyracksTaskContext getHyracksTastContext() {
return ctx;
}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
index 05dc1b5..5edf74e 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
@@ -36,6 +36,7 @@
import edu.uci.ics.hyracks.storage.am.common.CheckTuple;
import edu.uci.ics.hyracks.storage.am.common.api.IIndex;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.common.LSMInvertedIndexTestHarness;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.exceptions.InvertedIndexException;
@@ -118,18 +119,20 @@
}
// Create index and test context.
IInvertedIndex invIndex;
+ assert harness.getVirtualBufferCaches().size() > 0;
switch (invIndexType) {
case INMEMORY: {
- invIndex = InvertedIndexUtils.createInMemoryBTreeInvertedindex(harness.getVirtualBufferCache(),
- harness.getVirtualFreePageManager(), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
- tokenCmpFactories, tokenizerFactory, new FileReference(new File(harness.getOnDiskDir())));
+ invIndex = InvertedIndexUtils.createInMemoryBTreeInvertedindex(harness.getVirtualBufferCaches().get(0),
+ new VirtualFreePageManager(harness.getVirtualBufferCaches().get(0).getNumPages()),
+ invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
+ new FileReference(new File(harness.getOnDiskDir())));
break;
}
case PARTITIONED_INMEMORY: {
invIndex = InvertedIndexUtils.createPartitionedInMemoryBTreeInvertedindex(harness
- .getVirtualBufferCache(), harness.getVirtualFreePageManager(), invListTypeTraits,
- invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, new FileReference(
- new File(harness.getOnDiskDir())));
+ .getVirtualBufferCaches().get(0), new VirtualFreePageManager(harness.getVirtualBufferCaches()
+ .get(0).getNumPages()), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
+ tokenCmpFactories, tokenizerFactory, new FileReference(new File(harness.getOnDiskDir())));
break;
}
case ONDISK: {
@@ -145,7 +148,7 @@
break;
}
case LSM: {
- invIndex = InvertedIndexUtils.createLSMInvertedIndex(harness.getVirtualBufferCache(),
+ invIndex = InvertedIndexUtils.createLSMInvertedIndex(harness.getVirtualBufferCaches(),
harness.getDiskFileMapProvider(), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, tokenizerFactory, harness.getDiskBufferCache(), harness.getOnDiskDir(),
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
@@ -154,7 +157,7 @@
break;
}
case PARTITIONED_LSM: {
- invIndex = InvertedIndexUtils.createPartitionedLSMInvertedIndex(harness.getVirtualBufferCache(),
+ invIndex = InvertedIndexUtils.createPartitionedLSMInvertedIndex(harness.getVirtualBufferCaches(),
harness.getDiskFileMapProvider(), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, tokenizerFactory, harness.getDiskBufferCache(), harness.getOnDiskDir(),
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
index 3fcb4e5..09daaab 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
@@ -54,7 +54,7 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeTestContext.create(harness.getVirtualBufferCache(), harness.getFileReference(),
+ return LSMRTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
numKeys, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
index 467916e..f5b4ae4 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
@@ -54,7 +54,7 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeTestContext.create(harness.getVirtualBufferCache(), harness.getFileReference(),
+ return LSMRTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
numKeys, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
index a35170c..c94bb48 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
@@ -37,7 +37,7 @@
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType) throws TreeIndexException {
- return LSMRTreeUtils.createLSMTree(harness.getVirtualBufferCache(), harness.getFileReference(),
+ return LSMRTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, rtreeCmpFactories,
btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(),
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
index 44da0c0..858f90a 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
@@ -54,7 +54,7 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeTestContext.create(harness.getVirtualBufferCache(), harness.getFileReference(),
+ return LSMRTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
numKeys, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java
index b97bebb..61692b8 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java
@@ -63,7 +63,7 @@
@Override
public void setup() throws Exception {
harness.setUp();
- testCtx = LSMRTreeTestContext.create(harness.getVirtualBufferCache(), harness.getFileReference(),
+ testCtx = LSMRTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
numKeys, RTreePolicyType.RTREE, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
index b13ce4d..436f9b1 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
@@ -53,7 +53,7 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeTestContext.create(harness.getVirtualBufferCache(), harness.getFileReference(),
+ return LSMRTreeTestContext.create(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
numKeys, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
index 2baf485..97a3dcc 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
@@ -54,7 +54,7 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getVirtualBufferCache(),
+ return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getVirtualBufferCaches(),
harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType, harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
index 9c440cc..843dcc7 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
@@ -54,7 +54,7 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getVirtualBufferCache(),
+ return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getVirtualBufferCaches(),
harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType, harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
index 909d0cd..6ffa59a 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
@@ -37,7 +37,7 @@
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType) throws TreeIndexException {
- return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getVirtualBufferCache(),
+ return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getVirtualBufferCaches(),
harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits,
rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
index 6b35192..1e1f46d 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
@@ -54,7 +54,7 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getVirtualBufferCache(),
+ return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getVirtualBufferCaches(),
harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType, harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java
index 6dcbdf2..607e980 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java
@@ -63,7 +63,7 @@
@Override
public void setup() throws Exception {
harness.setUp();
- testCtx = LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getVirtualBufferCache(),
+ testCtx = LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getVirtualBufferCaches(),
harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
fieldSerdes, valueProviderFactories, numKeys, RTreePolicyType.RTREE, harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
index dc81fc3..bdd80aa 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
@@ -53,7 +53,7 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getVirtualBufferCache(),
+ return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getVirtualBufferCaches(),
harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType, harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
index 850d540..0c0d804 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
@@ -57,7 +57,7 @@
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType) throws TreeIndexException {
- return LSMRTreeUtils.createLSMTree(harness.getVirtualBufferCache(), harness.getFileReference(),
+ return LSMRTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, rtreeCmpFactories,
btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(),
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
index 4a6b462..8b54373 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
@@ -57,7 +57,7 @@
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType) throws TreeIndexException {
- return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getVirtualBufferCache(),
+ return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getVirtualBufferCaches(),
harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits,
rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
index 8c59e31..9536e96 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
@@ -16,6 +16,7 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.util;
import java.util.Collection;
+import java.util.List;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
@@ -66,7 +67,7 @@
return lsmTree.getComparatorFactories();
}
- public static LSMRTreeTestContext create(IVirtualBufferCache virtualBufferCache, FileReference file,
+ public static LSMRTreeTestContext create(List<IVirtualBufferCache> virtualBufferCaches, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeyFields, RTreePolicyType rtreePolicyType,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
@@ -77,7 +78,7 @@
.serdesToComparatorFactories(fieldSerdes, numKeyFields);
IBinaryComparatorFactory[] btreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes,
fieldSerdes.length);
- LSMRTree lsmTree = LSMRTreeUtils.createLSMTree(virtualBufferCache, file, diskBufferCache, diskFileMapProvider,
+ LSMRTree lsmTree = LSMRTreeUtils.createLSMTree(virtualBufferCaches, file, diskBufferCache, diskFileMapProvider,
typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallbackProvider,
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length));
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
index 118b1bc..b67274e 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
@@ -18,7 +18,9 @@
import java.io.File;
import java.io.FilenameFilter;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Date;
+import java.util.List;
import java.util.Random;
import java.util.logging.Logger;
@@ -57,12 +59,13 @@
protected final int memNumPages;
protected final int hyracksFrameSize;
protected final double bloomFilterFalsePositiveRate;
+ protected final int numMutableComponents;
protected IOManager ioManager;
protected int ioDeviceId;
protected IBufferCache diskBufferCache;
protected IFileMapProvider diskFileMapProvider;
- protected IVirtualBufferCache virtualBufferCache;
+ protected List<IVirtualBufferCache> virtualBufferCaches;
protected IHyracksTaskContext ctx;
protected ILSMIOOperationScheduler ioScheduler;
protected ILSMIOOperationCallbackProvider ioOpCallbackProvider;
@@ -87,20 +90,7 @@
this.mergePolicy = NoMergePolicy.INSTANCE;
this.opTracker = new ThreadCountingTracker();
this.ioOpCallbackProvider = NoOpIOOperationCallback.INSTANCE;
- }
-
- public LSMRTreeTestHarness(int diskPageSize, int diskNumPages, int diskMaxOpenFiles, int memPageSize,
- int memNumPages, int hyracksFrameSize, double bloomFilterFalsePositiveRate) {
- this.diskPageSize = diskPageSize;
- this.diskNumPages = diskNumPages;
- this.diskMaxOpenFiles = diskMaxOpenFiles;
- this.memPageSize = memPageSize;
- this.memNumPages = memNumPages;
- this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
- this.hyracksFrameSize = hyracksFrameSize;
- this.ioScheduler = SynchronousScheduler.INSTANCE;
- this.mergePolicy = NoMergePolicy.INSTANCE;
- this.opTracker = new ThreadCountingTracker();
+ this.numMutableComponents = AccessMethodTestsConfig.LSM_RTREE_NUM_MUTABLE_COMPONENTS;
}
public void setUp() throws HyracksException {
@@ -113,7 +103,12 @@
TestStorageManagerComponentHolder.init(diskPageSize, diskNumPages, diskMaxOpenFiles);
diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
diskFileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- virtualBufferCache = new VirtualBufferCache(new HeapBufferAllocator(), memPageSize, memNumPages);
+ virtualBufferCaches = new ArrayList<IVirtualBufferCache>();
+ for (int i = 0; i < numMutableComponents; i++) {
+ IVirtualBufferCache virtualBufferCache = new VirtualBufferCache(new HeapBufferAllocator(), memPageSize,
+ memNumPages / numMutableComponents);
+ virtualBufferCaches.add(virtualBufferCache);
+ }
rnd.setSeed(RANDOM_SEED);
}
@@ -175,8 +170,8 @@
return diskFileMapProvider;
}
- public IVirtualBufferCache getVirtualBufferCache() {
- return virtualBufferCache;
+ public List<IVirtualBufferCache> getVirtualBufferCaches() {
+ return virtualBufferCaches;
}
public double getBoomFilterFalsePositiveRate() {
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
index 95716f6..069524c 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
@@ -16,6 +16,7 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.util;
import java.util.Collection;
+import java.util.List;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
@@ -67,7 +68,7 @@
return lsmTree.getComparatorFactories();
}
- public static LSMRTreeWithAntiMatterTuplesTestContext create(IVirtualBufferCache virtualBufferCache,
+ public static LSMRTreeWithAntiMatterTuplesTestContext create(List<IVirtualBufferCache> virtualBufferCaches,
FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider,
ISerializerDeserializer[] fieldSerdes, IPrimitiveValueProviderFactory[] valueProviderFactories,
int numKeyFields, RTreePolicyType rtreePolicyType, ILSMMergePolicy mergePolicy,
@@ -78,7 +79,7 @@
.serdesToComparatorFactories(fieldSerdes, numKeyFields);
IBinaryComparatorFactory[] btreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes,
fieldSerdes.length);
- LSMRTreeWithAntiMatterTuples lsmTree = LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(virtualBufferCache,
+ LSMRTreeWithAntiMatterTuples lsmTree = LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(virtualBufferCaches,
file, diskBufferCache, diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories,
valueProviderFactories, rtreePolicyType, mergePolicy, opTracker, ioScheduler, ioOpCallbackProvider,
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length));
diff --git a/pregelix/pregelix-dataflow/src/main/java/edu/uci/ics/pregelix/dataflow/context/RuntimeContext.java b/pregelix/pregelix-dataflow/src/main/java/edu/uci/ics/pregelix/dataflow/context/RuntimeContext.java
index 496d066..adcb702 100644
--- a/pregelix/pregelix-dataflow/src/main/java/edu/uci/ics/pregelix/dataflow/context/RuntimeContext.java
+++ b/pregelix/pregelix-dataflow/src/main/java/edu/uci/ics/pregelix/dataflow/context/RuntimeContext.java
@@ -54,14 +54,14 @@
private final ILocalResourceRepository localResourceRepository;
private final ResourceIdFactory resourceIdFactory;
private final IBufferCache bufferCache;
- private final IVirtualBufferCache vBufferCache;
+ private final List<IVirtualBufferCache> vbcs;
private final IFileMapManager fileMapManager;
private final IOManager ioManager;
private final Map<Long, List<FileReference>> iterationToFiles = new ConcurrentHashMap<Long, List<FileReference>>();
private final Map<StateKey, IStateObject> appStateMap = new ConcurrentHashMap<StateKey, IStateObject>();
private final Map<String, Long> jobIdToSuperStep = new ConcurrentHashMap<String, Long>();
private final Map<String, Boolean> jobIdToMove = new ConcurrentHashMap<String, Boolean>();
-
+
private final ThreadFactory threadFactory = new ThreadFactory() {
public Thread newThread(Runnable r) {
return new Thread(r);
@@ -81,8 +81,10 @@
new PreDelayPageCleanerPolicy(Long.MAX_VALUE), fileMapManager, pageSize, numPages, 1000000,
threadFactory);
int numPagesInMemComponents = numPages / 8;
- vBufferCache = new MultitenantVirtualBufferCache(new VirtualBufferCache(new HeapBufferAllocator(), pageSize,
- numPagesInMemComponents));
+ vbcs = new ArrayList<IVirtualBufferCache>();
+ IVirtualBufferCache vBufferCache = new MultitenantVirtualBufferCache(new VirtualBufferCache(
+ new HeapBufferAllocator(), pageSize, numPagesInMemComponents));
+ vbcs.add(vBufferCache);
ioManager = (IOManager) appCtx.getRootContext().getIOManager();
lcManager = new NoBudgetIndexLifecycleManager();
localResourceRepository = new TransientLocalResourceRepository();
@@ -129,8 +131,8 @@
return bufferCache;
}
- public IVirtualBufferCache getVirtualBufferCache() {
- return vBufferCache;
+ public List<IVirtualBufferCache> getVirtualBufferCaches() {
+ return vbcs;
}
public IFileMapProvider getFileMapManager() {
diff --git a/pregelix/pregelix-runtime/src/main/java/edu/uci/ics/pregelix/runtime/bootstrap/VirtualBufferCacheProvider.java b/pregelix/pregelix-runtime/src/main/java/edu/uci/ics/pregelix/runtime/bootstrap/VirtualBufferCacheProvider.java
index ec51047..f15b1c2 100644
--- a/pregelix/pregelix-runtime/src/main/java/edu/uci/ics/pregelix/runtime/bootstrap/VirtualBufferCacheProvider.java
+++ b/pregelix/pregelix-runtime/src/main/java/edu/uci/ics/pregelix/runtime/bootstrap/VirtualBufferCacheProvider.java
@@ -14,6 +14,8 @@
*/
package edu.uci.ics.pregelix.runtime.bootstrap;
+import java.util.List;
+
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCacheProvider;
@@ -33,7 +35,7 @@
}
@Override
- public synchronized IVirtualBufferCache getVirtualBufferCache(IHyracksTaskContext ctx) {
- return RuntimeContext.get(ctx).getVirtualBufferCache();
+ public synchronized List<IVirtualBufferCache> getVirtualBufferCaches(IHyracksTaskContext ctx) {
+ return RuntimeContext.get(ctx).getVirtualBufferCaches();
}
}