Added lsm inverted index insert test based on new testing framework.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_inverted_index_updates_new@1834 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java
index 1009127..c2edb84 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java
@@ -35,6 +35,17 @@
cmpFactories, typeTraits.length, file);
return btree;
}
+
+ public static BTree createBTree(IBufferCache bufferCache, IFreePageManager freePageManager,
+ IFileMapProvider fileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
+ BTreeLeafFrameType leafType, FileReference file) throws BTreeException {
+ TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
+ ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
+ BTree btree = new BTree(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory,
+ cmpFactories, typeTraits.length, file);
+ return btree;
+ }
// Creates a new MultiComparator by constructing new IBinaryComparators.
public static MultiComparator getSearchMultiComparator(IBinaryComparatorFactory[] cmpFactories,
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/AbstractTreeIndex.java
index 5c6c913..5831805 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/AbstractTreeIndex.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/AbstractTreeIndex.java
@@ -176,7 +176,6 @@
if (!isActivated) {
throw new HyracksDataException("Failed to clear the index since it is not activated.");
}
-
initEmptyTree();
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 6e22d53..047d305 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -106,7 +106,7 @@
public void flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Flushing LSM-Tree.");
+ LOGGER.info("Flushing LSM-Index: " + lsmIndex);
}
Object newComponent = lsmIndex.flush(operation);
@@ -169,7 +169,7 @@
public void merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Merging LSM-Tree.");
+ LOGGER.info("Merging LSM-Index: " + lsmIndex);
}
// Point to the current searcher ref count, so we can wait for it later
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 361d954..560072c 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -19,7 +19,6 @@
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.logging.Logger;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
@@ -32,7 +31,6 @@
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
@@ -42,8 +40,8 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFinalizer;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
@@ -60,19 +58,17 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexFileManager.LSMInvertedIndexFileNameComponent;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInvertedIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexUtils;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-public class LSMInvertedIndex implements ILSMIndex, IIndex {
- private final Logger LOGGER = Logger.getLogger(LSMInvertedIndex.class.getName());
-
+public class LSMInvertedIndex implements ILSMIndex, IInvertedIndex {
public class LSMInvertedIndexComponent {
private final IInvertedIndex invIndex;
private final BTree deleteKeysBTree;
@@ -123,7 +119,7 @@
private boolean isActivated = false;
public LSMInvertedIndex(IBufferCache memBufferCache, InMemoryFreePageManager memFreePageManager,
- OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory diskBTreeFactory, ILSMIndexFileManager fileManager,
+ OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory, ILSMIndexFileManager fileManager,
IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
@@ -132,15 +128,16 @@
InMemoryInvertedIndex memInvIndex = InvertedIndexUtils.createInMemoryBTreeInvertedindex(memBufferCache,
memFreePageManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories,
tokenizerFactory);
- BTree deleteKeysBTree = BTreeUtils.createBTree(memBufferCache, diskFileMapProvider, invListTypeTraits,
- invListCmpFactories, BTreeLeafFrameType.REGULAR_NSM, memDeleteKeysBTreeFile);
+ BTree deleteKeysBTree = BTreeUtils.createBTree(memBufferCache,
+ ((InMemoryBufferCache) memBufferCache).getFileMapProvider(), invListTypeTraits, invListCmpFactories,
+ BTreeLeafFrameType.REGULAR_NSM, memDeleteKeysBTreeFile);
memComponent = new LSMInvertedIndexComponent(memInvIndex, deleteKeysBTree);
this.memBufferCache = memBufferCache;
this.memFreePageManager = memFreePageManager;
this.tokenizerFactory = tokenizerFactory;
this.fileManager = fileManager;
this.diskInvIndexFactory = diskInvIndexFactory;
- this.deletedKeysBTreeFactory = diskBTreeFactory;
+ this.deletedKeysBTreeFactory = deletedKeysBTreeFactory;
this.diskBufferCache = diskInvIndexFactory.getBufferCache();
this.diskFileMapProvider = diskFileMapProvider;
this.invListTypeTraits = invListTypeTraits;
@@ -169,7 +166,9 @@
try {
((InMemoryBufferCache) memComponent.getInvIndex().getBufferCache()).open();
memComponent.getInvIndex().create();
+ memComponent.getInvIndex().activate();
memComponent.getDeletedKeysBTree().create();
+ memComponent.getDeletedKeysBTree().activate();
List<Object> validFileNames = fileManager.cleanupAndGetValidFiles(componentFinalizer);
for (Object o : validFileNames) {
LSMInvertedIndexFileNameComponent component = (LSMInvertedIndexFileNameComponent) o;
@@ -242,6 +241,11 @@
throw new HyracksDataException(e);
}
+ for (Object o : diskComponents) {
+ LSMInvertedIndexComponent component = (LSMInvertedIndexComponent) o;
+ component.getInvIndex().deactivate();
+ component.getDeletedKeysBTree().deactivate();
+ }
memComponent.getInvIndex().deactivate();
memComponent.getDeletedKeysBTree().deactivate();
memComponent.getInvIndex().destroy();
@@ -280,7 +284,7 @@
public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
// TODO: Ignore opcallbacks for now.
- return new LSMInvertedIndexAccessor(lsmHarness, createOpContext());
+ return new LSMInvertedIndexAccessor(lsmHarness, fileManager, createOpContext());
}
private LSMInvertedIndexOpContext createOpContext() {
@@ -298,14 +302,7 @@
// TODO Auto-generated method stub
return null;
}
-
- @Override
- public Object merge(List<Object> mergedComponents, ILSMIOOperation operation) throws HyracksDataException,
- IndexException {
- // TODO Auto-generated method stub
- return null;
- }
-
+
@Override
public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput) throws IndexException {
// TODO Auto-generated method stub
@@ -320,62 +317,40 @@
}
@Override
+ // TODO: Deal with deletions properly.
public void search(IIndexCursor cursor, List<Object> diskComponents, ISearchPredicate pred, IIndexOpContext ictx,
boolean includeMemComponent, AtomicInteger searcherRefCount) throws HyracksDataException, IndexException {
- IIndexAccessor componentAccessor;
-
- // Over-provision by 1 if includeMemComponent == false, but that's okay!
- ArrayList<IIndexAccessor> indexAccessors = new ArrayList<IIndexAccessor>(diskComponents.size() + 1);
-
+ int numComponents = (includeMemComponent) ? diskComponents.size() : diskComponents.size() + 1;
+ ArrayList<IIndexAccessor> indexAccessors = new ArrayList<IIndexAccessor>(numComponents);
if (includeMemComponent) {
- componentAccessor = memoryInvertedIndex.createAccessor();
- indexAccessors.add(componentAccessor);
+ IIndexAccessor accessor = memComponent.getInvIndex().createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ indexAccessors.add(accessor);
}
-
for (int i = 0; i < diskComponents.size(); i++) {
- componentAccessor = ((IInvertedIndex) diskComponents.get(i)).createAccessor();
- indexAccessors.add(componentAccessor);
+ LSMInvertedIndexComponent component = (LSMInvertedIndexComponent) diskComponents.get(i);
+ IIndexAccessor accessor = component.getInvIndex().createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ indexAccessors.add(accessor);
}
-
LSMInvertedIndexCursorInitialState initState = new LSMInvertedIndexCursorInitialState(indexAccessors, ictx,
includeMemComponent, searcherRefCount, lsmHarness);
- LSMInvertedIndexSearchCursor lsmCursor = (LSMInvertedIndexSearchCursor) cursor;
- lsmCursor.open(initState, pred);
- }
-
- public void mergeSearch(IIndexCursor cursor, List<Object> diskComponents, ISearchPredicate pred,
- IIndexOpContext ictx, boolean includeMemComponent, AtomicInteger searcherRefCount)
- throws HyracksDataException, IndexException {
- IIndexAccessor componentAccessor;
-
- // Over-provision by 1 if includeMemComponent == false, but that's okay!
- ArrayList<IIndexAccessor> indexAccessors = new ArrayList<IIndexAccessor>(diskComponents.size() + 1);
-
- if (includeMemComponent) {
- componentAccessor = memoryInvertedIndex.createAccessor();
- indexAccessors.add(componentAccessor);
- }
-
- for (int i = 0; i < diskComponents.size(); i++) {
- componentAccessor = ((IInvertedIndex) diskComponents.get(i)).createAccessor();
- indexAccessors.add(componentAccessor);
- }
-
- LSMInvertedIndexCursorInitialState initState = new LSMInvertedIndexCursorInitialState(indexAccessors, ictx,
- includeMemComponent, searcherRefCount, lsmHarness);
- LSMInvertedIndexRangeSearchCursor rangeSearchCursor = (LSMInvertedIndexRangeSearchCursor) cursor;
- rangeSearchCursor.open(initState, pred);
+ cursor.open(initState, pred);
}
@Override
- public Object merge(List<Object> mergedComponents) throws HyracksDataException, IndexException {
- LSMInvertedIndexOpContext ctx = createOpContext();
+ // TODO: Deal with deletions properly.
+ public Object merge(List<Object> mergedComponents, ILSMIOOperation operation) throws HyracksDataException,
+ IndexException {
+ LSMInvertedIndexMergeOperation mergeOp = (LSMInvertedIndexMergeOperation) operation;
+ LSMInvertedIndexOpContext ctx = createOpContext();
+ ctx.reset(IndexOp.SEARCH);
IIndexCursor cursor = new LSMInvertedIndexRangeSearchCursor();
RangePredicate mergePred = new RangePredicate(null, null, true, true, null, null);
- //Scan diskInvertedIndexes ignoring the memoryInvertedIndex.
- List<Object> mergingComponents = lsmHarness.mergeSearch(cursor, mergePred, ctx, false);
+ // Scan diskInvertedIndexes ignoring the memoryInvertedIndex.
+ List<Object> mergingComponents = lsmHarness.search(cursor, mergePred, ctx, false);
mergedComponents.addAll(mergingComponents);
// Nothing to merge.
@@ -384,39 +359,30 @@
return null;
}
- // Bulk load the tuples from all diskInvertedIndexes into the new diskInvertedIndex.
- LSMInvertedFileNameComponent fNameComponent = getMergeTargetFileName(mergedComponents);
- BTree diskBTree = createDiskBTree(fileManager.createMergeFile(fNameComponent.getDictBTreeFileName()), true);
- // - Create an InvertedIndex instance
- OnDiskInvertedIndex mergedDiskInvertedIndex = createDiskInvertedIndex(
- fileManager.createMergeFile(fNameComponent.getInvertedFileName()), true, diskBTree);
-
- IIndexBulkLoadContext bulkLoadCtx = mergedDiskInvertedIndex.beginBulkLoad(1.0f);
+ // Create an inverted index instance.
+ IInvertedIndex mergedDiskInvertedIndex = createDiskInvIndex(diskInvIndexFactory,
+ mergeOp.getDictBTreeMergeTarget(), true);
+ IIndexBulkLoader invIndexBulkLoader = mergedDiskInvertedIndex.createBulkLoader(1.0f, false);
try {
while (cursor.hasNext()) {
cursor.next();
ITupleReference tuple = cursor.getTuple();
- mergedDiskInvertedIndex.bulkLoadAddTuple(tuple, bulkLoadCtx);
+ invIndexBulkLoader.add(tuple);
}
} finally {
cursor.close();
}
- mergedDiskInvertedIndex.endBulkLoad(bulkLoadCtx);
+ invIndexBulkLoader.end();
+
+ // Create an empty deleted keys BTree.
+ ITreeIndex deletedKeysBTree = createDiskTree(deletedKeysBTreeFactory, mergeOp.getDeletedKeysBTreeMergeTarget(),
+ true);
+ IIndexBulkLoader deletedKeysBulkLoader = deletedKeysBTree.createBulkLoader(1.0f, false);
+ deletedKeysBulkLoader.end();
return mergedDiskInvertedIndex;
}
- private LSMInvertedFileNameComponent getMergeTargetFileName(List<Object> mergingDiskTrees)
- throws HyracksDataException {
- BTree firstTree = ((OnDiskInvertedIndex) mergingDiskTrees.get(0)).getBTree();
- BTree lastTree = ((OnDiskInvertedIndex) mergingDiskTrees.get(mergingDiskTrees.size() - 1)).getBTree();
- FileReference firstFile = diskFileMapProvider.lookupFileName(firstTree.getFileId());
- FileReference lastFile = diskFileMapProvider.lookupFileName(lastTree.getFileId());
- LSMInvertedFileNameComponent component = (LSMInvertedFileNameComponent) ((LSMInvertedIndexFileManager) fileManager)
- .getRelMergeFileName(firstFile.getFile().getName(), lastFile.getFile().getName());
- return component;
- }
-
@Override
public void addMergedComponent(Object newComponent, List<Object> mergedComponents) {
diskComponents.removeAll(mergedComponents);
@@ -469,7 +435,7 @@
flushOp.getDeletedKeysBTreeFlushTarget(), true);
// Create a scan cursor on the deleted keys BTree underlying the in-memory inverted index.
- IIndexAccessor deletedKeysBTreeAccessor = (InMemoryInvertedIndexAccessor) memComponent.getDeletedKeysBTree()
+ IIndexAccessor deletedKeysBTreeAccessor = memComponent.getDeletedKeysBTree()
.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
IIndexCursor deletedKeysScanCursor = deletedKeysBTreeAccessor.createSearchCursor();
deletedKeysBTreeAccessor.search(deletedKeysScanCursor, nullPred);
@@ -539,4 +505,35 @@
public IndexType getIndexType() {
return IndexType.INVERTED;
}
+
+ @Override
+ public IInvertedListCursor createInvertedListCursor() {
+ throw new UnsupportedOperationException("Cannot create inverted list cursor on lsm inverted index.");
+ }
+
+ @Override
+ public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey, IIndexOpContext ictx)
+ throws HyracksDataException, IndexException {
+ throw new UnsupportedOperationException("Cannot open inverted list cursor on lsm inverted index.");
+ }
+
+ @Override
+ public ITypeTraits[] getInvListTypeTraits() {
+ return invListTypeTraits;
+ }
+
+ @Override
+ public IBinaryComparatorFactory[] getInvListCmpFactories() {
+ return invListCmpFactories;
+ }
+
+ @Override
+ public ITypeTraits[] getTokenTypeTraits() {
+ return tokenTypeTraits;
+ }
+
+ @Override
+ public IBinaryComparatorFactory[] getTokenCmpFactories() {
+ return tokenCmpFactories;
+ }
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
index e042892..57c560b 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
@@ -16,22 +16,30 @@
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMHarness;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMMergeInProgressException;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexFileManager.LSMInvertedIndexFileNameComponent;
public class LSMInvertedIndexAccessor implements ILSMIndexAccessor {
- protected LSMHarness lsmHarness;
- protected IIndexOpContext ctx;
-
- public LSMInvertedIndexAccessor(LSMHarness lsmHarness, IIndexOpContext ctx) {
+ protected final LSMHarness lsmHarness;
+ protected final ILSMIndexFileManager fileManager;
+ protected final IIndexOpContext ctx;
+
+ public LSMInvertedIndexAccessor(LSMHarness lsmHarness, ILSMIndexFileManager fileManager, IIndexOpContext ctx) {
this.lsmHarness = lsmHarness;
+ this.fileManager = fileManager;
this.ctx = ctx;
}
@@ -40,29 +48,63 @@
lsmHarness.insertUpdateOrDelete(tuple, ctx);
}
- public void update(ITupleReference tuple) throws HyracksDataException, IndexException {
- //not supported yet
- }
-
- public void delete(ITupleReference tuple) throws HyracksDataException, IndexException {
- //not supported yet
- }
-
public IIndexCursor createSearchCursor() {
return new LSMInvertedIndexSearchCursor();
}
public void search(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException, IndexException {
ctx.reset(IndexOp.SEARCH);
- //search include in-memory components
lsmHarness.search(cursor, searchPred, ctx, true);
}
- public void flush() throws HyracksDataException, IndexException {
- lsmHarness.flush();
+ @Override
+ public ILSMIOOperation createFlushOperation(ILSMIOOperationCallback callback) {
+ LSMInvertedIndexFileNameComponent fileNameComponent = (LSMInvertedIndexFileNameComponent) fileManager
+ .getRelFlushFileName();
+ FileReference dictBTreeFileRef = fileManager.createFlushFile(fileNameComponent.getDictBTreeFileName());
+ FileReference deletedKeysBTreeFileRef = fileManager.createFlushFile(fileNameComponent
+ .getDeletedKeysBTreeFileName());
+ return new LSMInvertedIndexFlushOperation(lsmHarness.getIndex(), dictBTreeFileRef, deletedKeysBTreeFileRef,
+ callback);
+ }
+
+ @Override
+ public void upsert(ITupleReference tuple) throws HyracksDataException, IndexException {
+ // TODO Auto-generated method stub
+
}
- public void merge() throws HyracksDataException, IndexException {
- lsmHarness.merge();
+ @Override
+ public ILSMIOOperation createMergeOperation(ILSMIOOperationCallback callback) throws HyracksDataException,
+ LSMMergeInProgressException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public void flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ lsmHarness.flush(operation);
+ }
+
+ @Override
+ public void merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ lsmHarness.merge(operation);
+ }
+
+ @Override
+ public void physicalDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ // TODO: Do we need this?
+ }
+
+ @Override
+ public void update(ITupleReference tuple) throws HyracksDataException, IndexException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void delete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ // TODO Auto-generated method stub
+
}
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexCursorInitialState.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexCursorInitialState.java
index 4b859ac..b7897a8 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexCursorInitialState.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexCursorInitialState.java
@@ -21,6 +21,8 @@
import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMHarness;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
@@ -31,6 +33,8 @@
private final LSMHarness lsmHarness;
private final List<IIndexAccessor> indexAccessors;
private final IIndexOpContext opContext;
+ private ISearchOperationCallback searchCallback;
+ private MultiComparator originalCmp;
public LSMInvertedIndexCursorInitialState(List<IIndexAccessor> indexAccessors, IIndexOpContext ctx,
boolean includeMemComponent, AtomicInteger searcherfRefCount, LSMHarness lsmHarness) {
@@ -69,4 +73,24 @@
public IIndexOpContext getOpContext() {
return opContext;
}
+
+ @Override
+ public ISearchOperationCallback getSearchOperationCallback() {
+ return searchCallback;
+ }
+
+ @Override
+ public void setSearchOperationCallback(ISearchOperationCallback searchCallback) {
+ this.searchCallback = searchCallback;
+ }
+
+ @Override
+ public MultiComparator getOriginalKeyComparator() {
+ return originalCmp;
+ }
+
+ @Override
+ public void setOriginialKeyComparator(MultiComparator originalCmp) {
+ this.originalCmp = originalCmp;
+ }
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
index 1132f36..436aba2 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
@@ -27,10 +27,9 @@
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFinalizer;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMIndexFileManager;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNameMapper;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@@ -41,7 +40,7 @@
private static final String DELETED_KEYS_BTREE_SUFFIX = "d";
// We only need a BTree factory because the inverted indexes consistency is validated against its dictionary BTree.
- private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
+ private final BTreeFactory btreeFactory;
private static FilenameFilter dictBTreeFilter = new FilenameFilter() {
public boolean accept(File dir, String name) {
@@ -56,7 +55,7 @@
};
public LSMInvertedIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
- TreeIndexFactory<? extends ITreeIndex> btreeFactory) {
+ BTreeFactory btreeFactory) {
super(ioManager, fileMapProvider, file, null);
this.btreeFactory = btreeFactory;
}
@@ -65,14 +64,15 @@
public Object getRelFlushFileName() {
String baseName = (String) super.getRelFlushFileName();
return new LSMInvertedIndexFileNameComponent(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX, baseName
- + DELETED_KEYS_BTREE_SUFFIX);
+ + SPLIT_STRING + DELETED_KEYS_BTREE_SUFFIX);
}
@Override
public Object getRelMergeFileName(String firstFileName, String lastFileName) throws HyracksDataException {
String baseName = (String) super.getRelMergeFileName(firstFileName, lastFileName);
- return new LSMInvertedIndexFileNameComponent(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX, baseName + DELETED_KEYS_BTREE_SUFFIX);
+ return new LSMInvertedIndexFileNameComponent(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX, baseName
+ + SPLIT_STRING + DELETED_KEYS_BTREE_SUFFIX);
}
@Override
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
new file mode 100644
index 0000000..5169c57
--- /dev/null
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.io.IODeviceHandle;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndex.LSMInvertedIndexComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex;
+
+public class LSMInvertedIndexMergeOperation implements ILSMIOOperation {
+ private final ILSMIndex index;
+ private final List<Object> mergingComponents;
+ private final ITreeIndexCursor cursor;
+ private final FileReference dictBTreeMergeTarget;
+ private final FileReference deletedKeysBTreeMergeTarget;
+ private final ILSMIOOperationCallback callback;
+
+ public LSMInvertedIndexMergeOperation(ILSMIndex index, List<Object> mergingComponents, ITreeIndexCursor cursor,
+ FileReference dictBTreeMergeTarget, FileReference deletedKeysBTreeMergeTarget, ILSMIOOperationCallback callback) {
+ this.index = index;
+ this.mergingComponents = mergingComponents;
+ this.cursor = cursor;
+ this.dictBTreeMergeTarget = dictBTreeMergeTarget;
+ this.deletedKeysBTreeMergeTarget = deletedKeysBTreeMergeTarget;
+ this.callback = callback;
+ }
+
+ @Override
+ public List<IODeviceHandle> getReadDevices() {
+ List<IODeviceHandle> devs = new ArrayList<IODeviceHandle>();
+ for (Object o : mergingComponents) {
+ LSMInvertedIndexComponent component = (LSMInvertedIndexComponent) o;
+ OnDiskInvertedIndex invIndex = (OnDiskInvertedIndex) component.getInvIndex();
+ devs.add(invIndex.getBTree().getFileReference().getDevideHandle());
+ devs.add(component.getDeletedKeysBTree().getFileReference().getDevideHandle());
+ }
+ return devs;
+ }
+
+ @Override
+ public List<IODeviceHandle> getWriteDevices() {
+ List<IODeviceHandle> devs = new ArrayList<IODeviceHandle>(2);
+ devs.add(dictBTreeMergeTarget.getDevideHandle());
+ devs.add(deletedKeysBTreeMergeTarget.getDevideHandle());
+ return devs;
+ }
+
+ @Override
+ public void perform() throws HyracksDataException, IndexException {
+ ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ accessor.merge(this);
+ }
+
+ @Override
+ public ILSMIOOperationCallback getCallback() {
+ return callback;
+ }
+
+ public FileReference getDictBTreeMergeTarget() {
+ return dictBTreeMergeTarget;
+ }
+
+ public FileReference getDeletedKeysBTreeMergeTarget() {
+ return deletedKeysBTreeMergeTarget;
+ }
+
+ public ITreeIndexCursor getCursor() {
+ return cursor;
+ }
+
+ public List<Object> getMergingComponents() {
+ return mergingComponents;
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
index 1e961e9..cdaa233 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
@@ -27,9 +27,9 @@
public class LSMInvertedIndexOpContext implements IIndexOpContext {
private IndexOp op;
- private final MultiComparator cmp;
- private final int invListFieldCount;
- private final int tokenFieldCount;
+ //private final MultiComparator cmp;
+ //private final int invListFieldCount;
+ //private final int tokenFieldCount;
private final IInvertedIndex memInvIndex;
private final IIndex memDeletedKeysBTree;
@@ -72,6 +72,7 @@
op = newOp;
}
+ /*
public int getInvListFieldCount() {
return invListFieldCount;
}
@@ -83,4 +84,5 @@
public MultiComparator getComparator() {
return cmp;
}
+ */
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
index 4d28758..8dac1d0 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
@@ -22,11 +22,8 @@
import java.util.PriorityQueue;
import java.util.concurrent.atomic.AtomicInteger;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
@@ -35,7 +32,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMHarness;
-import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.InvertedIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex.OnDiskInvertedIndexAccessor;
public class LSMInvertedIndexRangeSearchCursor implements IIndexCursor {
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexSearchCursor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexSearchCursor.java
index c5c8645..9215d77 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexSearchCursor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexSearchCursor.java
@@ -36,11 +36,11 @@
private List<IIndexAccessor> indexAccessors;
private List<IIndexCursor> indexCursors;// = new ArrayList<IIndexCursor>();
private ISearchPredicate searchPred;
-
+
public LSMInvertedIndexSearchCursor() {
indexCursors = new ArrayList<IIndexCursor>();
}
-
+
@Override
public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
LSMInvertedIndexCursorInitialState lsmInitialState = (LSMInvertedIndexCursorInitialState) initialState;
@@ -49,7 +49,7 @@
searcherRefCount = lsmInitialState.getSearcherRefCount();
indexAccessors = lsmInitialState.getIndexAccessors();
indexCursors.clear();
-// indexCursors = new ArrayList<IIndexCursor>(indexAccessors.size());
+ // indexCursors = new ArrayList<IIndexCursor>(indexAccessors.size());
cursorIndex = 0;
this.searchPred = searchPred;
@@ -81,11 +81,11 @@
public boolean hasNext() throws HyracksDataException {
IIndexAccessor currentAccessor;
IIndexCursor currentCursor;
-
- if(cursorIndex >= indexAccessors.size()) {
+
+ if (cursorIndex >= indexAccessors.size()) {
return false;
}
-
+
currentCursor = indexCursors.get(cursorIndex);
if (currentCursor.hasNext()) {
return true;
@@ -110,7 +110,7 @@
}
}
}
-
+
return false;
}
@@ -163,7 +163,7 @@
}
@Override
- public ITupleReference getTuple() throws HyracksDataException {
+ public ITupleReference getTuple() {
if (cursorIndex < indexCursors.size()) {
return indexCursors.get(cursorIndex).getTuple();
} else {
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
index c4a337f..f9caad9 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
@@ -76,7 +76,7 @@
btreeTypeTraits[tokenTypeTraits.length + i] = invListTypeTraits[i];
btreeCmpFactories[tokenTypeTraits.length + i] = invListCmpFactories[i];
}
- this.btree = BTreeUtils.createBTree(memBufferCache,
+ this.btree = BTreeUtils.createBTree(memBufferCache, memFreePageManager,
((InMemoryBufferCache) memBufferCache).getFileMapProvider(), btreeTypeTraits, btreeCmpFactories,
BTreeLeafFrameType.REGULAR_NSM, memBTreeFile);
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
index dceb16a..fba2426 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
@@ -20,13 +20,32 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndex;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeElementInvertedListBuilder;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@@ -55,36 +74,39 @@
return new FileReference(new File(invListsFile.getFile().getPath() + "_btree"));
}
- /*
public static LSMInvertedIndex createLSMInvertedIndex(InMemoryBufferCache memBufferCache,
- InMemoryFreePageManager memFreePageManager, ITypeTraits[] tokenTypeTraits, ITypeTraits[] invListTypeTraits,
- IBinaryComparatorFactory[] tokenCmpFactories, IBinaryComparatorFactory[] invListCmpFactories,
- IBinaryTokenizer tokenizer, IBufferCache diskBufferCache,
- LinkedListFreePageManagerFactory diskFreePageManagerFactory, IOManager ioManager, String onDiskDir,
- IFileMapProvider diskFileMapProvider) {
- InMemoryInvertedIndex memoryInvertedIndex = LSMInvertedIndexUtils.createInMemoryBTreeInvertedindex(
- memBufferCache, memFreePageManager, tokenTypeTraits, invListTypeTraits, tokenCmpFactories,
- invListCmpFactories, tokenizer);
- ITypeTraits[] combinedTraits = concatArrays(tokenTypeTraits, new ITypeTraits[] { IntegerPointable.TYPE_TRAITS,
- IntegerPointable.TYPE_TRAITS, IntegerPointable.TYPE_TRAITS, IntegerPointable.TYPE_TRAITS });
- ITreeIndexTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(combinedTraits);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
- BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFreePageManagerFactory,
- tokenCmpFactories, tokenCmpFactories.length + 4, interiorFrameFactory, leafFrameFactory);
- LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider,
- onDiskDir);
- IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
- OnDiskInvertedIndexFactory diskInvertedIndexFactory = new OnDiskInvertedIndexFactory(diskBufferCache, invListTypeTraits,
- invListCmpFactories, invListBuilder, tokenizer, fileManager);
- return new LSMInvertedIndex(memoryInvertedIndex, diskBTreeFactory, diskInvertedIndexFactory, fileManager,
- diskFileMapProvider);
- }
+ InMemoryFreePageManager memFreePageManager, IFileMapProvider diskFileMapProvider,
+ ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
+ ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
+ IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache,
+ IIOManager ioManager, String onDiskDir,
+ ILSMFlushController flushController, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler) throws IndexException {
- private static <T> T[] concatArrays(T[] first, T[] last) {
- T[] concatenated = Arrays.copyOf(first, first.length + last.length);
- System.arraycopy(last, 0, concatenated, first.length, last.length);
- return concatenated;
+ TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(invListTypeTraits);
+ ITreeIndexFrameFactory leafFrameFactory = BTreeUtils.getLeafFrameFactory(tupleWriterFactory,
+ BTreeLeafFrameType.REGULAR_NSM);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+ LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
+ metaFrameFactory);
+ BTreeFactory deletedKeysBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ freePageManagerFactory, interiorFrameFactory, leafFrameFactory, invListCmpFactories,
+ invListCmpFactories.length);
+
+ FileReference onDiskDirFileRef = new FileReference(new File(onDiskDir));
+ LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider,
+ onDiskDirFileRef, deletedKeysBTreeFactory);
+
+ IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
+ OnDiskInvertedIndexFactory invIndexFactory = new OnDiskInvertedIndexFactory(diskBufferCache,
+ diskFileMapProvider, invListBuilder, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
+ tokenCmpFactories, fileManager);
+
+ LSMInvertedIndex invIndex = new LSMInvertedIndex(memBufferCache, memFreePageManager, invIndexFactory,
+ deletedKeysBTreeFactory, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories,
+ tokenTypeTraits, tokenCmpFactories, tokenizerFactory, flushController, mergePolicy, opTracker,
+ ioScheduler);
+ return invIndex;
}
- */
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexInsertTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexInsertTest.java
new file mode 100644
index 0000000..84ae96e
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexInsertTest.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.invertedindex;
+
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.common.AbstractInvertedIndexLoadTest;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexTestContext.InvertedIndexType;
+
+public class LSMInvertedIndexInsertTest extends AbstractInvertedIndexLoadTest {
+
+ public LSMInvertedIndexInsertTest() {
+ super(InvertedIndexType.LSM, false);
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexLoadTest.java
index 3eaeb96..7c3a91f 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexLoadTest.java
@@ -59,8 +59,7 @@
@Test
public void wordTokensInvIndexTest() throws IOException, IndexException {
- InvertedIndexTestContext testCtx = InvertedIndexTestUtils.createWordInvIndexTestContext(harness,
- getBufferCache(), invIndexType);
+ InvertedIndexTestContext testCtx = InvertedIndexTestUtils.createWordInvIndexTestContext(harness, invIndexType);
TupleGenerator tupleGen = InvertedIndexTestUtils.createStringDocumentTupleGen(harness.getRandom());
runTest(testCtx, tupleGen);
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java
index a785398..a5e4723 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java
@@ -93,6 +93,7 @@
searchDocument.reset(documentCorpus.get(queryIndex));
}
+ // DEBUG
/*
StringBuilder builder = new StringBuilder();
UTF8StringPointable.toString(builder, searchDocument.getFieldData(0), searchDocument.getFieldStart(0));
@@ -152,8 +153,7 @@
@Test
public void wordTokensInvIndexTest() throws IOException, IndexException {
- InvertedIndexTestContext testCtx = InvertedIndexTestUtils.createWordInvIndexTestContext(harness,
- getBufferCache(), invIndexType);
+ InvertedIndexTestContext testCtx = InvertedIndexTestUtils.createWordInvIndexTestContext(harness, invIndexType);
TupleGenerator tupleGen = InvertedIndexTestUtils.createStringDocumentTupleGen(harness.getRandom());
IInvertedIndexSearchModifier searchModifier = new ConjunctiveSearchModifier();
runTest(testCtx, tupleGen, searchModifier);
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexTest.java
index 93c736e..5f28813 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexTest.java
@@ -21,12 +21,11 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexTestContext.InvertedIndexType;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
public abstract class AbstractInvertedIndexTest {
protected final LSMInvertedIndexTestHarness harness = new LSMInvertedIndexTestHarness();
- protected int NUM_DOCS_TO_INSERT = 10000;
+ protected int NUM_DOCS_TO_INSERT = 10000;
protected final InvertedIndexType invIndexType;
@@ -43,6 +42,4 @@
public void tearDown() throws HyracksDataException {
harness.tearDown();
}
-
- public abstract IBufferCache getBufferCache();
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
index 8ba880d..4a85fad 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
@@ -27,12 +27,19 @@
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.control.nc.io.IOManager;
-import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.DualIndexInMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.DualIndexInMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.FlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ImmediateScheduler;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoMergePolicy;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.RefCountingOperationTracker;
import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@@ -45,9 +52,9 @@
private static final int DEFAULT_DISK_PAGE_SIZE = 256;
private static final int DEFAULT_DISK_NUM_PAGES = 1000;
private static final int DEFAULT_DISK_MAX_OPEN_FILES = 200;
- private static final int DEFAULT_MEM_PAGE_SIZE = 4096;
+ private static final int DEFAULT_MEM_PAGE_SIZE = 256;
private static final int DEFAULT_MEM_NUM_PAGES = 200;
- private static final int DEFAULT_HYRACKS_FRAME_SIZE = 128;
+ private static final int DEFAULT_HYRACKS_FRAME_SIZE = 512;
private static final int DUMMY_FILE_ID = -1;
protected final int diskPageSize;
@@ -60,11 +67,14 @@
protected IOManager ioManager;
protected IBufferCache diskBufferCache;
protected IFileMapProvider diskFileMapProvider;
- protected IFreePageManager diskFreePageManager;
protected InMemoryBufferCache memBufferCache;
protected InMemoryFreePageManager memFreePageManager;
protected IHyracksTaskContext ctx;
-
+ protected ILSMIOOperationScheduler ioScheduler;
+ protected ILSMFlushController flushController;
+ protected ILSMMergePolicy mergePolicy;
+ protected ILSMOperationTracker opTracker;
+
protected final Random rnd = new Random();
protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
protected final static String sep = System.getProperty("file.separator");
@@ -80,6 +90,10 @@
this.memPageSize = DEFAULT_MEM_PAGE_SIZE;
this.memNumPages = DEFAULT_MEM_NUM_PAGES;
this.hyracksFrameSize = DEFAULT_HYRACKS_FRAME_SIZE;
+ this.ioScheduler = ImmediateScheduler.INSTANCE;
+ this.mergePolicy = NoMergePolicy.INSTANCE;
+ this.flushController = new FlushController();
+ this.opTracker = new RefCountingOperationTracker();
}
public LSMInvertedIndexTestHarness(int diskPageSize, int diskNumPages, int diskMaxOpenFiles, int memPageSize,
@@ -90,6 +104,10 @@
this.memPageSize = memPageSize;
this.memNumPages = memNumPages;
this.hyracksFrameSize = hyracksFrameSize;
+ this.ioScheduler = ImmediateScheduler.INSTANCE;
+ this.mergePolicy = NoMergePolicy.INSTANCE;
+ this.flushController = new FlushController();
+ this.opTracker = new RefCountingOperationTracker();
}
public void setUp() throws HyracksException {
@@ -103,9 +121,7 @@
memFreePageManager = new DualIndexInMemoryFreePageManager(memNumPages, new LIFOMetaDataFrameFactory());
ioManager = TestStorageManagerComponentHolder.getIOManager();
rnd.setSeed(RANDOM_SEED);
-
- File btreeFile = new File(onDiskDir + btreeFileName);
- btreeFile.deleteOnExit();
+
File invIndexFile = new File(onDiskDir + invIndexFileName);
invIndexFile.deleteOnExit();
invIndexFileRef = new FileReference(invIndexFile);
@@ -194,4 +210,20 @@
public Random getRandom() {
return rnd;
}
+
+ public ILSMIOOperationScheduler getIOScheduler() {
+ return ioScheduler;
+ }
+
+ public ILSMOperationTracker getOperationTracker() {
+ return opTracker;
+ }
+
+ public ILSMFlushController getFlushController() {
+ return flushController;
+ }
+
+ public ILSMMergePolicy getMergePolicy() {
+ return mergePolicy;
+ }
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexInsertTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexInsertTest.java
index c1ee925..5bcdf37 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexInsertTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexInsertTest.java
@@ -17,16 +17,10 @@
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.common.AbstractInvertedIndexLoadTest;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexTestContext.InvertedIndexType;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
public class InMemoryInvertedIndexInsertTest extends AbstractInvertedIndexLoadTest {
public InMemoryInvertedIndexInsertTest() {
super(InvertedIndexType.INMEMORY, false);
}
-
- @Override
- public IBufferCache getBufferCache() {
- return harness.getMemBufferCache();
- }
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexSearchTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexSearchTest.java
index 47e5b78..b9a2c99 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexSearchTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexSearchTest.java
@@ -17,16 +17,10 @@
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.common.AbstractInvertedIndexSearchTest;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexTestContext.InvertedIndexType;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
public class InMemoryInvertedIndexSearchTest extends AbstractInvertedIndexSearchTest {
public InMemoryInvertedIndexSearchTest() {
super(InvertedIndexType.INMEMORY, false);
}
-
- @Override
- public IBufferCache getBufferCache() {
- return harness.getMemBufferCache();
- }
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexBulkLoadTest.java
index 700b70f..1151d76 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexBulkLoadTest.java
@@ -17,16 +17,10 @@
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.common.AbstractInvertedIndexLoadTest;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexTestContext.InvertedIndexType;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
public class OnDiskInvertedIndexBulkLoadTest extends AbstractInvertedIndexLoadTest {
public OnDiskInvertedIndexBulkLoadTest() {
super(InvertedIndexType.ONDISK, true);
}
-
- @Override
- public IBufferCache getBufferCache() {
- return harness.getDiskBufferCache();
- }
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchTest.java
index 702dc49..8fcd50a 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchTest.java
@@ -17,16 +17,10 @@
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.common.AbstractInvertedIndexSearchTest;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexTestContext.InvertedIndexType;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
public class OnDiskInvertedIndexSearchTest extends AbstractInvertedIndexSearchTest {
public OnDiskInvertedIndexSearchTest() {
super(InvertedIndexType.ONDISK, true);
}
-
- @Override
- public IBufferCache getBufferCache() {
- return harness.getDiskBufferCache();
- }
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexTestContext.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexTestContext.java
index 2fbede6..54cdc8c 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexTestContext.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexTestContext.java
@@ -27,20 +27,17 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexTestContext;
import edu.uci.ics.hyracks.storage.am.common.CheckTuple;
-import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.common.LSMInvertedIndexTestHarness;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.exceptions.InvertedIndexException;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@SuppressWarnings("rawtypes")
public class InvertedIndexTestContext extends OrderedIndexTestContext {
@@ -90,8 +87,7 @@
return allCmpFactories;
}
- public static InvertedIndexTestContext create(IBufferCache bufferCache, IFreePageManager freePageManager,
- IFileMapProvider fileMapProvider, FileReference invListsFile, ISerializerDeserializer[] fieldSerdes,
+ public static InvertedIndexTestContext create(LSMInvertedIndexTestHarness harness, ISerializerDeserializer[] fieldSerdes,
int tokenFieldCount, IBinaryTokenizerFactory tokenizerFactory, InvertedIndexType invIndexType) throws IndexException {
ITypeTraits[] allTypeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] allCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes,
@@ -115,13 +111,24 @@
IInvertedIndex invIndex;
switch (invIndexType) {
case INMEMORY: {
- invIndex = InvertedIndexUtils.createInMemoryBTreeInvertedindex(bufferCache, freePageManager,
- invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory);
+ invIndex = InvertedIndexUtils.createInMemoryBTreeInvertedindex(harness.getMemBufferCache(),
+ harness.getMemFreePageManager(), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
+ tokenCmpFactories, tokenizerFactory);
break;
}
case ONDISK: {
- invIndex = InvertedIndexUtils.createOnDiskInvertedIndex(bufferCache, fileMapProvider,
- invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, invListsFile);
+ invIndex = InvertedIndexUtils.createOnDiskInvertedIndex(harness.getDiskBufferCache(),
+ harness.getDiskFileMapProvider(), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
+ tokenCmpFactories, harness.getInvListsFileRef());
+ break;
+ }
+ case LSM: {
+ invIndex = InvertedIndexUtils.createLSMInvertedIndex(harness.getMemBufferCache(),
+ harness.getMemFreePageManager(), harness.getDiskFileMapProvider(), invListTypeTraits,
+ invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
+ harness.getDiskBufferCache(), harness.getIOManager(), harness.getOnDiskDir(),
+ harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTracker(),
+ harness.getIOScheduler());
break;
}
default: {
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexTestUtils.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexTestUtils.java
index b0393e7..a7accc0 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexTestUtils.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexTestUtils.java
@@ -75,15 +75,14 @@
}
public static InvertedIndexTestContext createWordInvIndexTestContext(LSMInvertedIndexTestHarness harness,
- IBufferCache bufferCache, InvertedIndexType invIndexType) throws IOException, IndexException {
+ InvertedIndexType invIndexType) throws IOException, IndexException {
ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] {
UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
ITokenFactory tokenFactory = new UTF8WordTokenFactory();
IBinaryTokenizerFactory tokenizerFactory = new DelimitedUTF8StringBinaryTokenizerFactory(true, false,
tokenFactory);
- InvertedIndexTestContext testCtx = InvertedIndexTestContext.create(bufferCache,
- harness.getMemFreePageManager(), harness.getDiskFileMapProvider(), harness.getInvListsFileRef(),
- fieldSerdes, 1, tokenizerFactory, invIndexType);
+ InvertedIndexTestContext testCtx = InvertedIndexTestContext.create(harness, fieldSerdes, 1, tokenizerFactory,
+ invIndexType);
return testCtx;
}