Almost done with flush operation of lsm inverted index.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_inverted_index_updates_new@1833 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexFileNameMapper.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexFileNameMapper.java
new file mode 100644
index 0000000..d7ec129
--- /dev/null
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexFileNameMapper.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api;
+
+/**
+ * Maps from the dictionary BTree file/path to a corresponding inverted-lists file/path.
+ */
+public interface IInvertedIndexFileNameMapper {
+ public String getInvListsFilePath(String dictBTreeFilePath);
+}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 01ace03..361d954 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -28,6 +28,7 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
@@ -43,15 +44,14 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFinalizer;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
@@ -62,6 +62,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexFileManager.LSMInvertedIndexFileNameComponent;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInvertedIndex;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInvertedIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
@@ -104,7 +105,7 @@
// For creating inverted indexes in flush and merge.
protected final OnDiskInvertedIndexFactory diskInvIndexFactory;
// For creating deleted-keys BTrees in flush and merge.
- protected final BTreeFactory diskBTreeFactory;
+ protected final BTreeFactory deletedKeysBTreeFactory;
protected final IBufferCache diskBufferCache;
protected final IFileMapProvider diskFileMapProvider;
// List of LSMInvertedIndexComponent instances. Using Object for better sharing via
@@ -139,7 +140,7 @@
this.tokenizerFactory = tokenizerFactory;
this.fileManager = fileManager;
this.diskInvIndexFactory = diskInvIndexFactory;
- this.diskBTreeFactory = diskBTreeFactory;
+ this.deletedKeysBTreeFactory = diskBTreeFactory;
this.diskBufferCache = diskInvIndexFactory.getBufferCache();
this.diskFileMapProvider = diskFileMapProvider;
this.invListTypeTraits = invListTypeTraits;
@@ -161,30 +162,34 @@
}
@Override
- // TODO: Properly implement this one.
public synchronized void activate() throws HyracksDataException {
if (isActivated) {
return;
}
-
- ((InMemoryBufferCache) memComponent.getInvIndex().getBufferCache()).open();
- memComponent.getInvIndex().create();
- memComponent.getDeletedKeysBTree().create();
- List<Object> validFileNames = fileManager.cleanupAndGetValidFiles(componentFinalizer);
- for (Object o : validFileNames) {
- LSMInvertedIndexFileNameComponent component = (LSMInvertedIndexFileNameComponent) o;
- FileReference rtreeFile = new FileReference(new File(component.getRTreeFileName()));
- FileReference btreeFile = new FileReference(new File(component.getBTreeFileName()));
- RTree rtree = (RTree) createDiskTree(diskRTreeFactory, rtreeFile, false);
- BTree btree = (BTree) createDiskTree(diskBTreeFactory, btreeFile, false);
- LSMRTreeComponent diskComponent = new LSMRTreeComponent(rtree, btree);
- diskComponents.add(diskComponent);
+ try {
+ ((InMemoryBufferCache) memComponent.getInvIndex().getBufferCache()).open();
+ memComponent.getInvIndex().create();
+ memComponent.getDeletedKeysBTree().create();
+ List<Object> validFileNames = fileManager.cleanupAndGetValidFiles(componentFinalizer);
+ for (Object o : validFileNames) {
+ LSMInvertedIndexFileNameComponent component = (LSMInvertedIndexFileNameComponent) o;
+ FileReference dictBTreeFile = new FileReference(new File(component.getDictBTreeFileName()));
+ FileReference deletedKeysBTreeFile = new FileReference(
+ new File(component.getDeletedKeysBTreeFileName()));
+ IInvertedIndex invIndex = createDiskInvIndex(diskInvIndexFactory, dictBTreeFile, false);
+ BTree deletedKeysBTree = (BTree) createDiskTree(deletedKeysBTreeFactory, deletedKeysBTreeFile, false);
+ LSMInvertedIndexComponent diskComponent = new LSMInvertedIndexComponent(invIndex, deletedKeysBTree);
+ diskComponents.add(diskComponent);
+ }
+ isActivated = true;
+ // TODO: Maybe we can make activate throw an index exception?
+ } catch (IndexException e) {
+ throw new HyracksDataException(e);
}
- isActivated = true;
}
- protected IInvertedIndex createDiskInvIndex(OnDiskInvertedIndexFactory invIndexFactory, FileReference fileRef, boolean create) throws HyracksDataException, IndexException {
- IInvertedIndex invIndex = invIndexFactory.createIndexInstance(fileRef);
+ protected IInvertedIndex createDiskInvIndex(OnDiskInvertedIndexFactory invIndexFactory, FileReference dictBTreeFileRef, boolean create) throws HyracksDataException, IndexException {
+ IInvertedIndex invIndex = invIndexFactory.createIndexInstance(dictBTreeFileRef);
if (create) {
invIndex.create();
}
@@ -193,8 +198,8 @@
return invIndex;
}
- protected ITreeIndex createDiskDeletedKeysBTree(BTreeFactory btreeFactory, FileReference fileRef, boolean create) throws HyracksDataException, IndexException {
- ITreeIndex btree = btreeFactory.createIndexInstance(fileRef);
+ protected ITreeIndex createDiskTree(BTreeFactory btreeFactory, FileReference btreeFileRef, boolean create) throws HyracksDataException, IndexException {
+ ITreeIndex btree = btreeFactory.createIndexInstance(btreeFileRef);
if (create) {
btree.create();
}
@@ -208,8 +213,7 @@
if (!isActivated) {
throw new HyracksDataException("Failed to clear the index since it is not activated.");
}
- memComponent.getInvIndex().clear();
- memComponent.getDeletedKeysBTree().clear();
+ resetInMemoryComponent();
for (Object o : diskComponents) {
LSMInvertedIndexComponent component = (LSMInvertedIndexComponent) o;
component.getInvIndex().deactivate();
@@ -262,28 +266,31 @@
}
@Override
+ public void validate() throws HyracksDataException {
+ memComponent.getInvIndex().validate();
+ memComponent.getDeletedKeysBTree().validate();
+ for (Object o : diskComponents) {
+ LSMInvertedIndexComponent component = (LSMInvertedIndexComponent) o;
+ component.getInvIndex().validate();
+ component.getDeletedKeysBTree().validate();
+ }
+ }
+
+ @Override
public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
// TODO: Ignore opcallbacks for now.
return new LSMInvertedIndexAccessor(lsmHarness, createOpContext());
}
-
- @Override
- public void validate() throws HyracksDataException {
- // TODO Auto-generated method stub
-
+
+ private LSMInvertedIndexOpContext createOpContext() {
+ return new LSMInvertedIndexOpContext(memComponent.getInvIndex(), memComponent.getDeletedKeysBTree());
}
-
+
@Override
public long getInMemorySize() {
- // TODO Auto-generated method stub
- return 0;
- }
-
- @Override
- public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput) throws IndexException {
- // TODO Auto-generated method stub
- return null;
+ InMemoryBufferCache memBufferCache = (InMemoryBufferCache) memComponent.getInvIndex().getBufferCache();
+ return memBufferCache.getNumPages() * memBufferCache.getPageSize();
}
@Override
@@ -300,49 +307,15 @@
}
@Override
- public Object flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput) throws IndexException {
// TODO Auto-generated method stub
return null;
}
-
- @Override
- public ILSMFlushController getFlushController() {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public ILSMOperationTracker getOperationTracker() {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public ILSMIOOperationScheduler getIOScheduler() {
- // TODO Auto-generated method stub
- return null;
- }
-
- private LSMInvertedIndexOpContext createOpContext() {
- return new LSMInvertedIndexOpContext(memoryInvertedIndex);
- }
-
- @Override
- public IBufferCache getBufferCache() {
- return diskBufferCache;
- }
-
- @Override
- public IndexType getIndexType() {
- return IndexType.INVERTED;
- }
-
+
public boolean insertUpdateOrDelete(ITupleReference tuple, IIndexOpContext ictx) throws HyracksDataException,
IndexException {
- // TODO: Only insert is supported for now. Will need the context for later when update and delete
- // are also supported.
LSMInvertedIndexOpContext ctx = (LSMInvertedIndexOpContext) ictx;
- memAccessor.insert(tuple);
+ ctx.insertAccessor.insert(tuple);
return true;
}
@@ -413,7 +386,7 @@
// Bulk load the tuples from all diskInvertedIndexes into the new diskInvertedIndex.
LSMInvertedFileNameComponent fNameComponent = getMergeTargetFileName(mergedComponents);
- BTree diskBTree = createDiskBTree(fileManager.createMergeFile(fNameComponent.getBTreeFileName()), true);
+ BTree diskBTree = createDiskBTree(fileManager.createMergeFile(fNameComponent.getDictBTreeFileName()), true);
// - Create an InvertedIndex instance
OnDiskInvertedIndex mergedDiskInvertedIndex = createDiskInvertedIndex(
fileManager.createMergeFile(fNameComponent.getInvertedFileName()), true, diskBTree);
@@ -453,137 +426,83 @@
@Override
public void cleanUpAfterMerge(List<Object> mergedComponents) throws HyracksDataException {
for (Object o : mergedComponents) {
- OnDiskInvertedIndex oldInvertedIndex = (OnDiskInvertedIndex) o;
- BTree oldBTree = oldInvertedIndex.getBTree();
-
- //delete a diskBTree file.
- FileReference fileRef = diskFileMapProvider.lookupFileName(oldBTree.getFileId());
- diskBufferCache.closeFile(oldBTree.getFileId());
- diskBufferCache.deleteFile(oldBTree.getFileId(), false);
- oldBTree.close();
- fileRef.getFile().delete();
-
- //delete a diskInvertedIndex file.
- fileRef = diskFileMapProvider.lookupFileName(oldInvertedIndex.getFileId());
- diskBufferCache.closeFile(oldInvertedIndex.getFileId());
- diskBufferCache.deleteFile(oldInvertedIndex.getFileId(), false);
- oldInvertedIndex.close();
- fileRef.getFile().delete();
+ LSMInvertedIndexComponent component = (LSMInvertedIndexComponent) o;
+ BTree oldDeletedKeysBTree = component.getDeletedKeysBTree();
+ oldDeletedKeysBTree.deactivate();
+ oldDeletedKeysBTree.destroy();
+ IInvertedIndex oldInvIndex = component.getInvIndex();
+ oldInvIndex.deactivate();
+ oldInvIndex.destroy();
}
}
@Override
- public Object flush() throws HyracksDataException, IndexException {
+ public Object flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ LSMInvertedIndexFlushOperation flushOp = (LSMInvertedIndexFlushOperation) operation;
- // ---------------------------------------------------
- // [Flow]
- // #. Create a scanCursor for the BTree of the memoryInvertedIndex to iterate all keys in it.
- // #. Create an diskInvertedIndex where all keys of memoryInvertedIndex will be bulkloaded.
- // - Create a BTree instance for diskBTree
- // - Create an InvertedIndex instance
- // #. Begin the bulkload of the diskInvertedIndex.
- // #. While iterating the scanCursor, add each key into the diskInvertedIndex in the bulkload mode.
- // #. End the bulkload.
- // #. Return the newly created diskInvertedIndex.
- // ---------------------------------------------------
+ // Create an inverted index instance to be bulk loaded.
+ IInvertedIndex diskInvertedIndex = createDiskInvIndex(diskInvIndexFactory, flushOp.getDictBTreeFlushTarget(),
+ true);
- // #. Create a scanCursor of memoryInvertedIndex to iterate all keys in it.
- BTree inMemBtree = ((InMemoryInvertedIndex) memoryInvertedIndex).getBTree();
- IIndexAccessor btreeAccessor = inMemBtree.createAccessor();
- MultiComparator btreeMultiComparator = MultiComparator.create(inMemBtree.getComparatorFactories());
- RangePredicate scanPred = new RangePredicate(null, null, true, true, btreeMultiComparator, btreeMultiComparator);
+ // Create a scan cursor on the BTree underlying the in-memory inverted index.
+ InMemoryInvertedIndexAccessor memInvIndexAccessor = (InMemoryInvertedIndexAccessor) memComponent.getInvIndex()
+ .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ BTreeAccessor memBTreeAccessor = memInvIndexAccessor.getBTreeAccessor();
+ RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null);
+ IIndexCursor scanCursor = memBTreeAccessor.createSearchCursor();
+ memBTreeAccessor.search(scanCursor, nullPred);
- IIndexCursor scanCursor = btreeAccessor.createSearchCursor();
- btreeAccessor.search(scanCursor, scanPred);
-
- // #. Create a diskInvertedIndex where all keys of memoryInvertedIndex will be bulkloaded.
- // - Create a BTree instance for diskBTree
- LSMInvertedFileNameComponent fNameComponent = (LSMInvertedFileNameComponent) fileManager.getRelFlushFileName();
- BTree diskBTree = createDiskBTree(fileManager.createFlushFile(fNameComponent.getBTreeFileName()), true);
- // - Create an InvertedIndex instance
- OnDiskInvertedIndex diskInvertedIndex = createDiskInvertedIndex(
- fileManager.createFlushFile(fNameComponent.getInvertedFileName()), true, diskBTree);
-
- // #. Begin the bulkload of the diskInvertedIndex.
- IIndexBulkLoadContext bulkLoadCtx = diskInvertedIndex.beginBulkLoad(1.0f);
+ // Bulk load the disk inverted index from the in-memory inverted index.
+ IIndexBulkLoader invIndexBulkLoader = diskInvertedIndex.createBulkLoader(1.0f, false);
try {
while (scanCursor.hasNext()) {
scanCursor.next();
- diskInvertedIndex.bulkLoadAddTuple(scanCursor.getTuple(), bulkLoadCtx);
+ invIndexBulkLoader.add(scanCursor.getTuple());
}
} finally {
scanCursor.close();
}
- diskInvertedIndex.endBulkLoad(bulkLoadCtx);
+ invIndexBulkLoader.end();
- return diskInvertedIndex;
- }
+ // Create an BTree instance for the deleted keys.
+ BTree diskDeletedKeysBTree = (BTree) createDiskTree(deletedKeysBTreeFactory,
+ flushOp.getDeletedKeysBTreeFlushTarget(), true);
- private BTree createBTreeFlushTarget() throws HyracksDataException {
- LSMInvertedFileNameComponent fNameComponent = (LSMInvertedFileNameComponent) fileManager.getRelFlushFileName();
- FileReference fileRef = fileManager.createFlushFile(fNameComponent.getBTreeFileName());
- return createDiskBTree(fileRef, true);
- }
+ // Create a scan cursor on the deleted keys BTree underlying the in-memory inverted index.
+ IIndexAccessor deletedKeysBTreeAccessor = (InMemoryInvertedIndexAccessor) memComponent.getDeletedKeysBTree()
+ .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ IIndexCursor deletedKeysScanCursor = deletedKeysBTreeAccessor.createSearchCursor();
+ deletedKeysBTreeAccessor.search(deletedKeysScanCursor, nullPred);
- private BTree createDiskBTree(FileReference fileRef, boolean createBTree) throws HyracksDataException {
- // File will be deleted during cleanup of merge().
- diskBufferCache.createFile(fileRef);
- int diskBTreeFileId = diskFileMapProvider.lookupFileId(fileRef);
- // File will be closed during cleanup of merge().
- diskBufferCache.openFile(diskBTreeFileId);
- // Create new BTree instance.
- BTree diskBTree = diskBTreeFactory.createIndexInstance();
- if (createBTree) {
- diskBTree.create(diskBTreeFileId);
+ // Bulk load the deleted-keys BTree.
+ IIndexBulkLoader deletedKeysBTreeBulkLoader = diskDeletedKeysBTree.createBulkLoader(1.0f, false);
+ try {
+ while (deletedKeysScanCursor.hasNext()) {
+ deletedKeysScanCursor.next();
+ deletedKeysBTreeBulkLoader.add(deletedKeysScanCursor.getTuple());
+ }
+ } finally {
+ deletedKeysScanCursor.close();
}
- // BTree will be closed during cleanup of merge().
- diskBTree.open(diskBTreeFileId);
- return diskBTree;
- }
+ deletedKeysBTreeBulkLoader.end();
- private OnDiskInvertedIndex createInvertedIndexFlushTarget(BTree diskBTree) throws HyracksDataException {
- FileReference fileRef = fileManager.createFlushFile((String) fileManager.getRelFlushFileName());
- return createDiskInvertedIndex(fileRef, true, diskBTree);
- }
-
- private OnDiskInvertedIndex createDiskInvertedIndex(FileReference fileRef, boolean createInvertedIndex, BTree diskBTree)
- throws HyracksDataException {
- // File will be deleted during cleanup of merge().
- diskBufferCache.createFile(fileRef);
- int diskInvertedIndexFileId = diskFileMapProvider.lookupFileId(fileRef);
- // File will be closed during cleanup of merge().
- diskBufferCache.openFile(diskInvertedIndexFileId);
- // Create new InvertedIndex instance.
- OnDiskInvertedIndex diskInvertedIndex = (OnDiskInvertedIndex) diskInvertedIndexFactory.createIndexInstance(diskBTree);
- if (createInvertedIndex) {
- diskInvertedIndex.create(diskInvertedIndexFileId);
- }
- // InvertedIndex will be closed during cleanup of merge().
- diskInvertedIndex.open(diskInvertedIndexFileId);
- return diskInvertedIndex;
+ return new LSMInvertedIndexComponent(diskInvertedIndex, diskDeletedKeysBTree);
}
public void addFlushedComponent(Object index) {
- diskInvertedIndexList.addFirst(index);
+ diskComponents.addFirst(index);
}
@Override
public InMemoryFreePageManager getInMemoryFreePageManager() {
- // TODO This code should be changed more generally if IInMemoryInvertedIndex interface is defined and
- // InMemoryBtreeInvertedIndex implements IInMemoryInvertedIndex
- InMemoryInvertedIndex memoryBTreeInvertedIndex = (InMemoryInvertedIndex) memoryInvertedIndex;
- return (InMemoryFreePageManager) memoryBTreeInvertedIndex.getBTree().getFreePageManager();
+ return memFreePageManager;
}
@Override
public void resetInMemoryComponent() throws HyracksDataException {
- // TODO This code should be changed more generally if IInMemoryInvertedIndex interface is defined and
- // InMemoryBtreeInvertedIndex implements IInMemoryInvertedIndex
- InMemoryInvertedIndex memoryBTreeInvertedIndex = (InMemoryInvertedIndex) memoryInvertedIndex;
- BTree memBTree = memoryBTreeInvertedIndex.getBTree();
- InMemoryFreePageManager memFreePageManager = (InMemoryFreePageManager) memBTree.getFreePageManager();
memFreePageManager.reset();
- memBTree.create(memBTree.getFileId());
+ memComponent.getInvIndex().clear();
+ memComponent.getDeletedKeysBTree().clear();
}
@Override
@@ -595,4 +514,29 @@
public ILSMComponentFinalizer getComponentFinalizer() {
return componentFinalizer;
}
+
+ @Override
+ public ILSMFlushController getFlushController() {
+ return lsmHarness.getFlushController();
+ }
+
+ @Override
+ public ILSMOperationTracker getOperationTracker() {
+ return lsmHarness.getOperationTracker();
+ }
+
+ @Override
+ public ILSMIOOperationScheduler getIOScheduler() {
+ return lsmHarness.getIOScheduler();
+ }
+
+ @Override
+ public IBufferCache getBufferCache() {
+ return diskBufferCache;
+ }
+
+ @Override
+ public IndexType getIndexType() {
+ return IndexType.INVERTED;
+ }
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
index 26fd72e..1132f36 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
@@ -31,20 +31,21 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFinalizer;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNameMapper;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-// TODO: Implement this one properly!
-public class LSMInvertedIndexFileManager extends LSMIndexFileManager {
- private static final String BTREE_SUFFIX = "b";
+// TODO: Refactor for better code sharing with other file managers.
+public class LSMInvertedIndexFileManager extends LSMIndexFileManager implements IInvertedIndexFileNameMapper {
+ private static final String DICT_BTREE_SUFFIX = "b";
private static final String INVLISTS_SUFFIX = "i";
private static final String DELETED_KEYS_BTREE_SUFFIX = "d";
- private final TreeIndexFactory<? extends ITreeIndex> rtreeFactory;
+ // We only need a BTree factory because the inverted indexes consistency is validated against its dictionary BTree.
private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
- private static FilenameFilter btreeFilter = new FilenameFilter() {
+ private static FilenameFilter dictBTreeFilter = new FilenameFilter() {
public boolean accept(File dir, String name) {
- return !name.startsWith(".") && name.endsWith(BTREE_SUFFIX);
+ return !name.startsWith(".") && name.endsWith(DICT_BTREE_SUFFIX);
}
};
@@ -55,32 +56,29 @@
};
public LSMInvertedIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
- TreeIndexFactory<? extends ITreeIndex> rtreeFactory, TreeIndexFactory<? extends ITreeIndex> btreeFactory) {
+ TreeIndexFactory<? extends ITreeIndex> btreeFactory) {
super(ioManager, fileMapProvider, file, null);
- this.rtreeFactory = rtreeFactory;
this.btreeFactory = btreeFactory;
}
@Override
public Object getRelFlushFileName() {
String baseName = (String) super.getRelFlushFileName();
- return new LSMInvertedIndexFileNameComponent(baseName + SPLIT_STRING + BTREE_SUFFIX, baseName + SPLIT_STRING
- + INVLISTS_SUFFIX, baseName + DELETED_KEYS_BTREE_SUFFIX);
+ return new LSMInvertedIndexFileNameComponent(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX, baseName
+ + DELETED_KEYS_BTREE_SUFFIX);
}
@Override
public Object getRelMergeFileName(String firstFileName, String lastFileName) throws HyracksDataException {
String baseName = (String) super.getRelMergeFileName(firstFileName, lastFileName);
- return new LSMInvertedIndexFileNameComponent(baseName + SPLIT_STRING + BTREE_SUFFIX, baseName + SPLIT_STRING
- + INVLISTS_SUFFIX, baseName + DELETED_KEYS_BTREE_SUFFIX);
+ return new LSMInvertedIndexFileNameComponent(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX, baseName + DELETED_KEYS_BTREE_SUFFIX);
}
@Override
public List<Object> cleanupAndGetValidFiles(ILSMComponentFinalizer componentFinalizer) throws HyracksDataException {
List<Object> validFiles = new ArrayList<Object>();
- ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<ComparableFileName>();
- ArrayList<ComparableFileName> allInvListsFiles = new ArrayList<ComparableFileName>();
+ ArrayList<ComparableFileName> allDictBTreeFiles = new ArrayList<ComparableFileName>();
ArrayList<ComparableFileName> allDeletedKeysBTreeFiles = new ArrayList<ComparableFileName>();
// Gather files from all IODeviceHandles.
@@ -88,27 +86,26 @@
cleanupAndGetValidFilesInternal(dev, deletedKeysBTreeFilter, btreeFactory, componentFinalizer,
allDeletedKeysBTreeFiles);
HashSet<String> deletedKeysBTreeFilesSet = new HashSet<String>();
- for (ComparableFileName cmpFileName : allBTreeFiles) {
+ for (ComparableFileName cmpFileName : allDictBTreeFiles) {
int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
deletedKeysBTreeFilesSet.add(cmpFileName.fileName.substring(0, index));
}
// We use the dictionary BTree of the inverted index for validation.
// List of valid dictionary BTree files that may or may not have a deleted-keys BTree buddy. Will check for buddies below.
ArrayList<ComparableFileName> tmpAllBTreeFiles = new ArrayList<ComparableFileName>();
- // TODO: Fix factory.
- cleanupAndGetValidFilesInternal(dev, btreeFilter, rtreeFactory, componentFinalizer, tmpAllBTreeFiles);
+ cleanupAndGetValidFilesInternal(dev, dictBTreeFilter, btreeFactory, componentFinalizer, tmpAllBTreeFiles);
// Look for buddy deleted-keys BTrees for all valid dictionary BTrees.
// If no buddy is found, delete the file, otherwise add the dictionary BTree to allBTreeFiles.
for (ComparableFileName cmpFileName : tmpAllBTreeFiles) {
int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
String file = cmpFileName.fileName.substring(0, index);
if (deletedKeysBTreeFilesSet.contains(file)) {
- allBTreeFiles.add(cmpFileName);
+ allDictBTreeFiles.add(cmpFileName);
} else {
// Couldn't find the corresponding BTree file; thus, delete
// the deleted-keys BTree file.
// There is no need to delete the inverted-lists file corresponding to the non-existent
- // dictionary BTree, because we flush the dictionary BTree first. So if it a dictionary BTree
+ // dictionary BTree, because we flush the dictionary BTree first. So if a dictionary BTree
// file does not exists, then neither can its inverted-list file.
File invalidDeletedKeysBTreeFile = new File(cmpFileName.fullPath);
invalidDeletedKeysBTreeFile.delete();
@@ -116,48 +113,47 @@
}
}
// Sanity check.
- if (allBTreeFiles.size() != allDeletedKeysBTreeFiles.size()) {
+ if (allDictBTreeFiles.size() != allDeletedKeysBTreeFiles.size()) {
throw new HyracksDataException("Unequal number of valid RTree and BTree files found. Aborting cleanup.");
}
// Trivial cases.
- if (allBTreeFiles.isEmpty() || allDeletedKeysBTreeFiles.isEmpty()) {
+ if (allDictBTreeFiles.isEmpty() || allDeletedKeysBTreeFiles.isEmpty()) {
return validFiles;
}
- // TODO: Continue from here.
- if (allBTreeFiles.size() == 1 && allDeletedKeysBTreeFiles.size() == 1) {
- validFiles.add(new LSMInvertedIndexFileNameComponent(allRTreeFiles.get(0).fullPath,
- allBTreeFiles.get(0).fullPath));
+ if (allDictBTreeFiles.size() == 1 && allDeletedKeysBTreeFiles.size() == 1) {
+ validFiles.add(new LSMInvertedIndexFileNameComponent(allDictBTreeFiles.get(0).fullPath,
+ allDeletedKeysBTreeFiles.get(0).fullPath));
return validFiles;
}
// Sorts files names from earliest to latest timestamp.
Collections.sort(allDeletedKeysBTreeFiles);
- Collections.sort(allBTreeFiles);
+ Collections.sort(allDictBTreeFiles);
- List<ComparableFileName> validComparableRTreeFiles = new ArrayList<ComparableFileName>();
- ComparableFileName lastRTree = allRTreeFiles.get(0);
- validComparableRTreeFiles.add(lastRTree);
+ List<ComparableFileName> validComparableDictBTreeFiles = new ArrayList<ComparableFileName>();
+ ComparableFileName lastDictBTree = allDictBTreeFiles.get(0);
+ validComparableDictBTreeFiles.add(lastDictBTree);
- List<ComparableFileName> validComparableBTreeFiles = new ArrayList<ComparableFileName>();
- ComparableFileName lastBTree = allBTreeFiles.get(0);
- validComparableBTreeFiles.add(lastBTree);
+ List<ComparableFileName> validComparableDeletedKeysBTreeFiles = new ArrayList<ComparableFileName>();
+ ComparableFileName lastDeletedKeysBTree = allDeletedKeysBTreeFiles.get(0);
+ validComparableDeletedKeysBTreeFiles.add(lastDeletedKeysBTree);
- for (int i = 1; i < allRTreeFiles.size(); i++) {
- ComparableFileName currentRTree = allRTreeFiles.get(i);
- ComparableFileName currentBTree = allBTreeFiles.get(i);
+ for (int i = 1; i < allDictBTreeFiles.size(); i++) {
+ ComparableFileName currentRTree = allDictBTreeFiles.get(i);
+ ComparableFileName currentBTree = allDictBTreeFiles.get(i);
// Current start timestamp is greater than last stop timestamp.
- if (currentRTree.interval[0].compareTo(lastRTree.interval[1]) > 0
- && currentBTree.interval[0].compareTo(lastBTree.interval[1]) > 0) {
- validComparableRTreeFiles.add(currentRTree);
- validComparableBTreeFiles.add(currentBTree);
- lastRTree = currentRTree;
- lastBTree = currentBTree;
- } else if (currentRTree.interval[0].compareTo(lastRTree.interval[0]) >= 0
- && currentRTree.interval[1].compareTo(lastRTree.interval[1]) <= 0
- && currentBTree.interval[0].compareTo(lastBTree.interval[0]) >= 0
- && currentBTree.interval[1].compareTo(lastBTree.interval[1]) <= 0) {
+ if (currentRTree.interval[0].compareTo(lastDeletedKeysBTree.interval[1]) > 0
+ && currentBTree.interval[0].compareTo(lastDeletedKeysBTree.interval[1]) > 0) {
+ validComparableDictBTreeFiles.add(currentRTree);
+ validComparableDeletedKeysBTreeFiles.add(currentBTree);
+ lastDictBTree = currentRTree;
+ lastDeletedKeysBTree = currentBTree;
+ } else if (currentRTree.interval[0].compareTo(lastDictBTree.interval[0]) >= 0
+ && currentRTree.interval[1].compareTo(lastDictBTree.interval[1]) <= 0
+ && currentBTree.interval[0].compareTo(lastDeletedKeysBTree.interval[0]) >= 0
+ && currentBTree.interval[1].compareTo(lastDeletedKeysBTree.interval[1]) <= 0) {
// Invalid files are completely contained in last interval.
File invalidRTreeFile = new File(currentRTree.fullPath);
invalidRTreeFile.delete();
@@ -171,37 +167,43 @@
// Sort valid files in reverse lexicographical order, such that newer
// files come first.
- Collections.sort(validComparableRTreeFiles, recencyCmp);
- Collections.sort(validComparableBTreeFiles, recencyCmp);
+ Collections.sort(validComparableDictBTreeFiles, recencyCmp);
+ Collections.sort(validComparableDeletedKeysBTreeFiles, recencyCmp);
- Iterator<ComparableFileName> rtreeFileIter = validComparableRTreeFiles.iterator();
- Iterator<ComparableFileName> btreeFileIter = validComparableBTreeFiles.iterator();
- while (rtreeFileIter.hasNext() && btreeFileIter.hasNext()) {
- ComparableFileName cmpRTreeFileName = rtreeFileIter.next();
- ComparableFileName cmpBTreeFileName = btreeFileIter.next();
- validFiles.add(new LSMInvertedIndexFileNameComponent(cmpRTreeFileName.fullPath, cmpBTreeFileName.fullPath));
+ Iterator<ComparableFileName> dictBTreeFileIter = validComparableDictBTreeFiles.iterator();
+ Iterator<ComparableFileName> deletedKeysBTreeIter = validComparableDeletedKeysBTreeFiles.iterator();
+ while (dictBTreeFileIter.hasNext() && deletedKeysBTreeIter.hasNext()) {
+ ComparableFileName cmpDictBTreeFile = dictBTreeFileIter.next();
+ ComparableFileName cmpDeletedKeysBTreeFile = deletedKeysBTreeIter.next();
+ validFiles.add(new LSMInvertedIndexFileNameComponent(cmpDictBTreeFile.fullPath,
+ cmpDeletedKeysBTreeFile.fullPath));
}
return validFiles;
}
+ @Override
+ public String getInvListsFilePath(String dictBTreeFilePath) {
+ int index = dictBTreeFilePath.lastIndexOf(SPLIT_STRING);
+ String file = dictBTreeFilePath.substring(0, index);
+ return file + SPLIT_STRING + INVLISTS_SUFFIX;
+ }
+
+ /**
+ * No need to store the inverted-lists file name because it can be mapped from the
+ * dictionary BTree file name.
+ */
public class LSMInvertedIndexFileNameComponent {
- private final String btreeFileName;
- private final String invListsFileName;
+ private final String dictBTreeFileName;
private final String deletedKeysBTreeFileName;
- LSMInvertedIndexFileNameComponent(String btreeFileName, String invListsFileName, String deletedKeysBTreeFileName) {
- this.btreeFileName = btreeFileName;
- this.invListsFileName = invListsFileName;
+ LSMInvertedIndexFileNameComponent(String dictBTreeFileName, String deletedKeysBTreeFileName) {
+ this.dictBTreeFileName = dictBTreeFileName;
this.deletedKeysBTreeFileName = deletedKeysBTreeFileName;
}
- public String getBTreeFileName() {
- return btreeFileName;
- }
-
- public String getInvListsFileName() {
- return invListsFileName;
+ public String getDictBTreeFileName() {
+ return dictBTreeFileName;
}
public String getDeletedKeysBTreeFileName() {
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
new file mode 100644
index 0000000..3055f52
--- /dev/null
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls;
+
+import java.util.Collections;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.io.IODeviceHandle;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+
+public class LSMInvertedIndexFlushOperation implements ILSMIOOperation {
+ private final ILSMIndex index;
+ private final FileReference dictBTreeFlushTarget;
+ private final FileReference deletedKeysBTreeFlushTarget;
+ private final ILSMIOOperationCallback callback;
+
+ public LSMInvertedIndexFlushOperation(ILSMIndex index, FileReference dictBTreeFlushTarget,
+ FileReference deletedKeysBTreeFlushTarget, ILSMIOOperationCallback callback) {
+ this.index = index;
+ this.dictBTreeFlushTarget = dictBTreeFlushTarget;
+ this.deletedKeysBTreeFlushTarget = deletedKeysBTreeFlushTarget;
+ this.callback = callback;
+ }
+
+ @Override
+ public List<IODeviceHandle> getReadDevices() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<IODeviceHandle> getWriteDevices() {
+ return Collections.singletonList(dictBTreeFlushTarget.getDevideHandle());
+ }
+
+ @Override
+ public void perform() throws HyracksDataException, IndexException {
+ ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ accessor.flush(this);
+ }
+
+ @Override
+ public ILSMIOOperationCallback getCallback() {
+ return callback;
+ }
+
+ public FileReference getDictBTreeFlushTarget() {
+ return dictBTreeFlushTarget;
+ }
+
+ public FileReference getDeletedKeysBTreeFlushTarget() {
+ return deletedKeysBTreeFlushTarget;
+ }
+}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
index 3ea50e0..1e961e9 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
@@ -15,12 +15,14 @@
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
-import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInvertedIndex;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor;
public class LSMInvertedIndexOpContext implements IIndexOpContext {
@@ -28,22 +30,45 @@
private final MultiComparator cmp;
private final int invListFieldCount;
private final int tokenFieldCount;
+ private final IInvertedIndex memInvIndex;
+ private final IIndex memDeletedKeysBTree;
- public LSMInvertedIndexOpContext(IInvertedIndex memoryInvertedIndex) {
- InMemoryInvertedIndex memoryBTreeInvertedIndex = (InMemoryInvertedIndex)memoryInvertedIndex;
- BTree btree = memoryBTreeInvertedIndex.getBTree();
+ // Accessor to the in-memory inverted index.
+ public IInvertedIndexAccessor insertAccessor;
+ // Accessor to the deleted-keys BTree.
+ public IIndexAccessor deleteAccessor;
+
+ public LSMInvertedIndexOpContext(IInvertedIndex memInvIndex, IIndex memDeletedKeysBTree) {
+ this.memInvIndex = memInvIndex;
+ this.memDeletedKeysBTree = memDeletedKeysBTree;
+ /*
this.cmp = MultiComparator.create(btree.getComparatorFactories());
this.invListFieldCount = memoryBTreeInvertedIndex.getInvListCmpFactories().length;
this.tokenFieldCount = cmp.getKeyFieldCount() - invListFieldCount;
+ */
}
@Override
public void reset() {
- // TODO Auto-generated method stub
}
@Override
+ // TODO: Ignore opcallback for now.
public void reset(IndexOp newOp) {
+ switch (newOp) {
+ case INSERT: {
+ if (insertAccessor == null) {
+ insertAccessor = (IInvertedIndexAccessor) memInvIndex.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ }
+ break;
+ }
+ case DELETE: {
+ if (deleteAccessor == null) {
+ deleteAccessor = memDeletedKeysBTree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ }
+ break;
+ }
+ }
op = newOp;
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java
index 4a97465..f4f48d2 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java
@@ -81,6 +81,10 @@
return searcher;
}
+ public BTreeAccessor getBTreeAccessor() {
+ return btreeAccessor;
+ }
+
@Override
public void delete(ITupleReference tuple) throws HyracksDataException, IndexException {
throw new UnsupportedOperationException("Delete not supported by in-memory inverted index.");
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java
index bd2605c..b044ebe 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java
@@ -23,6 +23,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.IndexFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNameMapper;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@@ -33,23 +34,26 @@
protected final IBinaryComparatorFactory[] invListCmpFactories;
protected final ITypeTraits[] tokenTypeTraits;
protected final IBinaryComparatorFactory[] tokenCmpFactories;
+ protected final IInvertedIndexFileNameMapper fileNameMapper;
public OnDiskInvertedIndexFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
IInvertedListBuilder invListBuilder, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
- IBinaryComparatorFactory[] tokenCmpFactories) {
+ IBinaryComparatorFactory[] tokenCmpFactories, IInvertedIndexFileNameMapper fileNameMapper) {
super(bufferCache, fileMapProvider, null);
this.invListBuilder = invListBuilder;
this.invListTypeTraits = invListTypeTraits;
this.invListCmpFactories = invListCmpFactories;
this.tokenTypeTraits = tokenTypeTraits;
this.tokenCmpFactories = tokenCmpFactories;
+ this.fileNameMapper = fileNameMapper;
}
@Override
- public IInvertedIndex createIndexInstance(FileReference file) throws IndexException {
- FileReference btreeFile = new FileReference(new File(file.getFile().getPath() + "_btree"));
- return new OnDiskInvertedIndex(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits, invListCmpFactories,
- tokenTypeTraits, tokenCmpFactories, btreeFile, file);
+ public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws IndexException {
+ String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getPath());
+ FileReference invListsFile = new FileReference(new File(invListsFilePath));
+ return new OnDiskInvertedIndex(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits,
+ invListCmpFactories, tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile);
}
}