Reworked in-memory inverted index that buffers insertions. Still needs testing.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_inverted_index_updates_new@1804 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndex.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndex.java
new file mode 100644
index 0000000..1672c57
--- /dev/null
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndex.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.invertedindex.api;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+
+public interface IInvertedIndex extends IIndex {
+ public IInvertedListCursor createInvertedListCursor();
+
+ public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference tupleReference)
+ throws HyracksDataException, IndexException;
+
+ public IBinaryComparatorFactory[] getInvListCmpFactories();
+
+ public ITypeTraits[] getInvListTypeTraits();
+}
\ No newline at end of file
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
index 44cc1b1..34b9972 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
@@ -28,9 +28,11 @@
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
@@ -41,11 +43,13 @@
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndex;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearcher;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListCursor;
@@ -62,7 +66,7 @@
* implemented features: updates (insert/update/delete) Limitations: a query
* cannot exceed the size of a Hyracks frame.
*/
-public class InvertedIndex implements IIndex {
+public class InvertedIndex implements IInvertedIndex {
private final IHyracksCommonContext ctx = new DefaultHyracksCommonContext();
private BTree btree;
@@ -374,11 +378,11 @@
return fileId;
}
- public IBinaryComparatorFactory[] getInvListElementCmpFactories() {
+ public IBinaryComparatorFactory[] getInvListCmpFactories() {
return invListCmpFactories;
}
- public ITypeTraits[] getTypeTraits() {
+ public ITypeTraits[] getInvListTypeTraits() {
return invListTypeTraits;
}
@@ -444,7 +448,7 @@
// results during inverted index searches.
// TODO: In the future we should use the real HyracksTaskContext to track
// frame usage.
- private class DefaultHyracksCommonContext implements IHyracksCommonContext {
+ public static class DefaultHyracksCommonContext implements IHyracksCommonContext {
private final int FRAME_SIZE = 32768;
@Override
@@ -481,4 +485,49 @@
public long getInMemorySize() {
return 0;
}
+
+ @Override
+ public IInvertedListCursor createInvertedListCursor() {
+ return new FixedSizeElementInvertedListCursor(bufferCache, fileId, invListTypeTraits);
+ }
+
+ @Override
+ public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference tupleReference)
+ throws HyracksDataException, IndexException {
+ // TODO: Ideally, we should not create this objects over and over.
+ // Probably need a fancier list cursor for with.
+ RangePredicate btreePred = new RangePredicate(null, null, true, true, null, null);
+ ITreeIndexFrame leafFrame = btree.getLeafFrameFactory().createFrame();
+ ITreeIndexCursor btreeCursor = new BTreeRangeSearchCursor((IBTreeLeafFrame) leafFrame, false);
+ MultiComparator searchCmp = MultiComparator.create(btree.getComparatorFactories());
+ btreePred.setLowKeyComparator(searchCmp);
+ btreePred.setHighKeyComparator(searchCmp);
+ btreePred.setLowKey(tupleReference, true);
+ btreePred.setHighKey(tupleReference, true);
+
+ // TODO: Worry about these callbacks later.
+ ITreeIndexAccessor btreeAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ btreeAccessor.search(btreeCursor, btreePred);
+ try {
+ if (btreeCursor.hasNext()) {
+ btreeCursor.next();
+ ITupleReference frameTuple = btreeCursor.getTuple();
+ // Hardcoded mapping of btree fields
+ int startPageId = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(1),
+ frameTuple.getFieldStart(1));
+ int endPageId = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(2),
+ frameTuple.getFieldStart(2));
+ int startOff = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(3),
+ frameTuple.getFieldStart(3));
+ int numElements = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(4),
+ frameTuple.getFieldStart(4));
+ listCursor.reset(startPageId, endPageId, startOff, numElements);
+ } else {
+ listCursor.reset(0, 0, 0, 0);
+ }
+ } finally {
+ btreeCursor.close();
+ btreeCursor.reset();
+ }
+ }
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
index c5f823a..84067cb 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
@@ -35,15 +35,9 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
-import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
-import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndex;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearcher;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListCursor;
@@ -65,22 +59,16 @@
protected List<ByteBuffer> prevResultBuffers = new ArrayList<ByteBuffer>();
protected List<ByteBuffer> swap = null;
protected int maxResultBufIdx = 0;
-
- protected final ITreeIndexFrame leafFrame;
- protected final ITreeIndexFrame interiorFrame;
- protected final ITreeIndexCursor btreeCursor;
- protected final FrameTupleReference searchKey = new FrameTupleReference();
- protected final RangePredicate btreePred = new RangePredicate(null, null, true, true, null, null);
- protected final ITreeIndexAccessor btreeAccessor;
-
+
protected RecordDescriptor queryTokenRecDesc = new RecordDescriptor(
new ISerializerDeserializer[] { UTF8StringSerializerDeserializer.INSTANCE });
protected ArrayTupleBuilder queryTokenBuilder = new ArrayTupleBuilder(queryTokenRecDesc.getFieldCount());
protected DataOutput queryTokenDos = queryTokenBuilder.getDataOutput();
protected FrameTupleAppender queryTokenAppender;
protected ByteBuffer queryTokenFrame;
+ protected final FrameTupleReference searchKey = new FrameTupleReference();
- protected final InvertedIndex invIndex;
+ protected final IInvertedIndex invIndex;
protected final MultiComparator invListCmp;
protected final ITypeTraits[] invListFieldsWithCount;
protected int occurrenceThreshold;
@@ -89,16 +77,12 @@
protected List<IInvertedListCursor> invListCursorCache = new ArrayList<IInvertedListCursor>(cursorCacheSize);
protected List<IInvertedListCursor> invListCursors = new ArrayList<IInvertedListCursor>(cursorCacheSize);
- public TOccurrenceSearcher(IHyracksCommonContext ctx, InvertedIndex invIndex) {
+ public TOccurrenceSearcher(IHyracksCommonContext ctx, IInvertedIndex invIndex) {
this.ctx = ctx;
this.invIndex = invIndex;
- this.invListCmp = MultiComparator.create(invIndex.getInvListElementCmpFactories());
+ this.invListCmp = MultiComparator.create(invIndex.getInvListCmpFactories());
- leafFrame = invIndex.getBTree().getLeafFrameFactory().createFrame();
- interiorFrame = invIndex.getBTree().getInteriorFrameFactory().createFrame();
-
- btreeCursor = new BTreeRangeSearchCursor((IBTreeLeafFrame) leafFrame, false);
- ITypeTraits[] invListFields = invIndex.getTypeTraits();
+ ITypeTraits[] invListFields = invIndex.getInvListTypeTraits();
invListFieldsWithCount = new ITypeTraits[invListFields.length + 1];
int tmp = 0;
for (int i = 0; i < invListFields.length; i++) {
@@ -115,23 +99,14 @@
newResultBuffers.add(ctx.allocateFrame());
prevResultBuffers.add(ctx.allocateFrame());
- MultiComparator searchCmp = MultiComparator.create(invIndex.getBTree().getComparatorFactories());
- btreePred.setLowKeyComparator(searchCmp);
- btreePred.setHighKeyComparator(searchCmp);
- btreePred.setLowKey(searchKey, true);
- btreePred.setHighKey(searchKey, true);
-
- // pre-create cursor objects
+ // Pre-create cursor objects.
for (int i = 0; i < cursorCacheSize; i++) {
- invListCursorCache.add(new FixedSizeElementInvertedListCursor(invIndex.getBufferCache(), invIndex
- .getInvListsFileId(), invIndex.getTypeTraits()));
+ invListCursorCache.add(invIndex.createInvertedListCursor());
}
queryTokenAppender = new FrameTupleAppender(ctx.getFrameSize());
queryTokenFrame = ctx.allocateFrame();
- btreeAccessor = invIndex.getBTree().createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
currentNumResults = 0;
}
@@ -175,19 +150,18 @@
queryTokenAccessor.reset(queryTokenFrame);
int numQueryTokens = queryTokenAccessor.getTupleCount();
- // expand cursor cache if necessary
+ // Expand cursor cache if necessary.
if (numQueryTokens > invListCursorCache.size()) {
int diff = numQueryTokens - invListCursorCache.size();
for (int i = 0; i < diff; i++) {
- invListCursorCache.add(new FixedSizeElementInvertedListCursor(invIndex.getBufferCache(), invIndex
- .getInvListsFileId(), invIndex.getTypeTraits()));
+ invListCursorCache.add(invIndex.createInvertedListCursor());
}
}
invListCursors.clear();
for (int i = 0; i < numQueryTokens; i++) {
searchKey.reset(queryTokenAccessor, i);
- invIndex.openCursor(btreeCursor, btreePred, btreeAccessor, invListCursorCache.get(i));
+ invIndex.openInvertedListCursor(invListCursorCache.get(i), searchKey);
invListCursors.add(invListCursorCache.get(i));
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixProbeOnly.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixProbeOnly.java
index 957ecc0..143b95d 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixProbeOnly.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixProbeOnly.java
@@ -30,7 +30,7 @@
public TOccurrenceSearcherSuffixProbeOnly(IHyracksTaskContext ctx, InvertedIndex invIndex) {
super(ctx, invIndex);
- this.invListCmp = MultiComparator.create(invIndex.getInvListElementCmpFactories());
+ this.invListCmp = MultiComparator.create(invIndex.getInvListCmpFactories());
}
protected int mergeSuffixLists(int numPrefixTokens, int numQueryTokens, int maxPrevBufIdx) throws HyracksDataException {
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixScanOnly.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixScanOnly.java
index bd9bd60..a0ad224 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixScanOnly.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixScanOnly.java
@@ -31,7 +31,7 @@
public TOccurrenceSearcherSuffixScanOnly(IHyracksTaskContext ctx, InvertedIndex invIndex) {
super(ctx, invIndex);
- this.invListCmp = MultiComparator.create(invIndex.getInvListElementCmpFactories());
+ this.invListCmp = MultiComparator.create(invIndex.getInvListCmpFactories());
}
protected int mergeSuffixLists(int numPrefixTokens, int numQueryTokens, int maxPrevBufIdx) throws HyracksDataException {
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index e1a745c..2a0be2f 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -65,7 +65,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMMergeOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexComponentFinalizer;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@@ -84,10 +84,10 @@
// On-disk components.
private final ILSMFileManager fileManager;
// For creating BTree's used in flush and merge.
- private final TreeFactory<BTree> diskBTreeFactory;
+ private final TreeIndexFactory<BTree> diskBTreeFactory;
// For creating BTree's used in bulk load. Different from diskBTreeFactory
// because it should have a different tuple writer in it's leaf frames.
- private final TreeFactory<BTree> bulkLoadBTreeFactory;
+ private final TreeIndexFactory<BTree> bulkLoadBTreeFactory;
private final IBufferCache diskBufferCache;
private final IFileMapProvider diskFileMapProvider;
// List of BTree instances. Using Object for better sharing via ILSMTree + LSMHarness.
@@ -105,7 +105,7 @@
public LSMBTree(IBufferCache memBufferCache, InMemoryFreePageManager memFreePageManager,
ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory,
ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMFileManager fileNameManager,
- TreeFactory<BTree> diskBTreeFactory, TreeFactory<BTree> bulkLoadBTreeFactory,
+ TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory,
IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
ILSMFlushController flushController, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler) {
@@ -309,7 +309,7 @@
return createDiskBTree(bulkLoadBTreeFactory, fileRef, true);
}
- private BTree createDiskBTree(TreeFactory<BTree> factory, FileReference fileRef, boolean createBTree)
+ private BTree createDiskBTree(TreeIndexFactory<BTree> factory, FileReference fileRef, boolean createBTree)
throws HyracksDataException {
// Create new BTree instance.
BTree diskBTree = factory.createIndexInstance(fileRef);
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
index 3b535f4..97c5d6a 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
@@ -39,7 +39,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeFileManager;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@@ -62,10 +62,10 @@
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
IFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
metaFrameFactory);
- TreeFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories,
typeTraits.length);
- TreeFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
ILSMFileManager fileNameManager = new LSMTreeFileManager(ioManager, diskFileMapProvider, file, diskBTreeFactory);
LSMBTree lsmTree = new LSMBTree(memBufferCache, memFreePageManager, interiorFrameFactory,
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BTreeFactory.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
index a0d80c8..70f8986 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
@@ -23,7 +23,7 @@
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-public class BTreeFactory extends TreeFactory<BTree> {
+public class BTreeFactory extends TreeIndexFactory<BTree> {
public BTreeFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
IFreePageManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory,
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/IndexFactory.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/IndexFactory.java
new file mode 100644
index 0000000..ac81515
--- /dev/null
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/IndexFactory.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
+
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManagerFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+
+public abstract class IndexFactory<T extends IIndex> {
+
+ protected final IBufferCache bufferCache;
+ protected final IFileMapProvider fileMapProvider;
+ protected final IFreePageManagerFactory freePageManagerFactory;
+
+ public IndexFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ IFreePageManagerFactory freePageManagerFactory) {
+ this.bufferCache = bufferCache;
+ this.fileMapProvider = fileMapProvider;
+ this.freePageManagerFactory = freePageManagerFactory;
+ }
+
+ public abstract T createIndexInstance(FileReference file);
+
+ public IBufferCache getBufferCache() {
+ return bufferCache;
+ }
+}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java
index 4f2e1df..ea7483f 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java
@@ -42,7 +42,7 @@
// where to flush and merge
protected final IIOManager ioManager;
protected final IFileMapProvider fileMapProvider;
- protected final TreeFactory<? extends ITreeIndex> treeFactory;
+ protected final TreeIndexFactory<? extends ITreeIndex> treeFactory;
// baseDir should reflect dataset name and partition name.
protected FileReference file;
@@ -61,7 +61,7 @@
};
public LSMTreeFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
- TreeFactory<? extends ITreeIndex> treeFactory) {
+ TreeIndexFactory<? extends ITreeIndex> treeFactory) {
this.file = file;
this.baseDir = file.getFile().getPath();
if (!baseDir.endsWith(System.getProperty("file.separator"))) {
@@ -154,7 +154,7 @@
}
protected void cleanupAndGetValidFilesInternal(IODeviceHandle dev, FilenameFilter filter,
- TreeFactory<? extends ITreeIndex> treeFactory, ILSMComponentFinalizer componentFinalizer,
+ TreeIndexFactory<? extends ITreeIndex> treeFactory, ILSMComponentFinalizer componentFinalizer,
ArrayList<ComparableFileName> allFiles) throws HyracksDataException {
File dir = new File(dev.getPath(), baseDir);
String[] files = dir.list(filter);
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java
similarity index 80%
rename from hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java
rename to hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java
index 7a50a3b..1ea174d 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java
@@ -23,28 +23,17 @@
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-public abstract class TreeFactory<T extends ITreeIndex> {
-
- protected final IBufferCache bufferCache;
-
- protected final IFileMapProvider fileMapProvider;
-
- protected final IFreePageManagerFactory freePageManagerFactory;
+public abstract class TreeIndexFactory<T extends ITreeIndex> extends IndexFactory<T> {
protected final ITreeIndexFrameFactory interiorFrameFactory;
-
protected final ITreeIndexFrameFactory leafFrameFactory;
-
protected final IBinaryComparatorFactory[] cmpFactories;
-
protected final int fieldCount;
- public TreeFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ public TreeIndexFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
IFreePageManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory,
ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
- this.bufferCache = bufferCache;
- this.fileMapProvider = fileMapProvider;
- this.freePageManagerFactory = freePageManagerFactory;
+ super(bufferCache, fileMapProvider, freePageManagerFactory);
this.interiorFrameFactory = interiorFrameFactory;
this.leafFrameFactory = leafFrameFactory;
this.cmpFactories = cmpFactories;
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndex.java
index 4c68cb5..7bb9cfb 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndex.java
@@ -14,198 +14,148 @@
*/
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls;
+import java.io.File;
import java.io.IOException;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeDuplicateKeyException;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndex;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListCursor;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizer;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IToken;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-public class InMemoryBtreeInvertedIndex implements IIndex {
+public class InMemoryBtreeInvertedIndex implements IInvertedIndex {
private final BTree btree;
- private final IIndexAccessor btreeAccessor;
-// private final ITypeTraits[] tokenTypeTraits;
-// private final IBinaryComparatorFactory[] tokenCmpFactories;
- private final ITypeTraits[] invertedListTypeTraits;
- private final IBinaryComparatorFactory[] invertedListCmpFactories;
+ private final FileReference memBTreeFile = new FileReference(new File("memBTree"));
+ private final ITypeTraits[] tokenTypeTraits;
+ private final IBinaryComparatorFactory[] tokenCmpFactories;
+ private final ITypeTraits[] invListTypeTraits;
+ private final IBinaryComparatorFactory[] invListCmpFactories;
private final IBinaryTokenizer tokenizer;
- private final int numTokenFields;
- private final ArrayTupleBuilder btreeTupleBuilder;
- private final ArrayTupleReference btreeTupleReference;
+ private final ITypeTraits[] btreeTypeTraits;
+ private final IBinaryComparatorFactory[] btreeCmpFactories;
-// public InMemoryBtreeInvertedIndex(IBufferCache inMemBufferCache, IFreePageManager inMemFreePageManager,
-// ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
-// ITypeTraits[] invertedListTypeTraits, IBinaryComparatorFactory[] invertedListCmpFactories,
-// IBinaryTokenizer tokenizer) {
-// this.tokenTypeTraits = tokenTypeTraits;
-// this.tokenCmpFactories = tokenCmpFactories;
-// this.invertedListTypeTraits = invertedListTypeTraits;
-// this.invertedListCmpFactories = invertedListCmpFactories;
-// this.tokenizer = tokenizer;
-//
-// this.btreeTupleBuilder = new ArrayTupleBuilder(btree.getFieldCount());
-// this.btreeTupleReference = new ArrayTupleReference();
-// }
-
- public InMemoryBtreeInvertedIndex(BTree btree, ITypeTraits[] invListTypeTraits,
- IBinaryComparatorFactory[] invListCmpFactories, IBinaryTokenizer tokenizer) {
- // membufcache, tokenCmpFactories, memfpmgr, tokentypetraits
- //
- // IBufferCache bufferCache, int fieldCount, IBinaryComparatorFactory[] cmpFactories, IFreePageManager freePageManager,
- // ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory
- this.btree = btree;
- this.btreeAccessor = btree.createAccessor();
- this.invertedListTypeTraits = invListTypeTraits;
- this.invertedListCmpFactories = invListCmpFactories;
+ public InMemoryBtreeInvertedIndex(IBufferCache memBufferCache, IFreePageManager memFreePageManager,
+ ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
+ ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizer tokenizer)
+ throws BTreeException {
+ this.tokenTypeTraits = tokenTypeTraits;
+ this.tokenCmpFactories = tokenCmpFactories;
+ this.invListTypeTraits = invListTypeTraits;
+ this.invListCmpFactories = invListCmpFactories;
this.tokenizer = tokenizer;
- this.numTokenFields = btree.getComparatorFactories().length - invListCmpFactories.length;
-
- // To generate in-memory BTree tuples
- this.btreeTupleBuilder = new ArrayTupleBuilder(btree.getFieldCount());
- this.btreeTupleReference = new ArrayTupleReference();
+ // BTree tuples: <tokens, inverted-list elements>.
+ int numBTreeFields = tokenTypeTraits.length + invListTypeTraits.length;
+ btreeTypeTraits = new ITypeTraits[numBTreeFields];
+ btreeCmpFactories = new IBinaryComparatorFactory[numBTreeFields];
+ for (int i = 0; i < tokenTypeTraits.length; i++) {
+ btreeTypeTraits[i] = tokenTypeTraits[i];
+ btreeCmpFactories[i] = tokenCmpFactories[i];
+ }
+ for (int i = 0; i < invListTypeTraits.length; i++) {
+ btreeTypeTraits[tokenTypeTraits.length + i] = invListTypeTraits[i];
+ btreeCmpFactories[tokenTypeTraits.length + i] = invListCmpFactories[i];
+ }
+ this.btree = BTreeUtils.createBTree(memBufferCache,
+ ((InMemoryBufferCache) memBufferCache).getFileMapProvider(), btreeTypeTraits, btreeCmpFactories,
+ BTreeLeafFrameType.REGULAR_NSM, memBTreeFile);
}
@Override
- public void create() throws HyracksDataException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void activate() throws HyracksDataException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void clear() throws HyracksDataException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void deactivate() throws HyracksDataException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void destroy() throws HyracksDataException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public IIndexAccessor createAccessor(
- IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback) {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public void validate() throws HyracksDataException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public long getInMemorySize() {
- // TODO Auto-generated method stub
- return 0;
- }
-
- @Override
- public IIndexBulkLoader createBulkLoader(float fillFactor,
- boolean verifyInput) throws IndexException {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public void open(int fileId) {
- btree.open(fileId);
+ public void create() throws HyracksDataException {
+ btree.create();
}
@Override
- public void create(int indexFileId) throws HyracksDataException {
- btree.create(indexFileId);
+ public void activate() throws HyracksDataException {
+ btree.activate();
}
@Override
- public void close() {
- btree.close();
+ public void clear() throws HyracksDataException {
+ btree.clear();
}
- public boolean insertUpdateOrDelete(ITupleReference tuple, IIndexOpContext ictx) throws HyracksDataException,
- IndexException {
- LSMInvertedIndexOpContext ctx = (LSMInvertedIndexOpContext) ictx;
+ @Override
+ public void deactivate() throws HyracksDataException {
+ btree.deactivate();
+ }
- //Tuple --> |Field1|Field2| ... |FieldN|doc-id|
- //Each field represents a document and doc-id always comes at the last field.
- //parse document
- //create a list of (term,doc-id)
- //sort the list in the order of term
- //insert a pair of (term, doc-id) into in-memory BTree until to the end of the list.
+ @Override
+ public void destroy() throws HyracksDataException {
+ btree.destroy();
+ }
- for (int i = 0; i < numTokenFields; i++) {
- tokenizer.reset(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
- while (tokenizer.hasNext()) {
- tokenizer.next();
- IToken token = tokenizer.getToken();
- btreeTupleBuilder.reset();
+ @Override
+ public void validate() throws HyracksDataException {
+ btree.validate();
+ }
- try {
- token.serializeToken(btreeTupleBuilder.getDataOutput());
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- btreeTupleBuilder.addFieldEndOffset();
-
- // This doesn't work for some reason.
- // btreeTupleBuilder.addField(token.getData(), token.getStart(), token.getTokenLength());
-
- btreeTupleBuilder.addField(tuple.getFieldData(0), tuple.getFieldStart(1), tuple.getFieldLength(1));
- btreeTupleReference.reset(btreeTupleBuilder.getFieldEndOffsets(), btreeTupleBuilder.getByteArray());
-
- try {
- btreeAccessor.insert(btreeTupleReference);
- } catch (BTreeDuplicateKeyException e) {
- // Consciously ignoring... guarantees uniqueness!
- // When duplication occurs, the current insert can be simply ignored
- // since the current inverted list stores doc-id only.
- // TODO
- // We may work around this duplication issue by pre-processing the inserted document.
- // This pre-processing will generate only unique <term, doc-id> pair for each document.
- // Therefore there will be no duplication in in-memory BTree.
- }
+ public boolean insert(ITupleReference tuple, BTreeAccessor btreeAccessor, IIndexOpContext ictx)
+ throws HyracksDataException, IndexException {
+ InMemoryBtreeInvertedIndexOpContext ctx = (InMemoryBtreeInvertedIndexOpContext) ictx;
+ // TODO: We can probably avoid copying the data into a new tuple here.
+ tokenizer.reset(tuple.getFieldData(0), tuple.getFieldStart(0), tuple.getFieldLength(0));
+ while (tokenizer.hasNext()) {
+ tokenizer.next();
+ IToken token = tokenizer.getToken();
+ ctx.btreeTupleBuilder.reset();
+ // Add token field.
+ try {
+ token.serializeToken(ctx.btreeTupleBuilder.getDataOutput());
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ ctx.btreeTupleBuilder.addFieldEndOffset();
+ // Add inverted-list element fields.
+ for (int i = 0; i < invListTypeTraits.length; i++) {
+ ctx.btreeTupleBuilder.addField(tuple.getFieldData(i + 1), tuple.getFieldStart(i + 1),
+ tuple.getFieldLength(i + 1));
+ }
+ // Reset tuple reference for insert operation.
+ ctx.btreeTupleReference.reset(ctx.btreeTupleBuilder.getFieldEndOffsets(),
+ ctx.btreeTupleBuilder.getByteArray());
+ try {
+ btreeAccessor.insert(ctx.btreeTupleReference);
+ } catch (BTreeDuplicateKeyException e) {
+ // This exception may be caused by duplicate tokens in the same insert "document".
+ // We ignore such duplicate tokens in all inverted-index implementations, hence
+ // we can safely ignore this exception.
}
}
return true;
}
@Override
+ public long getInMemorySize() {
+ InMemoryBufferCache memBufferCache = (InMemoryBufferCache) btree.getBufferCache();
+ return memBufferCache.getNumPages() * memBufferCache.getPageSize();
+ }
+
+ @Override
public IInvertedListCursor createInvertedListCursor() {
- return new InMemoryBtreeInvertedListCursor(btree, invertedListTypeTraits);
+ return new InMemoryBtreeInvertedListCursor(btree, invListTypeTraits);
}
@Override
@@ -216,23 +166,10 @@
}
@Override
- public IIndexAccessor createAccessor() {
- return new InMemoryBtreeInvertedIndexAccessor(this, new LSMInvertedIndexOpContext(this), tokenizer);
- }
-
- @Override
- public IIndexBulkLoadContext beginBulkLoad(float fillFactor) throws IndexException, HyracksDataException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void endBulkLoad(IIndexBulkLoadContext ictx) throws HyracksDataException {
- throw new UnsupportedOperationException();
+ public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
+ ISearchOperationCallback searchCallback) {
+ return new InMemoryBtreeInvertedIndexAccessor(this, new InMemoryBtreeInvertedIndexOpContext(
+ btreeTypeTraits.length), tokenizer);
}
@Override
@@ -245,17 +182,22 @@
return IndexType.INVERTED;
}
- @Override
- public IBinaryComparatorFactory[] getInvListElementCmpFactories() {
- return invertedListCmpFactories;
- }
-
- @Override
- public ITypeTraits[] getTypeTraits() {
- return invertedListTypeTraits;
- }
-
public BTree getBTree() {
return btree;
}
+
+ @Override
+ public IBinaryComparatorFactory[] getInvListCmpFactories() {
+ return invListCmpFactories;
+ }
+
+ @Override
+ public ITypeTraits[] getInvListTypeTraits() {
+ return invListTypeTraits;
+ }
+
+ @Override
+ public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput) throws IndexException {
+ throw new UnsupportedOperationException("Bulk load not supported by in-memory inverted index.");
+ }
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndexAccessor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndexAccessor.java
index 1187027..5ef3bb6 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndexAccessor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndexAccessor.java
@@ -1,48 +1,58 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls;
import edu.uci.ics.hyracks.api.context.IHyracksCommonContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearcher;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex.DefaultHyracksCommonContext;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndexSearchCursor;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndexSearchPredicate;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.TOccurrenceSearcher;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizer;
public class InMemoryBtreeInvertedIndexAccessor implements IIndexAccessor {
- private final IHyracksCommonContext hyracksCtx = new DefaultHyracksCommonContext();
- private final IInvertedIndexSearcher searcher;
- protected IIndexOpContext ctx;
+ // TODO: This ctx needs to go away.
+ protected final IHyracksCommonContext hyracksCtx = new DefaultHyracksCommonContext();
+ protected final IInvertedIndexSearcher searcher;
+ protected IIndexOpContext opCtx;
protected InMemoryBtreeInvertedIndex memoryBtreeInvertedIndex;
+ protected BTreeAccessor btreeAccessor;
- public InMemoryBtreeInvertedIndexAccessor(InMemoryBtreeInvertedIndex memoryBtreeInvertedIndex, IIndexOpContext ctx,
+ public InMemoryBtreeInvertedIndexAccessor(InMemoryBtreeInvertedIndex memoryBtreeInvertedIndex, IIndexOpContext opCtx,
IBinaryTokenizer tokenizer) {
- this.ctx = ctx;
+ this.opCtx = opCtx;
this.memoryBtreeInvertedIndex = memoryBtreeInvertedIndex;
- this.searcher = new TOccurrenceSearcher(hyracksCtx, memoryBtreeInvertedIndex, tokenizer);
+ this.searcher = new TOccurrenceSearcher(hyracksCtx, memoryBtreeInvertedIndex);
+ // TODO: Ignore opcallbacks for now.
+ this.btreeAccessor = (BTreeAccessor) memoryBtreeInvertedIndex.getBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
}
public void insert(ITupleReference tuple) throws HyracksDataException, IndexException {
- ctx.reset(IndexOp.INSERT);
- memoryBtreeInvertedIndex.insertUpdateOrDelete(tuple, ctx);
- }
-
- @Override
- public void update(ITupleReference tuple) throws HyracksDataException, IndexException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void delete(ITupleReference tuple) throws HyracksDataException, IndexException {
- // TODO Auto-generated method stub
-
+ opCtx.reset(IndexOp.INSERT);
+ memoryBtreeInvertedIndex.insert(tuple, btreeAccessor, opCtx);
}
@Override
@@ -55,4 +65,18 @@
searcher.search((InvertedIndexSearchCursor) cursor, (InvertedIndexSearchPredicate) searchPred);
}
+ @Override
+ public void delete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ throw new UnsupportedOperationException("Delete not supported by in-memory inverted index.");
+ }
+
+ @Override
+ public void update(ITupleReference tuple) throws HyracksDataException, IndexException {
+ throw new UnsupportedOperationException("Update not supported by in-memory inverted index.");
+ }
+
+ @Override
+ public void upsert(ITupleReference tuple) throws HyracksDataException, IndexException {
+ throw new UnsupportedOperationException("Upsert not supported by in-memory inverted index.");
+ }
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndexOpContext.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndexOpContext.java
new file mode 100644
index 0000000..150d74e
--- /dev/null
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndexOpContext.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls;
+
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+
+public class InMemoryBtreeInvertedIndexOpContext implements IIndexOpContext {
+ public IndexOp op;
+ public final int numBTreeFields;
+
+ // To generate in-memory btree tuples for insertions.
+ public ArrayTupleBuilder btreeTupleBuilder;
+ public ArrayTupleReference btreeTupleReference;
+
+ public InMemoryBtreeInvertedIndexOpContext(int numBTreeFields) {
+ this.numBTreeFields = numBTreeFields;
+ }
+
+ @Override
+ public void reset() {
+ op = null;
+ }
+
+ @Override
+ public void reset(IndexOp newOp) {
+ switch (newOp) {
+ case INSERT: {
+ btreeTupleBuilder = new ArrayTupleBuilder(numBTreeFields);
+ btreeTupleReference = new ArrayTupleReference();
+ break;
+ }
+ case SEARCH: {
+ break;
+ }
+ default: {
+ throw new UnsupportedOperationException("Unsupported operation " + newOp);
+ }
+ }
+ op = newOp;
+ }
+}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InvertedIndexFactory.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InvertedIndexFactory.java
index 526f2a4..2f4bac6 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InvertedIndexFactory.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InvertedIndexFactory.java
@@ -16,24 +16,24 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManagerFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndex;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizer;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFileManager;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.IndexFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-public class InvertedIndexFactory<T extends IInvertedIndex> {
+public class InvertedIndexFactory extends IndexFactory<IIndex> {
- protected IBufferCache bufferCache;
protected ITypeTraits[] invListTypeTraits;
protected IBinaryComparatorFactory[] invListCmpFactories;
protected IInvertedListBuilder invListBuilder;
@@ -48,10 +48,11 @@
protected ILSMFileManager fileManager;
- public InvertedIndexFactory(IBufferCache bufferCache, ITypeTraits[] invListTypeTraits,
+ public InvertedIndexFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ IFreePageManagerFactory freePageManagerFactory, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, IInvertedListBuilder invListBuilder,
IBinaryTokenizer tokenizer, ILSMFileManager fileManager) {
- this.bufferCache = bufferCache;
+ super(bufferCache, fileMapProvider, freePageManagerFactory);
this.invListTypeTraits = invListTypeTraits;
this.invListCmpFactories = invListCmpFactories;
this.invListBuilder = invListBuilder;
@@ -72,12 +73,9 @@
this.fileManager = fileManager;
}
- public T createIndexInstance(BTree btree) {
- return (T) new InvertedIndex(bufferCache, btree, invListTypeTraits, invListCmpFactories, invListBuilder,
- tokenizer);
- }
-
- public IBufferCache getBufferCache() {
- return bufferCache;
+ @Override
+ public IIndex createIndexInstance(FileReference file) {
+ return new InvertedIndex(bufferCache, btree, invListTypeTraits, invListCmpFactories, invListBuilder,
+ tokenizer); ;
}
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 477b940..63f73a4 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -18,74 +18,123 @@
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.http.TokenIterator;
+import java.util.logging.Logger;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndex;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListCursor;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFinalizer;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFileManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexFileManager.LSMInvertedFileNameComponent;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-public class LSMInvertedIndex implements ILSMIndex, IInvertedIndex {
+public class LSMInvertedIndex implements ILSMIndex, IIndex {
+ private final Logger LOGGER = Logger.getLogger(LSMInvertedIndex.class.getName());
+
+ public class LSMInvertedIndexComponent {
+ private final IIndex invIndex;
+ private final BTree deleteKeysBTree;
- private final LSMHarness lsmHarness;
- private final IInvertedIndex memoryInvertedIndex;
- private final BTreeFactory diskBTreeFactory;
- private final InvertedIndexFactory diskInvertedIndexFactory;
- private final ILSMFileManager fileManager;
- private final IFileMapProvider diskFileMapProvider;
- private final ILSMComponentFinalizer componentFinalizer;
- private LinkedList<Object> diskInvertedIndexList = new LinkedList<Object>();
- private final IBufferCache diskBufferCache;
+ LSMInvertedIndexComponent(IIndex invIndex, BTree deleteKeysBTree) {
+ this.invIndex = invIndex;
+ this.deleteKeysBTree = deleteKeysBTree;
+ }
- private final IIndexAccessor memAccessor;
+ public IIndex getInvIndex() {
+ return invIndex;
+ }
- private boolean isOpen;
+ public BTree getDeletedKeysBTree() {
+ return deleteKeysBTree;
+ }
+ }
+
+ protected final LSMHarness lsmHarness;
+
+ // In-memory components.
+ protected final LSMInvertedIndexComponent memComponent;
+ protected final IBufferCache memBufferCache;
+ protected final InMemoryFreePageManager memFreePageManager;
- public LSMInvertedIndex(IInvertedIndex memoryInvertedIndex, BTreeFactory diskBTreeFactory,
- InvertedIndexFactory diskInvertedIndexFactory, ILSMFileManager fileManager,
- IFileMapProvider diskFileMapProvider) {
- this.memoryInvertedIndex = memoryInvertedIndex;
- this.diskBTreeFactory = diskBTreeFactory;
- this.diskInvertedIndexFactory = diskInvertedIndexFactory;
+ // On-disk components.
+ protected final ILSMFileManager fileManager;
+ // For creating inverted indexes in flush and merge.
+ protected final InvertedIndexFactory diskInvIndexFactory;
+ protected final IBufferCache diskBufferCache;
+ protected final IFileMapProvider diskFileMapProvider;
+ // List of LSMInvertedIndexComponent instances. Using Object for better sharing via
+ // ILSMIndex + LSMHarness.
+ protected final LinkedList<Object> diskComponents = new LinkedList<Object>();
+ // Helps to guarantees physical consistency of LSM components.
+ protected final ILSMComponentFinalizer componentFinalizer;
+
+ // Type traits and comparators for tokens and inverted-list elements.
+ protected final ITypeTraits[] invListTypeTraits;
+ protected final IBinaryComparatorFactory[] invListCmpFactories;
+ protected final ITypeTraits[] tokenTypeTraits;
+ protected final IBinaryComparatorFactory[] tokenCmpFactories;
+
+ private boolean isActivated = false;
+
+ public LSMInvertedIndex(IBufferCache memBufferCache, InMemoryFreePageManager memFreePageManager, InvertedIndexFactory diskInvIndexFactory,
+ ILSMFileManager fileManager, IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits,
+ IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
+ IBinaryComparatorFactory[] tokenCmpFactories, ILSMFlushController flushController,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler) {
+ // TODO: Finish this one properly.
+ InMemoryBtreeInvertedIndex memInvIndex = null;
+ BTree deleteKeysBTree = null;
+ memComponent = new LSMInvertedIndexComponent(memInvIndex, deleteKeysBTree);
+ this.memBufferCache = memBufferCache;
+ this.memFreePageManager = memFreePageManager;
this.fileManager = fileManager;
+ this.diskInvIndexFactory = diskInvIndexFactory;
+ this.diskBufferCache = diskInvIndexFactory.getBufferCache();
this.diskFileMapProvider = diskFileMapProvider;
- this.lsmHarness = new LSMHarness(this);
+ this.invListTypeTraits = invListTypeTraits;
+ this.invListCmpFactories = invListCmpFactories;
+ this.tokenTypeTraits = tokenTypeTraits;
+ this.tokenCmpFactories = tokenCmpFactories;
+
+ this.lsmHarness = new LSMHarness(this, flushController, mergePolicy, opTracker, ioScheduler);
this.componentFinalizer = new InvertedIndexComponentFinalizer(diskFileMapProvider);
- this.diskBufferCache = diskBTreeFactory.getBufferCache();
- this.memAccessor = memoryInvertedIndex.createAccessor();
}
@Override
- public void create(int indexFileId) throws HyracksDataException {
+ public synchronized void create(int indexFileId) throws HyracksDataException {
+ if (isActivated) {
+ return;
+ }
+
// TODO: What else is needed here?
memoryInvertedIndex.create(indexFileId);
fileManager.createDirs();
@@ -429,12 +478,104 @@
@Override
public IBinaryComparatorFactory[] getInvListElementCmpFactories() {
- return memoryInvertedIndex.getInvListElementCmpFactories();
+ return memoryInvertedIndex.getInvListCmpFactories();
}
@Override
public ITypeTraits[] getTypeTraits() {
- return memoryInvertedIndex.getTypeTraits();
+ return memoryInvertedIndex.getInvListTypeTraits();
+ }
+
+ @Override
+ public void create() throws HyracksDataException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void activate() throws HyracksDataException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void clear() throws HyracksDataException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void deactivate() throws HyracksDataException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void destroy() throws HyracksDataException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
+ ISearchOperationCallback searchCallback) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public void validate() throws HyracksDataException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public long getInMemorySize() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput) throws IndexException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public ILSMIOOperation createMergeOperation(ILSMIOOperationCallback callback) throws HyracksDataException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Object merge(List<Object> mergedComponents, ILSMIOOperation operation) throws HyracksDataException,
+ IndexException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Object flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public ILSMFlushController getFlushController() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public ILSMOperationTracker getOperationTracker() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public ILSMIOOperationScheduler getIOScheduler() {
+ // TODO Auto-generated method stub
+ return null;
}
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
index 53f7d3f..99f625c 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
@@ -32,7 +32,7 @@
InMemoryBtreeInvertedIndex memoryBTreeInvertedIndex = (InMemoryBtreeInvertedIndex)memoryInvertedIndex;
BTree btree = memoryBTreeInvertedIndex.getBTree();
this.cmp = MultiComparator.create(btree.getComparatorFactories());
- this.invListFieldCount = memoryBTreeInvertedIndex.getInvListElementCmpFactories().length;
+ this.invListFieldCount = memoryBTreeInvertedIndex.getInvListCmpFactories().length;
this.tokenFieldCount = cmp.getKeyFieldCount() - invListFieldCount;
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index 9473fef..a3a7eca 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -49,7 +49,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMHarness;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
@@ -99,9 +99,9 @@
protected final IBufferCache diskBufferCache;
protected final IFileMapProvider diskFileMapProvider;
// For creating RTree's used in flush and merge.
- protected final TreeFactory<RTree> diskRTreeFactory;
+ protected final TreeIndexFactory<RTree> diskRTreeFactory;
// List of LSMRTreeComponent instances. Using Object for better sharing via
- // ILSMTree + LSMHarness.
+ // ILSMIndex + LSMHarness.
protected final LinkedList<Object> diskComponents = new LinkedList<Object>();
// Helps to guarantees physical consistency of LSM components.
protected final ILSMComponentFinalizer componentFinalizer;
@@ -120,7 +120,7 @@
public AbstractLSMRTree(IBufferCache memBufferCache, InMemoryFreePageManager memFreePageManager,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
- ILSMFileManager fileManager, TreeFactory<RTree> diskRTreeFactory, IFileMapProvider diskFileMapProvider,
+ ILSMFileManager fileManager, TreeIndexFactory<RTree> diskRTreeFactory, IFileMapProvider diskFileMapProvider,
ILSMComponentFinalizer componentFinalizer, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMFlushController flushController,
@@ -221,7 +221,7 @@
}
@SuppressWarnings("rawtypes")
- protected ITreeIndex createDiskTree(TreeFactory diskTreeFactory, FileReference fileRef, boolean createTree)
+ protected ITreeIndex createDiskTree(TreeIndexFactory diskTreeFactory, FileReference fileRef, boolean createTree)
throws HyracksDataException {
// Create new tree instance.
ITreeIndex diskTree = diskTreeFactory.createIndexInstance(fileRef);
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index db5c197..8627d32 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -51,7 +51,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTreeFileManager.LSMRTreeFileNameComponent;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree.RTreeBulkLoader;
@@ -64,12 +64,12 @@
// On-disk components.
// For creating BTree's used in flush and merge.
- private final TreeFactory<BTree> diskBTreeFactory;
+ private final TreeIndexFactory<BTree> diskBTreeFactory;
public LSMRTree(IBufferCache memBufferCache, InMemoryFreePageManager memFreePageManager,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
- ILSMFileManager fileManager, TreeFactory<RTree> diskRTreeFactory, TreeFactory<BTree> diskBTreeFactory,
+ ILSMFileManager fileManager, TreeIndexFactory<RTree> diskRTreeFactory, TreeIndexFactory<BTree> diskBTreeFactory,
IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMFlushController flushController,
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
index 59dcf2e..03a8d97 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
@@ -30,15 +30,15 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFinalizer;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeFileManager;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class LSMRTreeFileManager extends LSMTreeFileManager {
private static final String RTREE_STRING = "r";
private static final String BTREE_STRING = "b";
- private final TreeFactory<? extends ITreeIndex> rtreeFactory;
- private final TreeFactory<? extends ITreeIndex> btreeFactory;
+ private final TreeIndexFactory<? extends ITreeIndex> rtreeFactory;
+ private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
private static FilenameFilter btreeFilter = new FilenameFilter() {
public boolean accept(File dir, String name) {
@@ -53,7 +53,7 @@
};
public LSMRTreeFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
- TreeFactory<? extends ITreeIndex> rtreeFactory, TreeFactory<? extends ITreeIndex> btreeFactory) {
+ TreeIndexFactory<? extends ITreeIndex> rtreeFactory, TreeIndexFactory<? extends ITreeIndex> btreeFactory) {
super(ioManager, fileMapProvider, file, null);
this.rtreeFactory = rtreeFactory;
this.btreeFactory = btreeFactory;
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index 37ee010..d5855b5 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -54,7 +54,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMMergeOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexComponentFinalizer;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree.RTreeBulkLoader;
@@ -70,12 +70,12 @@
// On-disk components.
// For creating RTree's used in bulk load. Different from diskRTreeFactory
// because it should have a different tuple writer in it's leaf frames.
- private final TreeFactory<RTree> bulkLoadRTreeFactory;
+ private final TreeIndexFactory<RTree> bulkLoadRTreeFactory;
public LSMRTreeWithAntiMatterTuples(IBufferCache memBufferCache, InMemoryFreePageManager memFreePageManager,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
- ILSMFileManager fileManager, TreeFactory<RTree> diskRTreeFactory, TreeFactory<RTree> bulkLoadRTreeFactory,
+ ILSMFileManager fileManager, TreeIndexFactory<RTree> diskRTreeFactory, TreeIndexFactory<RTree> bulkLoadRTreeFactory,
IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMFlushController flushController,
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
index 1f2dabf..4adbffb 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
@@ -19,12 +19,12 @@
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManagerFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-public class RTreeFactory extends TreeFactory<RTree> {
+public class RTreeFactory extends TreeIndexFactory<RTree> {
public RTreeFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
IFreePageManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory,
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
index f49cc87..0902f4d 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
@@ -39,7 +39,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeFileManager;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTree;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTreeFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTreeWithAntiMatterTuples;
@@ -80,10 +80,10 @@
LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
metaFrameFactory);
- TreeFactory<RTree> diskRTreeFactory = new RTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(diskBufferCache, diskFileMapProvider,
freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories,
typeTraits.length);
- TreeFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
freePageManagerFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, btreeCmpFactories,
typeTraits.length);
@@ -129,11 +129,11 @@
LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
metaFrameFactory);
- TreeFactory<RTree> diskRTreeFactory = new RTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(diskBufferCache, diskFileMapProvider,
freePageManagerFactory, rtreeInteriorFrameFactory, copyTupleLeafFrameFactory, rtreeCmpFactories,
typeTraits.length);
- TreeFactory<RTree> bulkLoadRTreeFactory = new RTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<RTree> bulkLoadRTreeFactory = new RTreeFactory(diskBufferCache, diskFileMapProvider,
freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories,
typeTraits.length);
diff --git a/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/DummyTreeFactory.java b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/DummyTreeFactory.java
index 0b955cd..d6494f8 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/DummyTreeFactory.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/DummyTreeFactory.java
@@ -2,9 +2,9 @@
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
-public class DummyTreeFactory extends TreeFactory<ITreeIndex> {
+public class DummyTreeFactory extends TreeIndexFactory<ITreeIndex> {
public DummyTreeFactory() {
super(null, null, null, null, null, null, 0);