Improved design and significantly reduced object creation of in-memory inverted index.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_inverted_index_updates_new@1807 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
index d8dcbef..a30d17d 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
@@ -907,6 +907,11 @@
public BTreeOpContext getOpContext() {
return ctx;
}
+
+ public ITreeIndexCursor createCountingSearchCursor() {
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
+ return new BTreeCountingSearchCursor(leafFrame, false);
+ }
}
@Override
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
new file mode 100644
index 0000000..5429007
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
@@ -0,0 +1,246 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.btree.impls;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleMode;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+
+public class BTreeCountingSearchCursor implements ITreeIndexCursor {
+
+ private int fileId = -1;
+ private ICachedPage page = null;
+ private IBufferCache bufferCache = null;
+
+ private int tupleIndex = 0;
+ private int stopTupleIndex;
+ private int count = -1;
+
+ private FindTupleMode lowKeyFtm;
+ private FindTupleMode highKeyFtm;
+
+ private FindTupleNoExactMatchPolicy lowKeyFtp;
+ private FindTupleNoExactMatchPolicy highKeyFtp;
+
+ private final IBTreeLeafFrame frame;
+ private final ITreeIndexTupleReference frameTuple;
+ private final boolean exclusiveLatchNodes;
+
+ private RangePredicate pred;
+ private MultiComparator lowKeyCmp;
+ private MultiComparator highKeyCmp;
+ private ITupleReference lowKey;
+ private ITupleReference highKey;
+
+ // For storing the count.
+ private byte[] countBuf = new byte[4];
+ private ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(1);
+ private ArrayTupleReference countTuple = new ArrayTupleReference();
+
+ public BTreeCountingSearchCursor(IBTreeLeafFrame frame, boolean exclusiveLatchNodes) {
+ this.frame = frame;
+ this.frameTuple = frame.createTupleReference();
+ this.exclusiveLatchNodes = exclusiveLatchNodes;
+ }
+
+ @Override
+ public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ // in case open is called multiple times without closing
+ if (page != null) {
+ if (exclusiveLatchNodes) {
+ page.releaseWriteLatch();
+ } else {
+ page.releaseReadLatch();
+ }
+ bufferCache.unpin(page);
+ }
+
+ page = ((BTreeCursorInitialState) initialState).getPage();
+ frame.setPage(page);
+
+ pred = (RangePredicate) searchPred;
+ lowKeyCmp = pred.getLowKeyComparator();
+ highKeyCmp = pred.getHighKeyComparator();
+
+ lowKey = pred.getLowKey();
+ highKey = pred.getHighKey();
+
+ // init
+ lowKeyFtm = FindTupleMode.EXCLUSIVE;
+ if (pred.lowKeyInclusive) {
+ lowKeyFtp = FindTupleNoExactMatchPolicy.LOWER_KEY;
+ } else {
+ lowKeyFtp = FindTupleNoExactMatchPolicy.HIGHER_KEY;
+ }
+
+ highKeyFtm = FindTupleMode.EXCLUSIVE;
+ if (pred.highKeyInclusive) {
+ highKeyFtp = FindTupleNoExactMatchPolicy.HIGHER_KEY;
+ } else {
+ highKeyFtp = FindTupleNoExactMatchPolicy.LOWER_KEY;
+ }
+
+ tupleIndex = getLowKeyIndex();
+ stopTupleIndex = getHighKeyIndex();
+ }
+
+ private void fetchNextLeafPage(int nextLeafPage) throws HyracksDataException {
+ do {
+ ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false);
+ if (exclusiveLatchNodes) {
+ nextLeaf.acquireWriteLatch();
+ page.releaseWriteLatch();
+ } else {
+ nextLeaf.acquireReadLatch();
+ page.releaseReadLatch();
+ }
+ bufferCache.unpin(page);
+ page = nextLeaf;
+ frame.setPage(page);
+ nextLeafPage = frame.getNextLeaf();
+ } while (frame.getTupleCount() == 0 && nextLeafPage > 0);
+ }
+
+ private int getLowKeyIndex() throws HyracksDataException {
+ if (lowKey == null) {
+ return 0;
+ }
+ int index = frame.findTupleIndex(lowKey, frameTuple, lowKeyCmp, lowKeyFtm, lowKeyFtp);
+ if (pred.lowKeyInclusive) {
+ index++;
+ } else {
+ if (index < 0) {
+ index = frame.getTupleCount();
+ }
+ }
+ return index;
+ }
+
+ private int getHighKeyIndex() throws HyracksDataException {
+ if (highKey == null) {
+ return frame.getTupleCount() - 1;
+ }
+ int index = frame.findTupleIndex(highKey, frameTuple, highKeyCmp, highKeyFtm, highKeyFtp);
+ if (pred.highKeyInclusive) {
+ if (index < 0) {
+ index = frame.getTupleCount() - 1;
+ } else {
+ index--;
+ }
+ }
+ return index;
+ }
+
+ @Override
+ public boolean hasNext() throws HyracksDataException {
+ // get the count for the current page
+ // follow the sibling pointer until last page
+ // if no more tuples on a page, then done
+
+ if (count < 0) {
+ count = 0;
+
+ while (stopTupleIndex >= 0) {
+ count += (stopTupleIndex - tupleIndex + 1);
+
+ int nextLeafPage = frame.getNextLeaf();
+ if (nextLeafPage >= 0) {
+ fetchNextLeafPage(nextLeafPage);
+ } else {
+ // No more pages. Done counting!
+ break;
+ }
+
+ tupleIndex = 0;
+ stopTupleIndex = getHighKeyIndex();
+ }
+
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ public void next() throws HyracksDataException {
+ // Do nothing. Count is performed just once!
+ IntegerSerializerDeserializer.putInt(count, countBuf, 0);
+ tupleBuilder.addField(countBuf, 0, 4);
+ countTuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ if (page != null) {
+ if (exclusiveLatchNodes) {
+ page.releaseWriteLatch();
+ } else {
+ page.releaseReadLatch();
+ }
+ bufferCache.unpin(page);
+ }
+ tupleIndex = 0;
+ page = null;
+ pred = null;
+ }
+
+ @Override
+ public void reset() {
+ try {
+ close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public ITupleReference getTuple() {
+ return countTuple;
+ }
+
+ @Override
+ public ICachedPage getPage() {
+ return page;
+ }
+
+ @Override
+ public void setBufferCache(IBufferCache bufferCache) {
+ this.bufferCache = bufferCache;
+ }
+
+ @Override
+ public void setFileId(int fileId) {
+ this.fileId = fileId;
+ }
+
+ @Override
+ public boolean exclusiveLatchNodes() {
+ return exclusiveLatchNodes;
+ }
+
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/ConcatenatingTupleReference.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/ConcatenatingTupleReference.java
new file mode 100644
index 0000000..25218be
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/ConcatenatingTupleReference.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.dataflow;
+
+import java.util.Arrays;
+
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+/**
+ * WARNING: getFieldData(), getFieldStart() and getFieldLength() are log and not constant time.
+ */
+public class ConcatenatingTupleReference implements ITupleReference {
+
+ private final ITupleReference[] tuples;
+ private final int[] fieldCounts;
+ private int numTuples;
+ private int totalFieldCount;
+
+ public ConcatenatingTupleReference(int maxTuples) {
+ tuples = new ITupleReference[maxTuples];
+ fieldCounts = new int[maxTuples];
+ reset();
+ }
+
+ public void reset() {
+ numTuples = 0;
+ totalFieldCount = 0;
+ }
+
+ public void addTuple(ITupleReference tuple) {
+ tuples[numTuples] = tuple;
+ totalFieldCount += tuple.getFieldCount();
+ if (numTuples > 0) {
+ fieldCounts[numTuples] = fieldCounts[numTuples - 1] + tuple.getFieldCount();
+ } else {
+ fieldCounts[numTuples] = tuple.getFieldCount();
+ }
+ ++numTuples;
+ }
+
+ public void removeLastTuple() {
+ if (numTuples > 0) {
+ ITupleReference lastTuple = tuples[numTuples--];
+ totalFieldCount -= lastTuple.getFieldCount();
+ }
+ }
+
+ @Override
+ public int getFieldCount() {
+ return totalFieldCount;
+ }
+
+ @Override
+ public byte[] getFieldData(int fIdx) {
+ int tupleIndex = getTupleIndex(fIdx);
+ int fieldIndex = getFieldIndex(tupleIndex, fIdx);
+ return tuples[tupleIndex].getFieldData(fieldIndex);
+ }
+
+ @Override
+ public int getFieldStart(int fIdx) {
+ int tupleIndex = getTupleIndex(fIdx);
+ int fieldIndex = getFieldIndex(tupleIndex, fIdx);
+ return tuples[tupleIndex].getFieldStart(fieldIndex);
+ }
+
+ @Override
+ public int getFieldLength(int fIdx) {
+ int tupleIndex = getTupleIndex(fIdx);
+ int fieldIndex = getFieldIndex(tupleIndex, fIdx);
+ return tuples[tupleIndex].getFieldLength(fieldIndex);
+ }
+
+ private int getTupleIndex(int fIdx) {
+ int tupleIndex = Arrays.binarySearch(fieldCounts, 0, numTuples - 1, fIdx);
+ if (tupleIndex < 0) {
+ tupleIndex = -tupleIndex - 1;
+ } else {
+ ++tupleIndex;
+ }
+ return tupleIndex;
+ }
+
+ private int getFieldIndex(int tupleIndex, int fIdx) {
+ int fieldIndex = -1;
+ if (tupleIndex > 0) {
+ fieldIndex = fIdx - fieldCounts[tupleIndex - 1];
+ } else {
+ fieldIndex = fIdx;
+ }
+ return fieldIndex;
+ }
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/PermutingTupleReference.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/PermutingTupleReference.java
new file mode 100644
index 0000000..db50135
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/PermutingTupleReference.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.dataflow;
+
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+public class PermutingTupleReference implements ITupleReference {
+
+ private final int[] fieldPermutation;
+ private ITupleReference sourceTuple;
+
+ public PermutingTupleReference(int[] fieldPermutation) {
+ this.fieldPermutation = fieldPermutation;
+ }
+
+ public void reset(ITupleReference sourceTuple) {
+ this.sourceTuple = sourceTuple;
+ }
+
+ @Override
+ public int getFieldCount() {
+ return fieldPermutation.length;
+ }
+
+ @Override
+ public byte[] getFieldData(int fIdx) {
+ return sourceTuple.getFieldData(fieldPermutation[fIdx]);
+ }
+
+ @Override
+ public int getFieldStart(int fIdx) {
+ return sourceTuple.getFieldStart(fieldPermutation[fIdx]);
+ }
+
+ @Override
+ public int getFieldLength(int fIdx) {
+ return sourceTuple.getFieldLength(fieldPermutation[fIdx]);
+ }
+}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndex.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndex.java
index 1672c57..2851c0f 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndex.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndex.java
@@ -19,13 +19,14 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
public interface IInvertedIndex extends IIndex {
public IInvertedListCursor createInvertedListCursor();
- public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference tupleReference)
+ public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey, IIndexOpContext ictx)
throws HyracksDataException, IndexException;
public IBinaryComparatorFactory[] getInvListCmpFactories();
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearcher.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearcher.java
index d633c34..bc6104f 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearcher.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearcher.java
@@ -21,12 +21,13 @@
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndexSearchCursor;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndexSearchPredicate;
public interface IInvertedIndexSearcher {
- public void search(InvertedIndexSearchCursor resultCursor, InvertedIndexSearchPredicate searchPred)
+ public void search(InvertedIndexSearchCursor resultCursor, InvertedIndexSearchPredicate searchPred, IIndexOpContext ictx)
throws HyracksDataException, IndexException;
public IFrameTupleAccessor createResultFrameTupleAccessor();
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedListCursor.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedListCursor.java
index 9435f3c..596c265 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedListCursor.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedListCursor.java
@@ -29,9 +29,9 @@
void unpinPages() throws HyracksDataException;
- boolean hasNext();
+ boolean hasNext() throws HyracksDataException;
- void next();
+ void next() throws HyracksDataException;
ITupleReference getTuple();
@@ -44,10 +44,7 @@
int getStartOff();
- // jump to a specific element
- void positionCursor(int elementIx);
-
- boolean containsKey(ITupleReference searchTuple, MultiComparator invListCmp);
+ boolean containsKey(ITupleReference searchTuple, MultiComparator invListCmp) throws HyracksDataException;
// for debugging
String printInvList(ISerializerDeserializer[] serdes) throws HyracksDataException;
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListCursor.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListCursor.java
index 446f171..431ba55 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListCursor.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListCursor.java
@@ -92,8 +92,7 @@
}
}
- @Override
- public void positionCursor(int elementIx) {
+ private void positionCursor(int elementIx) {
int numPages = endPageId - startPageId + 1;
currentPageIx = binarySearch(elementIndexes, 0, numPages, elementIx);
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
index 34b9972..707ae42 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
@@ -28,26 +28,23 @@
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
-import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndex;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearcher;
@@ -392,32 +389,13 @@
public class InvertedIndexAccessor implements IIndexAccessor {
private final IInvertedIndexSearcher searcher;
+ private final IIndexOpContext opCtx = new InvertedIndexOpContext(btree);
public InvertedIndexAccessor(InvertedIndex index) {
this.searcher = new TOccurrenceSearcher(ctx, index);
}
@Override
- public void insert(ITupleReference tuple) throws HyracksDataException, IndexException {
- // TODO Auto-generated method stub
- }
-
- @Override
- public void update(ITupleReference tuple) throws HyracksDataException, IndexException {
- // TODO Auto-generated method stub
- }
-
- @Override
- public void delete(ITupleReference tuple) throws HyracksDataException, IndexException {
- // TODO Auto-generated method stub
- }
-
- @Override
- public void upsert(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
- // TODO Auto-generated method stub
- }
-
- @Override
public IIndexCursor createSearchCursor() {
return new InvertedIndexSearchCursor(searcher);
}
@@ -425,12 +403,32 @@
@Override
public void search(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException,
IndexException {
- searcher.search((InvertedIndexSearchCursor) cursor, (InvertedIndexSearchPredicate) searchPred);
+ searcher.search((InvertedIndexSearchCursor) cursor, (InvertedIndexSearchPredicate) searchPred, opCtx);
}
public IInvertedIndexSearcher getSearcher() {
return searcher;
}
+
+ @Override
+ public void insert(ITupleReference tuple) throws HyracksDataException, IndexException {
+ throw new UnsupportedOperationException("Insert not supported by inverted index.");
+ }
+
+ @Override
+ public void update(ITupleReference tuple) throws HyracksDataException, IndexException {
+ throw new UnsupportedOperationException("Update not supported by inverted index.");
+ }
+
+ @Override
+ public void delete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ throw new UnsupportedOperationException("Delete not supported by inverted index.");
+ }
+
+ @Override
+ public void upsert(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
+ throw new UnsupportedOperationException("Upsert not supported by inverted index.");
+ }
}
@Override
@@ -472,7 +470,7 @@
try {
return new InvertedIndexBulkLoader(fillFactor, verifyInput, rootPageId, fileId);
} catch (HyracksDataException e) {
- throw new IndexException(e);
+ throw new InvertedIndexException(e);
}
}
@@ -492,27 +490,19 @@
}
@Override
- public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference tupleReference)
+ public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey, IIndexOpContext ictx)
throws HyracksDataException, IndexException {
- // TODO: Ideally, we should not create this objects over and over.
- // Probably need a fancier list cursor for with.
- RangePredicate btreePred = new RangePredicate(null, null, true, true, null, null);
- ITreeIndexFrame leafFrame = btree.getLeafFrameFactory().createFrame();
- ITreeIndexCursor btreeCursor = new BTreeRangeSearchCursor((IBTreeLeafFrame) leafFrame, false);
- MultiComparator searchCmp = MultiComparator.create(btree.getComparatorFactories());
- btreePred.setLowKeyComparator(searchCmp);
- btreePred.setHighKeyComparator(searchCmp);
- btreePred.setLowKey(tupleReference, true);
- btreePred.setHighKey(tupleReference, true);
-
- // TODO: Worry about these callbacks later.
- ITreeIndexAccessor btreeAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- btreeAccessor.search(btreeCursor, btreePred);
+ InvertedIndexOpContext ctx = (InvertedIndexOpContext) ictx;
+ ctx.btreePred.setLowKeyComparator(ctx.searchCmp);
+ ctx.btreePred.setHighKeyComparator(ctx.searchCmp);
+ ctx.btreePred.setLowKey(searchKey, true);
+ ctx.btreePred.setHighKey(searchKey, true);
+ ctx.btreeAccessor.search(ctx.btreeCursor, ctx.btreePred);
try {
- if (btreeCursor.hasNext()) {
- btreeCursor.next();
- ITupleReference frameTuple = btreeCursor.getTuple();
- // Hardcoded mapping of btree fields
+ if (ctx.btreeCursor.hasNext()) {
+ ctx.btreeCursor.next();
+ ITupleReference frameTuple = ctx.btreeCursor.getTuple();
+ // Hardcoded mapping of btree fields.
int startPageId = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(1),
frameTuple.getFieldStart(1));
int endPageId = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(2),
@@ -526,8 +516,8 @@
listCursor.reset(0, 0, 0, 0);
}
} finally {
- btreeCursor.close();
- btreeCursor.reset();
+ ctx.btreeCursor.close();
+ ctx.btreeCursor.reset();
}
}
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexException.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexException.java
index 9762a74..54567a5 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexException.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexException.java
@@ -20,6 +20,10 @@
public class InvertedIndexException extends IndexException {
private static final long serialVersionUID = 1L;
+ public InvertedIndexException(Exception e) {
+ super(e);
+ }
+
public InvertedIndexException(String msg) {
super(msg);
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexOpContext.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexOpContext.java
new file mode 100644
index 0000000..eb22aaf
--- /dev/null
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexOpContext.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
+
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+
+public class InvertedIndexOpContext implements IIndexOpContext {
+
+ public final RangePredicate btreePred = new RangePredicate(null, null, true, true, null, null);
+ public IIndexAccessor btreeAccessor;
+ public IIndexCursor btreeCursor;
+ public MultiComparator searchCmp;
+
+ public InvertedIndexOpContext(BTree btree) {
+ // TODO: Ignore opcallbacks for now.
+ btreeAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ btreeCursor = btreeAccessor.createSearchCursor();
+ searchCmp = MultiComparator.create(btree.getComparatorFactories());
+ }
+
+ @Override
+ public void reset() {
+ // Nothing to be done here, only search operation supported.
+ }
+
+ @Override
+ public void reset(IndexOp newOp) {
+ // Nothing to be done here, only search operation supported.
+ }
+}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
index 84067cb..2ddfa31 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
@@ -35,6 +35,7 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndex;
@@ -59,7 +60,7 @@
protected List<ByteBuffer> prevResultBuffers = new ArrayList<ByteBuffer>();
protected List<ByteBuffer> swap = null;
protected int maxResultBufIdx = 0;
-
+
protected RecordDescriptor queryTokenRecDesc = new RecordDescriptor(
new ISerializerDeserializer[] { UTF8StringSerializerDeserializer.INSTANCE });
protected ArrayTupleBuilder queryTokenBuilder = new ArrayTupleBuilder(queryTokenRecDesc.getFieldCount());
@@ -120,8 +121,8 @@
currentNumResults = 0;
}
- public void search(InvertedIndexSearchCursor resultCursor, InvertedIndexSearchPredicate searchPred)
- throws HyracksDataException, IndexException {
+ public void search(InvertedIndexSearchCursor resultCursor, InvertedIndexSearchPredicate searchPred,
+ IIndexOpContext ictx) throws HyracksDataException, IndexException {
ITupleReference queryTuple = searchPred.getQueryTuple();
int queryFieldIndex = searchPred.getQueryFieldIndex();
IInvertedIndexSearchModifier searchModifier = searchPred.getSearchModifier();
@@ -161,7 +162,7 @@
invListCursors.clear();
for (int i = 0; i < numQueryTokens; i++) {
searchKey.reset(queryTokenAccessor, i);
- invIndex.openInvertedListCursor(invListCursorCache.get(i), searchKey);
+ invIndex.openInvertedListCursor(invListCursorCache.get(i), searchKey, ictx);
invListCursors.add(invListCursorCache.get(i));
}
@@ -217,7 +218,7 @@
}
protected int mergeSuffixListProbe(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers,
- int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) {
+ int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) throws HyracksDataException {
int newBufIdx = 0;
ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
@@ -262,7 +263,8 @@
}
protected int mergeSuffixListScan(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers,
- int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) {
+ int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens)
+ throws HyracksDataException {
int newBufIdx = 0;
ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
@@ -356,7 +358,7 @@
}
protected int mergePrefixList(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers,
- int maxPrevBufIdx, List<ByteBuffer> newResultBuffers) {
+ int maxPrevBufIdx, List<ByteBuffer> newResultBuffers) throws HyracksDataException {
int newBufIdx = 0;
ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixProbeOnly.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixProbeOnly.java
index 143b95d..e25be73 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixProbeOnly.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixProbeOnly.java
@@ -49,7 +49,7 @@
}
protected int mergeSuffixListProbe(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers,
- int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) {
+ int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) throws HyracksDataException {
int newBufIdx = 0;
ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixScanOnly.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixScanOnly.java
index a0ad224..131dc48 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixScanOnly.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixScanOnly.java
@@ -50,7 +50,7 @@
}
protected int mergeSuffixListScan(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers,
- int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) {
+ int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) throws HyracksDataException {
int newBufIdx = 0;
ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndex.java
index 7bb9cfb..8b648ae 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndex.java
@@ -115,7 +115,7 @@
public boolean insert(ITupleReference tuple, BTreeAccessor btreeAccessor, IIndexOpContext ictx)
throws HyracksDataException, IndexException {
InMemoryBtreeInvertedIndexOpContext ctx = (InMemoryBtreeInvertedIndexOpContext) ictx;
- // TODO: We can probably avoid copying the data into a new tuple here.
+ // TODO: We can possibly avoid copying the data into a new tuple here.
tokenizer.reset(tuple.getFieldData(0), tuple.getFieldStart(0), tuple.getFieldLength(0));
while (tokenizer.hasNext()) {
tokenizer.next();
@@ -155,13 +155,17 @@
@Override
public IInvertedListCursor createInvertedListCursor() {
- return new InMemoryBtreeInvertedListCursor(btree, invListTypeTraits);
+ return new InMemoryBtreeInvertedListCursor(invListTypeTraits.length, tokenTypeTraits.length);
}
@Override
- public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference tupleReference)
+ public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference tupleReference, IIndexOpContext ictx)
throws HyracksDataException, IndexException {
+ InMemoryBtreeInvertedIndexOpContext ctx = (InMemoryBtreeInvertedIndexOpContext) ictx;
InMemoryBtreeInvertedListCursor inMemListCursor = (InMemoryBtreeInvertedListCursor) listCursor;
+ inMemListCursor.prepare(ctx.btreeAccessor, ctx.btreePred, ctx.tokenFieldsCmp, ctx.btreeCmp);
+
+
inMemListCursor.reset(tupleReference);
}
@@ -169,7 +173,7 @@
public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
return new InMemoryBtreeInvertedIndexAccessor(this, new InMemoryBtreeInvertedIndexOpContext(
- btreeTypeTraits.length), tokenizer);
+ btree, tokenCmpFactories), tokenizer);
}
@Override
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndexAccessor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndexAccessor.java
index 5ef3bb6..cc74b9e 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndexAccessor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndexAccessor.java
@@ -41,13 +41,14 @@
protected InMemoryBtreeInvertedIndex memoryBtreeInvertedIndex;
protected BTreeAccessor btreeAccessor;
- public InMemoryBtreeInvertedIndexAccessor(InMemoryBtreeInvertedIndex memoryBtreeInvertedIndex, IIndexOpContext opCtx,
- IBinaryTokenizer tokenizer) {
+ public InMemoryBtreeInvertedIndexAccessor(InMemoryBtreeInvertedIndex memoryBtreeInvertedIndex,
+ IIndexOpContext opCtx, IBinaryTokenizer tokenizer) {
this.opCtx = opCtx;
this.memoryBtreeInvertedIndex = memoryBtreeInvertedIndex;
this.searcher = new TOccurrenceSearcher(hyracksCtx, memoryBtreeInvertedIndex);
// TODO: Ignore opcallbacks for now.
- this.btreeAccessor = (BTreeAccessor) memoryBtreeInvertedIndex.getBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ this.btreeAccessor = (BTreeAccessor) memoryBtreeInvertedIndex.getBTree().createAccessor(
+ NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
}
public void insert(ITupleReference tuple) throws HyracksDataException, IndexException {
@@ -62,19 +63,19 @@
@Override
public void search(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException, IndexException {
- searcher.search((InvertedIndexSearchCursor) cursor, (InvertedIndexSearchPredicate) searchPred);
+ searcher.search((InvertedIndexSearchCursor) cursor, (InvertedIndexSearchPredicate) searchPred, opCtx);
}
@Override
public void delete(ITupleReference tuple) throws HyracksDataException, IndexException {
throw new UnsupportedOperationException("Delete not supported by in-memory inverted index.");
}
-
+
@Override
public void update(ITupleReference tuple) throws HyracksDataException, IndexException {
throw new UnsupportedOperationException("Update not supported by in-memory inverted index.");
}
-
+
@Override
public void upsert(ITupleReference tuple) throws HyracksDataException, IndexException {
throw new UnsupportedOperationException("Upsert not supported by in-memory inverted index.");
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndexOpContext.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndexOpContext.java
index 150d74e..eb75b88 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndexOpContext.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedIndexOpContext.java
@@ -15,37 +15,54 @@
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
public class InMemoryBtreeInvertedIndexOpContext implements IIndexOpContext {
public IndexOp op;
- public final int numBTreeFields;
-
- // To generate in-memory btree tuples for insertions.
+ public final BTree btree;
+
+ // Needed for search operations,
+ public RangePredicate btreePred;
+ public BTreeAccessor btreeAccessor;
+ public MultiComparator btreeCmp;
+ public IBinaryComparatorFactory[] tokenCmpFactories;
+ public MultiComparator tokenFieldsCmp;
+
+ // To generate in-memory BTree tuples for insertions.
public ArrayTupleBuilder btreeTupleBuilder;
public ArrayTupleReference btreeTupleReference;
-
- public InMemoryBtreeInvertedIndexOpContext(int numBTreeFields) {
- this.numBTreeFields = numBTreeFields;
- }
-
- @Override
- public void reset() {
- op = null;
+
+ public InMemoryBtreeInvertedIndexOpContext(BTree btree, IBinaryComparatorFactory[] tokenCmpFactories) {
+ this.btree = btree;
+ this.tokenCmpFactories = tokenCmpFactories;
}
@Override
public void reset(IndexOp newOp) {
switch (newOp) {
case INSERT: {
- btreeTupleBuilder = new ArrayTupleBuilder(numBTreeFields);
- btreeTupleReference = new ArrayTupleReference();
+ btreeTupleBuilder = new ArrayTupleBuilder(btree.getFieldCount());
+ btreeTupleReference = new ArrayTupleReference();
break;
}
case SEARCH: {
+ if (btreePred == null) {
+ btreePred = new RangePredicate(null, null, true, true, null, null);
+ // TODO: Ignore opcallbacks for now.
+ btreeAccessor = (BTreeAccessor) btree.createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ btreeCmp = MultiComparator.create(btree.getComparatorFactories());
+ tokenFieldsCmp = MultiComparator.create(tokenCmpFactories);
+ }
break;
}
default: {
@@ -54,4 +71,9 @@
}
op = newOp;
}
+
+ @Override
+ public void reset() {
+ op = null;
+ }
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedListCursor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedListCursor.java
index 76457a7..ffb0169 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedListCursor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/InMemoryBtreeInvertedListCursor.java
@@ -14,67 +14,72 @@
*/
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls;
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeCountingSearchCursor;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.ConcatenatingTupleReference;
import edu.uci.ics.hyracks.storage.am.common.dataflow.PermutingTupleReference;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListCursor;
public class InMemoryBtreeInvertedListCursor implements IInvertedListCursor {
- private final BTree btree;
- private final RangePredicate btreePred;
- private final IBTreeLeafFrame leafFrame;
- private final ITreeIndexAccessor btreeAccessor;
- private BTreeRangeSearchCursor btreeCursor;
+ private RangePredicate btreePred;
+ private BTreeAccessor btreeAccessor;
+ private IIndexCursor btreeCursor;
+ private IIndexCursor countingCursor;
private MultiComparator tokenFieldsCmp;
-
+ private MultiComparator btreeCmp;
+ private final PermutingTupleReference resultTuple;
+ private final ConcatenatingTupleReference btreeSearchTuple;
+
+ private final ArrayTupleBuilder tokenTupleBuilder;
+ private final ArrayTupleReference tokenTuple = new ArrayTupleReference();
+
private int numElements = -1;
- private ITupleReference tokenTuple;
-
- public InMemoryBtreeInvertedListCursor(BTree btree, ITypeTraits[] invListFields) {
- this.btree = btree;
- this.btreeAccessor = btree.createAccessor();
- this.btreePred = new RangePredicate(null, null, true, true, null, null);
- this.leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
- this.btreeCursor = new BTreeRangeSearchCursor(leafFrame, false);
- setTokenFieldComparators(btree.getComparatorFactories(), btree.getComparatorFactories().length
- - invListFields.length);
- btreePred.setLowKeyComparator(tokenFieldsCmp);
- btreePred.setHighKeyComparator(tokenFieldsCmp);
+
+ public InMemoryBtreeInvertedListCursor(int invListFieldCount, int tokenFieldCount) {
+ int[] fieldPermutation = new int[invListFieldCount];
+ for (int i = 0; i < invListFieldCount; i++) {
+ fieldPermutation[i] = tokenFieldCount + i;
+ }
+ resultTuple = new PermutingTupleReference(fieldPermutation);
+ // Concatenating the tuple with tokens, and the tuple with inverted-list elements.
+ btreeSearchTuple = new ConcatenatingTupleReference(2);
+ tokenTupleBuilder = new ArrayTupleBuilder(tokenFieldCount);
}
-
- private void setTokenFieldComparators(IBinaryComparatorFactory[] cmpFactories, int numTokenFields) {
- IBinaryComparatorFactory[] keyFieldCmpFactories = new IBinaryComparatorFactory[numTokenFields];
- System.arraycopy(cmpFactories, 0, keyFieldCmpFactories, 0, numTokenFields);
- tokenFieldsCmp = MultiComparator.create(keyFieldCmpFactories);
+
+ public void prepare(BTreeAccessor btreeAccessor, RangePredicate btreePred, MultiComparator tokenFieldsCmp,
+ MultiComparator btreeCmp) {
+ this.btreeAccessor = btreeAccessor;
+ this.btreeCursor = btreeAccessor.createSearchCursor();
+ this.countingCursor = btreeAccessor.createCountingSearchCursor();
+ this.btreePred = btreePred;
+ this.btreePred.setLowKeyComparator(tokenFieldsCmp);
+ this.btreePred.setHighKeyComparator(tokenFieldsCmp);
+ this.tokenFieldsCmp = tokenFieldsCmp;
+ this.btreeCmp = btreeCmp;
}
-
+
@Override
public int compareTo(IInvertedListCursor cursor) {
return getNumElements() - cursor.getNumElements();
}
- public void reset(ITupleReference tuple) throws HyracksDataException, IndexException {
+ public void reset(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
numElements = -1;
- tokenTuple = TupleUtils.copyTuple(tuple);
- btreeCursor = (BTreeRangeSearchCursor) btreeAccessor.createSearchCursor();
+ // Copy the tokens tuple for later use in btree probes.
+ TupleUtils.copyTuple(tokenTupleBuilder, tuple, tuple.getFieldCount());
+ tokenTuple.reset(tokenTupleBuilder.getFieldEndOffsets(), tokenTupleBuilder.getByteArray());
+ btreeSearchTuple.addTuple(tokenTuple);
btreePred.setLowKey(tuple, true);
btreePred.setHighKey(tuple, true);
btreeAccessor.search(btreeCursor, btreePred);
@@ -110,54 +115,40 @@
btreeCursor.next();
}
- public ITupleReference getTuple() throws HyracksDataException {
- PermutingTupleReference projectedTuple = new PermutingTupleReference();
- ITupleReference tuple = btreeCursor.getTuple();
- int tupleFieldCount = tuple.getFieldCount();
- int tokensFieldCount = tokenFieldsCmp.getKeyFieldCount();
-
- int[] fEndOffsets = new int[tupleFieldCount];
- int[] fieldPermutation = new int[tupleFieldCount - tokensFieldCount];
-
- for (int i = 0; i < tupleFieldCount; i++) {
- fEndOffsets[i] = tuple.getFieldStart(i) + tuple.getFieldLength(i);
- }
-
- for (int i = 0; i < fieldPermutation.length; i++) {
- fieldPermutation[i] = tokensFieldCount + i;
- }
-
- projectedTuple.reset(fEndOffsets, fieldPermutation, tuple.getFieldData(0));
-
- return projectedTuple;
+ @Override
+ public ITupleReference getTuple() {
+ resultTuple.reset(btreeCursor.getTuple());
+ return resultTuple;
}
@Override
public int getNumElements() {
if (numElements < 0) {
- // perform the count
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
- BTreeCountingSearchCursor countCursor = new BTreeCountingSearchCursor(leafFrame, false);
- RangePredicate predicate = new RangePredicate(tokenTuple, tokenTuple, true, true, tokenFieldsCmp,
- tokenFieldsCmp);
+ btreePred.setLowKeyComparator(tokenFieldsCmp);
+ btreePred.setHighKeyComparator(tokenFieldsCmp);
+ btreePred.setLowKey(tokenTuple, true);
+ btreePred.setHighKey(tokenTuple, true);
+
+ // Perform the count.
try {
- btreeAccessor.search(countCursor, predicate);
- while (countCursor.hasNext()) {
- countCursor.next();
- ITupleReference countTuple = countCursor.getTuple();
- ByteArrayInputStream bais = new ByteArrayInputStream(countTuple.getFieldData(0));
- DataInputStream dis = new DataInputStream(bais);
- numElements = IntegerSerializerDeserializer.INSTANCE.deserialize(dis).intValue();
+ btreeAccessor.search(countingCursor, btreePred);
+ while (countingCursor.hasNext()) {
+ countingCursor.next();
+ ITupleReference countTuple = countingCursor.getTuple();
+ numElements = IntegerSerializerDeserializer.getInt(countTuple.getFieldData(0), countTuple.getFieldStart(0));
}
- countCursor.close();
} catch (HyracksDataException e) {
e.printStackTrace();
} catch (IndexException e) {
e.printStackTrace();
+ } finally {
+ try {
+ countingCursor.close();
+ } catch (HyracksDataException e) {
+ e.printStackTrace();
+ }
}
-
}
-
return numElements;
}
@@ -177,38 +168,38 @@
}
@Override
- public boolean containsKey(ITupleReference searchTuple, MultiComparator invListCmp) throws HyracksDataException,
- IndexException {
- int numCompositeFields = tokenTuple.getFieldCount() + invListCmp.getKeyFieldCount();
- ArrayTupleBuilder tb = new ArrayTupleBuilder(numCompositeFields);
- ArrayTupleReference compositeTuple = new ArrayTupleReference();
- for (int i = 0; i < tokenTuple.getFieldCount(); i++) {
- tb.addField(tokenTuple.getFieldData(i), tokenTuple.getFieldStart(i), tokenTuple.getFieldLength(i));
+ public boolean containsKey(ITupleReference searchTuple, MultiComparator invListCmp) throws HyracksDataException {
+ btreeSearchTuple.addTuple(searchTuple);
+ btreePred.setLowKeyComparator(btreeCmp);
+ btreePred.setHighKeyComparator(btreeCmp);
+ btreePred.setLowKey(btreeSearchTuple, true);
+ btreePred.setHighKey(btreeSearchTuple, true);
+ try {
+ btreeAccessor.search(btreeCursor, btreePred);
+ } catch (TreeIndexException e) {
+ btreeSearchTuple.removeLastTuple();
+ throw new HyracksDataException(e);
}
- for (int i = 0; i < invListCmp.getKeyFieldCount(); i++) {
- tb.addField(searchTuple.getFieldData(i), searchTuple.getFieldStart(i), searchTuple.getFieldLength(i));
- }
- compositeTuple.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-
- MultiComparator cmp = MultiComparator.create(btree.getComparatorFactories());
- RangePredicate predicate = new RangePredicate(compositeTuple, compositeTuple, true, true, cmp, cmp);
- BTreeRangeSearchCursor cursor = (BTreeRangeSearchCursor) btreeAccessor.createSearchCursor();
- btreeAccessor.search(cursor, predicate);
-
- boolean containsKey = cursor.hasNext();
- cursor.close();
-
+ boolean containsKey = false;
+ try {
+ containsKey = btreeCursor.hasNext();
+ } finally {
+ btreeCursor.close();
+ btreeCursor.reset();
+ btreeSearchTuple.removeLastTuple();
+ }
return containsKey;
}
+ @SuppressWarnings("rawtypes")
@Override
public String printInvList(ISerializerDeserializer[] serdes) throws HyracksDataException {
return null;
}
+ @SuppressWarnings("rawtypes")
@Override
public String printCurrentElement(ISerializerDeserializer[] serdes) throws HyracksDataException {
return null;
}
-
}