Changed the search algorithm for in-memory length-partitioned inverted indexes to only latch one inverted list at a time.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_lsm_length_filter@2485 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IPartitionedInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IPartitionedInvertedIndex.java
index 013cff4..98cc476 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IPartitionedInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IPartitionedInvertedIndex.java
@@ -15,8 +15,6 @@
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api;
-import java.util.ArrayList;
-
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
@@ -26,6 +24,4 @@
public void openInvertedListPartitionCursors(IInvertedIndexSearcher searcher, IIndexOperationContext ictx,
int numTokensLowerBound, int numTokensUpperBound, InvertedListPartitions invListPartitions)
throws HyracksDataException, IndexException;
-
- public void cleanupPartitionState(ArrayList<IInvertedListCursor> invListCursors) throws HyracksDataException;
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
index ceae345..6af3bd2 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
@@ -44,13 +44,13 @@
private MultiComparator tokenFieldsCmp;
private MultiComparator btreeCmp;
private final PermutingTupleReference resultTuple;
- private final ConcatenatingTupleReference btreeSearchTuple;
-
+ private final ConcatenatingTupleReference btreeSearchTuple;
+
private final ArrayTupleBuilder tokenTupleBuilder;
private final ArrayTupleReference tokenTuple = new ArrayTupleReference();
-
+
private int numElements = -1;
-
+
public InMemoryInvertedListCursor(int invListFieldCount, int tokenFieldCount) {
int[] fieldPermutation = new int[invListFieldCount];
for (int i = 0; i < invListFieldCount; i++) {
@@ -61,7 +61,7 @@
btreeSearchTuple = new ConcatenatingTupleReference(2);
tokenTupleBuilder = new ArrayTupleBuilder(tokenFieldCount);
}
-
+
public void prepare(BTreeAccessor btreeAccessor, RangePredicate btreePred, MultiComparator tokenFieldsCmp,
MultiComparator btreeCmp) throws HyracksDataException, IndexException {
// Avoid object creation if this.btreeAccessor == btreeAccessor.
@@ -76,7 +76,7 @@
this.btreeCmp = btreeCmp;
}
}
-
+
@Override
public int compareTo(IInvertedListCursor cursor) {
return size() - cursor.size();
@@ -139,14 +139,15 @@
btreePred.setHighKeyComparator(tokenFieldsCmp);
btreePred.setLowKey(tokenTuple, true);
btreePred.setHighKey(tokenTuple, true);
-
+
// Perform the count.
try {
btreeAccessor.search(countingCursor, btreePred);
while (countingCursor.hasNext()) {
countingCursor.next();
ITupleReference countTuple = countingCursor.getTuple();
- numElements = IntegerSerializerDeserializer.getInt(countTuple.getFieldData(0), countTuple.getFieldStart(0));
+ numElements = IntegerSerializerDeserializer.getInt(countTuple.getFieldData(0),
+ countTuple.getFieldStart(0));
}
} catch (HyracksDataException e) {
e.printStackTrace();
@@ -179,7 +180,8 @@
}
@Override
- public boolean containsKey(ITupleReference searchTuple, MultiComparator invListCmp) throws HyracksDataException, IndexException {
+ public boolean containsKey(ITupleReference searchTuple, MultiComparator invListCmp) throws HyracksDataException,
+ IndexException {
// Close cursor if necessary.
unpinPages();
btreeSearchTuple.addTuple(searchTuple);
@@ -200,7 +202,7 @@
btreeCursor.close();
btreeCursor.reset();
btreeSearchTuple.removeLastTuple();
- }
+ }
return containsKey;
}
@@ -217,10 +219,10 @@
DataInput dataIn = new DataInputStream(inStream);
Object o = serdes[0].deserialize(dataIn);
strBuilder.append(o.toString() + " ");
- }
+ }
} finally {
btreeCursor.close();
- btreeCursor.reset();
+ btreeCursor.reset();
}
try {
btreeAccessor.search(btreeCursor, btreePred);
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java
index 21fc0fb..dc5610f 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java
@@ -14,7 +14,6 @@
*/
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.inmemory;
-import java.util.ArrayList;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -31,7 +30,6 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearcher;
-import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IPartitionedInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.search.InvertedListPartitions;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.search.PartitionedTOccurrenceSearcher;
@@ -130,12 +128,4 @@
invListPartitions.addInvertedListCursor(inMemListCursor, i);
}
}
-
- @Override
- public void cleanupPartitionState(ArrayList<IInvertedListCursor> invListCursors) throws HyracksDataException {
- // Make sure to unpin/close cursors if the entire partition is pruned.
- //for (IInvertedListCursor cursor : invListCursors) {
- // cursor.unpinPages();
- //}
- }
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
index f9a146d..84228ac 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
@@ -15,8 +15,6 @@
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk;
-import java.util.ArrayList;
-
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
@@ -104,9 +102,4 @@
ctx.btreeCursor.reset();
}
}
-
- @Override
- public void cleanupPartitionState(ArrayList<IInvertedListCursor> invListCursors) throws HyracksDataException {
- // Do nothing.
- }
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java
index 81591e6..4f33339 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java
@@ -112,7 +112,6 @@
}
// Prune partition because no element in it can satisfy the occurrence threshold.
if (partitionCursors[i].size() < occurrenceThreshold) {
- partInvIndex.cleanupPartitionState(partitionCursors[i]);
continue;
}
// Merge inverted lists of current partition.
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/PartitionedLSMInvertedIndexMultiThreadTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/PartitionedLSMInvertedIndexMultiThreadTest.java
index bc6964f..1adaf61 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/PartitionedLSMInvertedIndexMultiThreadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/PartitionedLSMInvertedIndexMultiThreadTest.java
@@ -15,16 +15,11 @@
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.multithread;
-import java.util.logging.Logger;
-
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestContext.InvertedIndexType;
public class PartitionedLSMInvertedIndexMultiThreadTest extends LSMInvertedIndexMultiThreadTest {
- protected final Logger LOGGER = Logger.getLogger(PartitionedLSMInvertedIndexMultiThreadTest.class.getName());
-
protected InvertedIndexType getIndexType() {
return InvertedIndexType.PARTITIONED_LSM;
}
-
}