Simplified a bunch of inverted-index interfaces. Added lsm inverted index search test.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_inverted_index_updates_new@1840 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexAccessor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexAccessor.java
index 48f147f..d5266ff 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexAccessor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexAccessor.java
@@ -25,6 +25,4 @@
public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey)
throws HyracksDataException, IndexException;
-
- public IInvertedIndexSearcher getSearcher();
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexSearchModifier.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexSearchModifier.java
index 60f0cfc..619937f 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexSearchModifier.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexSearchModifier.java
@@ -15,10 +15,9 @@
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api;
-import java.util.List;
public interface IInvertedIndexSearchModifier {
- public int getOccurrenceThreshold(List<IInvertedListCursor> invListCursors);
+ public int getOccurrenceThreshold(int numQueryTokens);
- public int getPrefixLists(List<IInvertedListCursor> invListCursors);
+ public int getNumPrefixLists(int numQueryTokens);
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
index 57c560b..c76799f 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
@@ -29,9 +29,11 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMMergeInProgressException;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexFileManager.LSMInvertedIndexFileNameComponent;
-public class LSMInvertedIndexAccessor implements ILSMIndexAccessor {
+public class LSMInvertedIndexAccessor implements ILSMIndexAccessor, IInvertedIndexAccessor {
protected final LSMHarness lsmHarness;
protected final ILSMIndexFileManager fileManager;
@@ -48,16 +50,22 @@
lsmHarness.insertUpdateOrDelete(tuple, ctx);
}
- public IIndexCursor createSearchCursor() {
- return new LSMInvertedIndexSearchCursor();
- }
-
public void search(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException, IndexException {
ctx.reset(IndexOp.SEARCH);
lsmHarness.search(cursor, searchPred, ctx, true);
}
@Override
+ public void delete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ // TODO Auto-generated method stub
+
+ }
+
+ public IIndexCursor createSearchCursor() {
+ return new LSMInvertedIndexSearchCursor();
+ }
+
+ @Override
public ILSMIOOperation createFlushOperation(ILSMIOOperationCallback callback) {
LSMInvertedIndexFileNameComponent fileNameComponent = (LSMInvertedIndexFileNameComponent) fileManager
.getRelFlushFileName();
@@ -69,12 +77,6 @@
}
@Override
- public void upsert(ITupleReference tuple) throws HyracksDataException, IndexException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
public ILSMIOOperation createMergeOperation(ILSMIOOperationCallback callback) throws HyracksDataException,
LSMMergeInProgressException {
// TODO Auto-generated method stub
@@ -103,8 +105,18 @@
}
@Override
- public void delete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ public void upsert(ITupleReference tuple) throws HyracksDataException, IndexException {
// TODO Auto-generated method stub
}
+
+ @Override
+ public IInvertedListCursor createInvertedListCursor() {
+ throw new UnsupportedOperationException("Cannot create inverted list cursor on lsm inverted index.");
+ }
+
+ @Override
+ public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey)
+ throws HyracksDataException, IndexException {
+ throw new UnsupportedOperationException("Cannot open inverted list cursor on lsm inverted index.");}
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java
index f4f48d2..c0acf81 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java
@@ -76,11 +76,6 @@
index.openInvertedListCursor(listCursor, searchKey, opCtx);
}
- @Override
- public IInvertedIndexSearcher getSearcher() {
- return searcher;
- }
-
public BTreeAccessor getBTreeAccessor() {
return btreeAccessor;
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
index 35a177f..bd88c8d 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
@@ -452,11 +452,6 @@
}
@Override
- public IInvertedIndexSearcher getSearcher() {
- return searcher;
- }
-
- @Override
public IInvertedListCursor createInvertedListCursor() {
return index.createInvertedListCursor();
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/ConjunctiveSearchModifier.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/ConjunctiveSearchModifier.java
index 0c89b41..183e7f6 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/ConjunctiveSearchModifier.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/ConjunctiveSearchModifier.java
@@ -15,23 +15,17 @@
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.search;
-import java.util.Collections;
-import java.util.List;
-
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifier;
-import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
public class ConjunctiveSearchModifier implements IInvertedIndexSearchModifier {
@Override
- public int getOccurrenceThreshold(List<IInvertedListCursor> invListCursors) {
- return invListCursors.size();
+ public int getOccurrenceThreshold(int numQueryTokens) {
+ return numQueryTokens;
}
@Override
- public int getPrefixLists(List<IInvertedListCursor> invListCursors) {
- Collections.sort(invListCursors);
+ public int getNumPrefixLists(int numQueryTokens) {
return 1;
}
-
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/EditDistanceSearchModifier.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/EditDistanceSearchModifier.java
index bbed897..bba0c97 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/EditDistanceSearchModifier.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/EditDistanceSearchModifier.java
@@ -15,11 +15,7 @@
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.search;
-import java.util.Collections;
-import java.util.List;
-
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifier;
-import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
public class EditDistanceSearchModifier implements IInvertedIndexSearchModifier {
@@ -32,14 +28,13 @@
}
@Override
- public int getOccurrenceThreshold(List<IInvertedListCursor> invListCursors) {
- return invListCursors.size() - edThresh * gramLength;
+ public int getOccurrenceThreshold(int numQueryTokens) {
+ return numQueryTokens - edThresh * gramLength;
}
@Override
- public int getPrefixLists(List<IInvertedListCursor> invListCursors) {
- Collections.sort(invListCursors);
- return invListCursors.size() - getOccurrenceThreshold(invListCursors) + 1;
+ public int getNumPrefixLists(int numQueryTokens) {
+ return numQueryTokens - getOccurrenceThreshold(numQueryTokens) + 1;
}
public int getGramLength() {
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/JaccardSearchModifier.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/JaccardSearchModifier.java
index 2e71c25..022cd69 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/JaccardSearchModifier.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/JaccardSearchModifier.java
@@ -15,11 +15,7 @@
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.search;
-import java.util.Collections;
-import java.util.List;
-
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifier;
-import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
public class JaccardSearchModifier implements IInvertedIndexSearchModifier {
@@ -30,17 +26,16 @@
}
@Override
- public int getOccurrenceThreshold(List<IInvertedListCursor> invListCursors) {
- return (int) Math.floor((float) invListCursors.size() * jaccThresh);
+ public int getOccurrenceThreshold(int numQueryTokens) {
+ return (int) Math.floor((float) numQueryTokens * jaccThresh);
}
@Override
- public int getPrefixLists(List<IInvertedListCursor> invListCursors) {
- Collections.sort(invListCursors);
- if (invListCursors.size() == 0) {
+ public int getNumPrefixLists(int numQueryTokens) {
+ if (numQueryTokens == 0) {
return 0;
}
- return invListCursors.size() - getOccurrenceThreshold(invListCursors) + 1;
+ return numQueryTokens - getOccurrenceThreshold(numQueryTokens) + 1;
}
public float getJaccThresh() {
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java
index 477e0f4..1613a0b 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java
@@ -19,6 +19,7 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
@@ -170,15 +171,15 @@
invIndex.openInvertedListCursor(invListCursorCache.get(i), searchKey, ictx);
invListCursors.add(invListCursorCache.get(i));
}
-
- occurrenceThreshold = searchModifier.getOccurrenceThreshold(invListCursors);
-
+ Collections.sort(invListCursors);
+
+ occurrenceThreshold = searchModifier.getOccurrenceThreshold(invListCursors.size());
// TODO: deal with panic cases properly
if (occurrenceThreshold <= 0) {
throw new OccurrenceThresholdPanicException("Merge Threshold is <= 0. Failing Search.");
}
-
- int numPrefixLists = searchModifier.getPrefixLists(invListCursors);
+
+ int numPrefixLists = searchModifier.getNumPrefixLists(invListCursors.size());
maxResultBufIdx = mergePrefixLists(numPrefixLists, numQueryTokens);
maxResultBufIdx = mergeSuffixLists(numPrefixLists, numQueryTokens, maxResultBufIdx);
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexSearchTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexSearchTest.java
new file mode 100644
index 0000000..6e47f66
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexSearchTest.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.invertedindex;
+
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.common.AbstractInvertedIndexSearchTest;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexTestContext.InvertedIndexType;
+
+public class LSMInvertedIndexSearchTest extends AbstractInvertedIndexSearchTest {
+
+ public LSMInvertedIndexSearchTest() {
+ super(InvertedIndexType.LSM, false);
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java
index a5e4723..7042d01 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java
@@ -19,6 +19,7 @@
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
@@ -38,7 +39,6 @@
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.exceptions.OccurrenceThresholdPanicException;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.search.ConjunctiveSearchModifier;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.search.InvertedIndexSearchPredicate;
-import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.search.TOccurrenceSearcher;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexTestContext;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexTestContext.InvertedIndexType;
@@ -49,17 +49,17 @@
protected int NUM_QUERIES = 10000;
protected int[] scanCountArray = new int[NUM_DOCS_TO_INSERT];
protected final boolean bulkLoad;
-
+
// Probability that a randomly generated query is used, instead of a document from the corpus.
protected final float randomQueryProb = 0.9f;
-
+
public AbstractInvertedIndexSearchTest(InvertedIndexType invIndexType, boolean bulkLoad) {
super(invIndexType);
this.bulkLoad = bulkLoad;
}
- protected void runTest(InvertedIndexTestContext testCtx, TupleGenerator tupleGen, IInvertedIndexSearchModifier searchModifier) throws IOException,
- IndexException {
+ protected void runTest(InvertedIndexTestContext testCtx, TupleGenerator tupleGen,
+ IInvertedIndexSearchModifier searchModifier) throws IOException, IndexException {
IIndex invIndex = testCtx.getIndex();
invIndex.create();
invIndex.activate();
@@ -70,7 +70,7 @@
InvertedIndexTestUtils.insertIntoInvIndex(testCtx, tupleGen, NUM_DOCS_TO_INSERT);
}
invIndex.validate();
-
+
IInvertedIndexAccessor accessor = (IInvertedIndexAccessor) invIndex.createAccessor(
NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
IBinaryTokenizer tokenizer = testCtx.getTokenizerFactory().createTokenizer();
@@ -80,7 +80,7 @@
// Project away the primary-key field.
int[] fieldPermutation = new int[] { 0 };
PermutingTupleReference searchDocument = new PermutingTupleReference(fieldPermutation);
-
+
IIndexCursor resultCursor = accessor.createSearchCursor();
for (int i = 0; i < NUM_QUERIES; i++) {
if (rnd.nextFloat() <= randomQueryProb) {
@@ -104,7 +104,7 @@
System.out.println("HERE WE GO, DEBUG IT!");
}
*/
-
+
// Set query tuple in search predicate.
searchPred.setQueryTuple(searchDocument);
searchPred.setQueryFieldIndex(0);
@@ -117,31 +117,37 @@
// ignore panic queries
panic = true;
}
-
+
if (!panic) {
- TOccurrenceSearcher searcher = (TOccurrenceSearcher) accessor.getSearcher();
- List<Integer> expectedResults = new ArrayList<Integer>();
- InvertedIndexTestUtils.getExpectedResults(scanCountArray, testCtx.getCheckTuples(), searchDocument,
- tokenizer, testCtx.getFieldSerdes()[0], searcher.getOccurrenceThreshold(), expectedResults);
-
- Iterator<Integer> expectedIter = expectedResults.iterator();
- int count = 0;
- while (expectedIter.hasNext() && resultCursor.hasNext()) {
- int expected = expectedIter.next();
+ // Consume cursor and deserialize results so we can sort them. Some search cursors may not deliver the result sorted (e.g., LSM search cursor).
+ ArrayList<Integer> actualResults = new ArrayList<Integer>();
+ while (resultCursor.hasNext()) {
resultCursor.next();
ITupleReference resultTuple = resultCursor.getTuple();
- int actual = IntegerSerializerDeserializer
- .getInt(resultTuple.getFieldData(0), resultTuple.getFieldStart(0));
+ int actual = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0),
+ resultTuple.getFieldStart(0));
+ actualResults.add(Integer.valueOf(actual));
+ }
+ Collections.sort(actualResults);
+
+ // Get expected results.
+ List<Integer> expectedResults = new ArrayList<Integer>();
+ InvertedIndexTestUtils.getExpectedResults(scanCountArray, testCtx.getCheckTuples(), searchDocument,
+ tokenizer, testCtx.getFieldSerdes()[0], searchModifier, expectedResults);
+
+ Iterator<Integer> expectedIter = expectedResults.iterator();
+ Iterator<Integer> actualIter = actualResults.iterator();
+ while (expectedIter.hasNext() && actualIter.hasNext()) {
+ int expected = expectedIter.next();
+ int actual = actualIter.next();
if (actual != expected) {
fail("Query results do not match. Encountered: " + actual + ". Expected: " + expected + "");
}
- count++;
}
if (expectedIter.hasNext()) {
- System.out.println("COUNTS: " + expectedResults.size() + " " + count);
fail("Query results do not match. Actual results missing.");
}
- if (resultCursor.hasNext()) {
+ if (actualIter.hasNext()) {
fail("Query results do not match. Actual contains too many results.");
}
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchPerfTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchPerfTest.java
index 15734a0..ec0d305 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchPerfTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchPerfTest.java
@@ -37,7 +37,6 @@
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.search.ConjunctiveSearchModifier;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.search.InvertedIndexSearchPredicate;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.search.JaccardSearchModifier;
-import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.search.TOccurrenceSearcher;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.DelimitedUTF8StringBinaryTokenizer;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.UTF8WordTokenFactory;
@@ -202,8 +201,8 @@
}
if (!panic) {
- TOccurrenceSearcher searcher = (TOccurrenceSearcher) accessor.getSearcher();
- fillExpectedResults(queryTokenIndexes, numQueryTokens, searcher.getOccurrenceThreshold());
+ int occurrenceThreshold = searchModifier.getOccurrenceThreshold(numQueryTokens);
+ fillExpectedResults(queryTokenIndexes, numQueryTokens, occurrenceThreshold);
// verify results
int checkIndex = 0;
while (resultCursor.hasNext()) {
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexTestUtils.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexTestUtils.java
index a7accc0..a39ec46 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexTestUtils.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexTestUtils.java
@@ -50,6 +50,7 @@
import edu.uci.ics.hyracks.storage.am.common.tuples.PermutingTupleReference;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifier;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.common.LSMInvertedIndexTestHarness;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.DelimitedUTF8StringBinaryTokenizerFactory;
@@ -59,7 +60,6 @@
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.UTF8WordTokenFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexTestContext.InvertedIndexType;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
@SuppressWarnings("rawtypes")
public class InvertedIndexTestUtils {
@@ -201,7 +201,7 @@
@SuppressWarnings("unchecked")
public static void getExpectedResults(int[] scanCountArray, TreeSet<CheckTuple> checkTuples,
ITupleReference searchDocument, IBinaryTokenizer tokenizer, ISerializerDeserializer tokenSerde,
- int occurrenceThreshold, List<Integer> expectedResults) throws IOException {
+ IInvertedIndexSearchModifier searchModifier, List<Integer> expectedResults) throws IOException {
// Reset scan count array.
Arrays.fill(scanCountArray, 0);
expectedResults.clear();
@@ -209,6 +209,7 @@
ByteArrayAccessibleOutputStream baaos = new ByteArrayAccessibleOutputStream();
tokenizer.reset(searchDocument.getFieldData(0), searchDocument.getFieldStart(0),
searchDocument.getFieldLength(0));
+ int numQueryTokens = 0;
while (tokenizer.hasNext()) {
tokenizer.next();
IToken token = tokenizer.getToken();
@@ -232,8 +233,10 @@
Integer element = (Integer) checkTuple.getField(1);
scanCountArray[element]++;
}
+ numQueryTokens++;
}
+ int occurrenceThreshold = searchModifier.getOccurrenceThreshold(numQueryTokens);
// Run through scan count array, and see whether elements satisfy the given occurrence threshold.
expectedResults.clear();
for (int i = 0; i < scanCountArray.length; i++) {