Added basic search test for in-memory inverted index based on new framework. Fixed a few bugs in the in-memory inverted list cursor.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_inverted_index_updates_new@1824 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java
index 87d1055..ca1fa1e 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java
@@ -53,7 +53,7 @@
public void removeLastTuple() {
if (numTuples > 0) {
- ITupleReference lastTuple = tuples[numTuples--];
+ ITupleReference lastTuple = tuples[--numTuples];
totalFieldCount -= lastTuple.getFieldCount();
}
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
index 1527a40..c4a337f 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
@@ -141,13 +141,13 @@
}
@Override
- public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference tupleReference,
+ public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey,
IIndexOpContext ictx) throws HyracksDataException, IndexException {
InMemoryInvertedIndexOpContext ctx = (InMemoryInvertedIndexOpContext) ictx;
ctx.reset(IndexOp.SEARCH);
InMemoryInvertedListCursor inMemListCursor = (InMemoryInvertedListCursor) listCursor;
inMemListCursor.prepare(ctx.btreeAccessor, ctx.btreePred, ctx.tokenFieldsCmp, ctx.btreeCmp);
- inMemListCursor.reset(tupleReference);
+ inMemListCursor.reset(searchKey);
}
@Override
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java
index 1278283..4a97465 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java
@@ -77,6 +77,11 @@
}
@Override
+ public IInvertedIndexSearcher getSearcher() {
+ return searcher;
+ }
+
+ @Override
public void delete(ITupleReference tuple) throws HyracksDataException, IndexException {
throw new UnsupportedOperationException("Delete not supported by in-memory inverted index.");
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
index 5b003ee..0401741 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
@@ -14,6 +14,10 @@
*/
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.inmemory;
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
@@ -81,8 +85,8 @@
tokenTuple.reset(tokenTupleBuilder.getFieldEndOffsets(), tokenTupleBuilder.getByteArray());
btreeSearchTuple.reset();
btreeSearchTuple.addTuple(tokenTuple);
- btreePred.setLowKey(tuple, true);
- btreePred.setHighKey(tuple, true);
+ btreePred.setLowKey(tokenTuple, true);
+ btreePred.setHighKey(tokenTuple, true);
btreeAccessor.search(btreeCursor, btreePred);
}
@@ -140,6 +144,7 @@
} finally {
try {
countingCursor.close();
+ countingCursor.reset();
} catch (HyracksDataException e) {
e.printStackTrace();
}
@@ -190,7 +195,27 @@
@SuppressWarnings("rawtypes")
@Override
public String printInvList(ISerializerDeserializer[] serdes) throws HyracksDataException {
- return null;
+ StringBuilder strBuilder = new StringBuilder();
+ try {
+ while (btreeCursor.hasNext()) {
+ btreeCursor.next();
+ ITupleReference tuple = btreeCursor.getTuple();
+ ByteArrayInputStream inStream = new ByteArrayInputStream(tuple.getFieldData(1), tuple.getFieldStart(1),
+ tuple.getFieldLength(1));
+ DataInput dataIn = new DataInputStream(inStream);
+ Object o = serdes[0].deserialize(dataIn);
+ strBuilder.append(o.toString() + " ");
+ }
+ } finally {
+ btreeCursor.close();
+ btreeCursor.reset();
+ }
+ try {
+ btreeAccessor.search(btreeCursor, btreePred);
+ } catch (TreeIndexException e) {
+ throw new HyracksDataException(e);
+ }
+ return strBuilder.toString();
}
@SuppressWarnings("rawtypes")
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java
index 396c06e..477e0f4 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java
@@ -270,6 +270,7 @@
protected int mergeSuffixListScan(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers,
int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens)
throws HyracksDataException {
+
int newBufIdx = 0;
ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java
index ecf0e5b..a785398 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java
@@ -51,7 +51,7 @@
protected final boolean bulkLoad;
// Probability that a randomly generated query is used, instead of a document from the corpus.
- protected final float randomQueryProb = 0.2f;
+ protected final float randomQueryProb = 0.9f;
public AbstractInvertedIndexSearchTest(InvertedIndexType invIndexType, boolean bulkLoad) {
super(invIndexType);
@@ -93,6 +93,17 @@
searchDocument.reset(documentCorpus.get(queryIndex));
}
+ /*
+ StringBuilder builder = new StringBuilder();
+ UTF8StringPointable.toString(builder, searchDocument.getFieldData(0), searchDocument.getFieldStart(0));
+ String query = builder.toString();
+
+ System.out.println("QUERY: " + i + " " + query + " " + isRandom);
+ if (query.equals("Patricia Mary")) {
+ System.out.println("HERE WE GO, DEBUG IT!");
+ }
+ */
+
// Set query tuple in search predicate.
searchPred.setQueryTuple(searchDocument);
searchPred.setQueryFieldIndex(0);
@@ -113,6 +124,7 @@
tokenizer, testCtx.getFieldSerdes()[0], searcher.getOccurrenceThreshold(), expectedResults);
Iterator<Integer> expectedIter = expectedResults.iterator();
+ int count = 0;
while (expectedIter.hasNext() && resultCursor.hasNext()) {
int expected = expectedIter.next();
resultCursor.next();
@@ -122,8 +134,10 @@
if (actual != expected) {
fail("Query results do not match. Encountered: " + actual + ". Expected: " + expected + "");
}
+ count++;
}
if (expectedIter.hasNext()) {
+ System.out.println("COUNTS: " + expectedResults.size() + " " + count);
fail("Query results do not match. Actual results missing.");
}
if (resultCursor.hasNext()) {
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexSearchTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexSearchTest.java
new file mode 100644
index 0000000..47e5b78
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexSearchTest.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.inmemory;
+
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.common.AbstractInvertedIndexSearchTest;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexTestContext.InvertedIndexType;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+public class InMemoryInvertedIndexSearchTest extends AbstractInvertedIndexSearchTest {
+
+ public InMemoryInvertedIndexSearchTest() {
+ super(InvertedIndexType.INMEMORY, false);
+ }
+
+ @Override
+ public IBufferCache getBufferCache() {
+ return harness.getMemBufferCache();
+ }
+}