Fixed a few multi-threading issues in the lsm-inverted-index. Still some more to go.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_inverted_index_updates_new@1871 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
index b73c0d8..152ac32 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
@@ -35,7 +35,6 @@
import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeNonExistentKeyException;
import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeNotUpdateableException;
-import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeUnsortedInputException;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrame;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext.PageValidationInfo;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
@@ -53,6 +52,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.api.UnsortedInputException;
import edu.uci.ics.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
import edu.uci.ics.hyracks.storage.am.common.impls.AbstractTreeIndex;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
@@ -998,7 +998,7 @@
HyracksDataException {
// New tuple should be strictly greater than last tuple.
if (cmp.compare(tuple, prevTuple) != 1) {
- throw new BTreeUnsortedInputException("Input stream given to BTree bulk load is not sorted.");
+ throw new UnsortedInputException("Input stream given to BTree bulk load is not sorted.");
}
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexCursor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexCursor.java
index 838be6f..1c75da2 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexCursor.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexCursor.java
@@ -28,7 +28,7 @@
public void close() throws HyracksDataException;
- public void reset() throws HyracksDataException;
+ public void reset() throws HyracksDataException, IndexException;
public ITupleReference getTuple();
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/exceptions/BTreeUnsortedInputException.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/UnsortedInputException.java
similarity index 70%
rename from hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/exceptions/BTreeUnsortedInputException.java
rename to hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/UnsortedInputException.java
index 1645d51..e3304c9 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/exceptions/BTreeUnsortedInputException.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/UnsortedInputException.java
@@ -13,19 +13,19 @@
* limitations under the License.
*/
-package edu.uci.ics.hyracks.storage.am.btree.exceptions;
+package edu.uci.ics.hyracks.storage.am.common.api;
/**
- * Thrown when trying to bulk load a tree with an unsorted input stream.
+ * Thrown when trying to bulk load an index with an unsorted input stream.
*/
-public class BTreeUnsortedInputException extends BTreeException {
+public class UnsortedInputException extends IndexException {
private static final long serialVersionUID = 1L;
- public BTreeUnsortedInputException(Exception e) {
+ public UnsortedInputException(Exception e) {
super(e);
}
- public BTreeUnsortedInputException(String message) {
+ public UnsortedInputException(String message) {
super(message);
}
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/MultiComparator.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/MultiComparator.java
index a8ddc19..01c6cc8 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/MultiComparator.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/MultiComparator.java
@@ -95,4 +95,19 @@
}
return new MultiComparator(cmps);
}
+
+ public static MultiComparator create(IBinaryComparatorFactory[]... cmpFactories) {
+ int size = 0;
+ for (int i = 0; i < cmpFactories.length; i++) {
+ size += cmpFactories[i].length;
+ }
+ IBinaryComparator[] cmps = new IBinaryComparator[size];
+ int x = 0;
+ for (int i = 0; i < cmpFactories.length; i++) {
+ for (int j = 0; j < cmpFactories[i].length; j++) {
+ cmps[x++] = cmpFactories[i][j].createBinaryComparator();
+ }
+ }
+ return new MultiComparator(cmps);
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeSearchCursor.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeSearchCursor.java
index 8625b13..cad4a4a 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeSearchCursor.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeSearchCursor.java
@@ -59,7 +59,7 @@
}
@Override
- public void reset() throws HyracksDataException {
+ public void reset() throws HyracksDataException, IndexException {
outputElement = null;
needPush = false;
@@ -140,7 +140,7 @@
return false;
}
- protected boolean isDeleted(PriorityQueueElement checkElement) throws HyracksDataException {
+ protected boolean isDeleted(PriorityQueueElement checkElement) throws HyracksDataException, IndexException {
return ((ILSMTreeTupleReference) checkElement.getTuple()).isAntimatter();
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedListCursor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedListCursor.java
index 489495c..c62cc57 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedListCursor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedListCursor.java
@@ -24,7 +24,7 @@
public interface IInvertedListCursor extends Comparable<IInvertedListCursor> {
public void reset(int startPageId, int endPageId, int startOff, int numElements);
- public void pinPages() throws HyracksDataException;
+ public void pinPages() throws HyracksDataException, IndexException;
public void unpinPages() throws HyracksDataException;
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index deb1c7d..2689294 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -457,7 +457,8 @@
IInvertedIndex mergedDiskInvertedIndex = createDiskInvIndex(diskInvIndexFactory,
mergeOp.getDictBTreeMergeTarget(), true);
IIndexCursor cursor = mergeOp.getCursor();
- IIndexBulkLoader invIndexBulkLoader = mergedDiskInvertedIndex.createBulkLoader(1.0f, false);
+ // TODO: Reset verifyInput to false.
+ IIndexBulkLoader invIndexBulkLoader = mergedDiskInvertedIndex.createBulkLoader(1.0f, true);
try {
while (cursor.hasNext()) {
cursor.next();
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
index b478b98..6046943 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
@@ -85,18 +85,17 @@
}
@Override
- public ILSMIOOperation createMergeOperation(ILSMIOOperationCallback callback) throws HyracksDataException,
- IndexException {
- ILSMIOOperation mergeOp = invIndex.createMergeOperation(callback);
- return mergeOp;
- }
-
- @Override
public void flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
lsmHarness.flush(operation);
}
@Override
+ public ILSMIOOperation createMergeOperation(ILSMIOOperationCallback callback) throws HyracksDataException,
+ IndexException {
+ return lsmHarness.createMergeOperation(callback);
+ }
+
+ @Override
public void merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
lsmHarness.merge(operation);
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
index 404fe44..d10cbe9 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
@@ -48,7 +48,6 @@
IInvertedIndexAccessor invIndexAccessor = (IInvertedIndexAccessor) lsmInitState.getIndexAccessors().get(i);
rangeCursors[i] = invIndexAccessor.createRangeSearchCursor();
invIndexAccessor.rangeSearch(rangeCursors[i], lsmInitState.getSearchPredicate());
-
}
// For searching the deleted-keys BTrees.
@@ -71,7 +70,7 @@
* Check deleted-keys BTrees whether they contain the key in the checkElement's tuple.
*/
@Override
- protected boolean isDeleted(PriorityQueueElement checkElement) throws HyracksDataException {
+ protected boolean isDeleted(PriorityQueueElement checkElement) throws HyracksDataException, IndexException {
keysOnlyTuple.reset(checkElement.getTuple());
int end = checkElement.getCursorIndex();
for (int i = 0; i < end; i++) {
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexSearchCursor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexSearchCursor.java
index aafe950..37e7f51 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexSearchCursor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexSearchCursor.java
@@ -67,7 +67,7 @@
keySearchPred = new RangePredicate(null, null, true, true, keyCmp, keyCmp);
}
- protected boolean isDeleted(ITupleReference key) throws HyracksDataException {
+ protected boolean isDeleted(ITupleReference key) throws HyracksDataException, IndexException {
keySearchPred.setLowKey(key, true);
keySearchPred.setHighKey(key, true);
for (int i = 0; i < accessorIndex; i++) {
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
index 68de656..21e2bf7 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
@@ -39,11 +39,12 @@
private RangePredicate btreePred;
private BTreeAccessor btreeAccessor;
private IIndexCursor btreeCursor;
+ private boolean cursorNeedsClose = false;
private IIndexCursor countingCursor;
private MultiComparator tokenFieldsCmp;
private MultiComparator btreeCmp;
private final PermutingTupleReference resultTuple;
- private final ConcatenatingTupleReference btreeSearchTuple;
+ private final ConcatenatingTupleReference btreeSearchTuple;
private final ArrayTupleBuilder tokenTupleBuilder;
private final ArrayTupleReference tokenTuple = new ArrayTupleReference();
@@ -78,18 +79,15 @@
return size() - cursor.size();
}
- public void reset(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
+ public void reset(ITupleReference tuple) throws HyracksDataException, IndexException {
numElements = -1;
// Copy the tokens tuple for later use in btree probes.
TupleUtils.copyTuple(tokenTupleBuilder, tuple, tuple.getFieldCount());
tokenTuple.reset(tokenTupleBuilder.getFieldEndOffsets(), tokenTupleBuilder.getByteArray());
btreeSearchTuple.reset();
btreeSearchTuple.addTuple(tokenTuple);
- btreePred.setLowKey(tokenTuple, true);
- btreePred.setHighKey(tokenTuple, true);
btreeCursor.reset();
countingCursor.reset();
- btreeAccessor.search(btreeCursor, btreePred);
}
@Override
@@ -98,13 +96,21 @@
}
@Override
- public void pinPages() throws HyracksDataException {
- // Do nothing
+ public void pinPages() throws HyracksDataException, IndexException {
+ btreePred.setLowKeyComparator(tokenFieldsCmp);
+ btreePred.setHighKeyComparator(tokenFieldsCmp);
+ btreePred.setLowKey(tokenTuple, true);
+ btreePred.setHighKey(tokenTuple, true);
+ btreeAccessor.search(btreeCursor, btreePred);
+ cursorNeedsClose = true;
}
@Override
public void unpinPages() throws HyracksDataException {
- btreeCursor.close();
+ if (cursorNeedsClose) {
+ btreeCursor.close();
+ cursorNeedsClose = false;
+ }
}
@Override
@@ -171,6 +177,8 @@
@Override
public boolean containsKey(ITupleReference searchTuple, MultiComparator invListCmp) throws HyracksDataException, IndexException {
+ // Close cursor if necessary.
+ unpinPages();
btreeSearchTuple.addTuple(searchTuple);
btreePred.setLowKeyComparator(btreeCmp);
btreePred.setHighKeyComparator(btreeCmp);
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
index eecb0cd..2740970 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
@@ -43,6 +43,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.api.UnsortedInputException;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.tuples.PermutingTupleReference;
@@ -294,11 +295,20 @@
private ICachedPage currentPage;
private final MultiComparator tokenCmp;
private final MultiComparator invListCmp;
-
+
+ private final boolean verifyInput;
+ private final MultiComparator allCmp;
+
public OnDiskInvertedIndexBulkLoader(float btreeFillFactor, boolean verifyInput, int startPageId, int fileId)
throws IndexException, HyracksDataException {
+ this.verifyInput = verifyInput;
this.tokenCmp = MultiComparator.create(btree.getComparatorFactories());
- this.invListCmp = MultiComparator.create(invListCmpFactories);
+ this.invListCmp = MultiComparator.create(invListCmpFactories);
+ if (verifyInput) {
+ allCmp = MultiComparator.create(btree.getComparatorFactories(), invListCmpFactories);
+ } else {
+ allCmp = null;
+ }
this.btreeTupleBuilder = new ArrayTupleBuilder(btree.getFieldCount());
this.btreeTupleReference = new ArrayTupleReference();
this.lastTupleBuilder = new ArrayTupleBuilder(numTokenFields + numInvListKeys);
@@ -381,6 +391,14 @@
}
}
+ if (verifyInput && lastTupleBuilder.getSize() != 0) {
+ if (allCmp.compare(tuple, lastTuple) <= 0) {
+ System.out.println("FOUND YOU!");
+ throw new UnsortedInputException(
+ "Input stream given to OnDiskInvertedIndex bulk load is not sorted.");
+ }
+ }
+
// Remember last tuple by creating a copy.
// TODO: This portion can be optimized by only copying the token when it changes, and using the last appended inverted-list element as a reference.
lastTupleBuilder.reset();
@@ -468,7 +486,7 @@
}
@Override
- public void rangeSearch(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException {
+ public void rangeSearch(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException, IndexException {
OnDiskInvertedIndexRangeSearchCursor rangeSearchCursor = (OnDiskInvertedIndexRangeSearchCursor) cursor;
rangeSearchCursor.open(null, searchPred);
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
index 540fb9f..372c144 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
@@ -66,7 +66,7 @@
}
@Override
- public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException, IndexException {
this.btreePred = (RangePredicate) searchPred;
try {
btreeAccessor.search(btreeCursor, btreePred);
@@ -122,13 +122,12 @@
}
@Override
- public void reset() throws HyracksDataException {
+ public void reset() throws HyracksDataException, IndexException {
if (unpinNeeded) {
invListCursor.unpinPages();
unpinNeeded = false;
}
btreeCursor.close();
- open(null, btreePred);
}
@Override
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java
index 60b0370..11fd697 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java
@@ -131,7 +131,7 @@
}
@Override
- public void reset() throws HyracksDataException {
+ public void reset() throws HyracksDataException, IndexException {
if (includeMemComponent) {
memRTreeCursor.reset();
memBTreeCursor.reset();
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexExamplesTest.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
index a93efa6..54d0c93 100644
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
@@ -35,10 +35,10 @@
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
-import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeUnsortedInputException;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
import edu.uci.ics.hyracks.storage.am.common.TestOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.UnsortedInputException;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
@@ -605,7 +605,7 @@
TupleUtils.createIntegerTuple(tb, tuple, key, 5);
try {
bulkLoader.add(tuple);
- } catch (BTreeUnsortedInputException e) {
+ } catch (UnsortedInputException e) {
if (j != i) {
fail("Unexpected exception: " + e.getMessage());
}
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java
index 4b70579..26f82a6 100644
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java
@@ -69,10 +69,10 @@
public static final int LSM_BTREE_HYRACKS_FRAME_SIZE = 128;
// Mem configuration for Inverted Index.
- public static final int LSM_INVINDEX_DISK_PAGE_SIZE = 1024;
+ public static final int LSM_INVINDEX_DISK_PAGE_SIZE = 256;
public static final int LSM_INVINDEX_DISK_NUM_PAGES = 1000;
public static final int LSM_INVINDEX_DISK_MAX_OPEN_FILES = 1000;
- public static final int LSM_INVINDEX_MEM_PAGE_SIZE = 1024;
+ public static final int LSM_INVINDEX_MEM_PAGE_SIZE = 256;
public static final int LSM_INVINDEX_MEM_NUM_PAGES = 100;
public static final int LSM_INVINDEX_HYRACKS_FRAME_SIZE = 256;
// Test parameters.
@@ -89,7 +89,7 @@
public static final int LSM_INVINDEX_NUM_DELETE_ROUNDS = 3;
// Allocate a generous size to make sure we have enough elements for all tests.
public static final int LSM_INVINDEX_SCAN_COUNT_ARRAY_SIZE = 1000000;
- public static final int LSM_INVINDEX_MULTITHREAD_NUM_OPERATIONS = 10000;
+ public static final int LSM_INVINDEX_MULTITHREAD_NUM_OPERATIONS = 1000;
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexMultiThreadTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexMultiThreadTest.java
index 7471492..28afc59 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexMultiThreadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexMultiThreadTest.java
@@ -44,7 +44,7 @@
protected final int REGULAR_NUM_THREADS = Runtime.getRuntime().availableProcessors();
// Excessive number of threads for testing.
protected final int EXCESSIVE_NUM_THREADS = Runtime.getRuntime().availableProcessors() * 4;
- protected final int NUM_OPERATIONS = AccessMethodTestsConfig.BTREE_MULTITHREAD_NUM_OPERATIONS;
+ protected final int NUM_OPERATIONS = AccessMethodTestsConfig.LSM_INVINDEX_MULTITHREAD_NUM_OPERATIONS;
private final LSMInvertedIndexTestHarness harness = new LSMInvertedIndexTestHarness();
private final LSMInvertedIndexWorkerFactory workerFactory = new LSMInvertedIndexWorkerFactory();
@@ -83,16 +83,19 @@
protected ArrayList<TestWorkloadConf> getTestWorkloadConf() {
ArrayList<TestWorkloadConf> workloadConfs = new ArrayList<TestWorkloadConf>();
+ /*
// Insert only workload.
TestOperation[] insertOnlyOps = new TestOperation[] { TestOperation.INSERT };
workloadConfs.add(new TestWorkloadConf(insertOnlyOps, ProbabilityHelper
.getUniformProbDist(insertOnlyOps.length)));
+ */
// Insert and merge workload.
TestOperation[] insertMergeOps = new TestOperation[] { TestOperation.INSERT, TestOperation.MERGE };
workloadConfs.add(new TestWorkloadConf(insertMergeOps, ProbabilityHelper
.getUniformProbDist(insertMergeOps.length)));
+ /*
// Inserts mixed with point searches and scans.
TestOperation[] insertSearchOnlyOps = new TestOperation[] { TestOperation.INSERT, TestOperation.POINT_SEARCH,
TestOperation.SCAN };
@@ -120,6 +123,7 @@
TestOperation[] allOps = new TestOperation[] { TestOperation.INSERT, TestOperation.DELETE,
TestOperation.POINT_SEARCH, TestOperation.SCAN, TestOperation.MERGE };
workloadConfs.add(new TestWorkloadConf(allOps, ProbabilityHelper.getUniformProbDist(allOps.length)));
+ */
return workloadConfs;
}
@@ -127,9 +131,9 @@
@Test
public void wordTokensInvIndexTest() throws IOException, IndexException, InterruptedException {
String dataMsg = "Documents";
- int[] numThreads = new int[] { EXCESSIVE_NUM_THREADS };
+ int[] numThreads = new int[] { REGULAR_NUM_THREADS };
for (int i = 0; i < numThreads.length; i++) {
- for (TestWorkloadConf conf : workloadConfs) {
+ for (TestWorkloadConf conf : workloadConfs) {
setUp();
LSMInvertedIndexTestContext testCtx = LSMInvertedIndexTestUtils.createWordInvIndexTestContext(harness,
InvertedIndexType.LSM);
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexTestWorker.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexTestWorker.java
index 0176d98..09ceb0e 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexTestWorker.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexTestWorker.java
@@ -123,11 +123,12 @@
accessor.merge(ioop);
}
} catch (LSMMergeInProgressException e) {
+ e.printStackTrace();
// Ignore ongoing merges. Do an insert instead.
- accessor.insert(tuple);
+ //accessor.insert(tuple);
// Add tuple to document corpus so we can delete it.
- ITupleReference copyTuple = TupleUtils.copyTuple(tuple);
- documentCorpus.add(copyTuple);
+ //ITupleReference copyTuple = TupleUtils.copyTuple(tuple);
+ //documentCorpus.add(copyTuple);
}
break;
}