Fixed incorrect sharing of inverted-list builder causing the multi-threaded tests to break. Finished multi-threading tests.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_inverted_index_updates_new@1873 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DataGenThread.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DataGenThread.java
index ff29b33..6fff620 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DataGenThread.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DataGenThread.java
@@ -38,7 +38,8 @@
private TupleBatch[] tupleBatches;
private int ringPos;
- public DataGenThread(int numConsumers, int maxNumBatches, int batchSize, ISerializerDeserializer[] fieldSerdes, int payloadSize, int rndSeed, int maxOutstandingBatches, boolean sorted) {
+ public DataGenThread(int numConsumers, int maxNumBatches, int batchSize, ISerializerDeserializer[] fieldSerdes,
+ int payloadSize, int rndSeed, int maxOutstandingBatches, boolean sorted) {
this.maxNumBatches = maxNumBatches;
this.maxOutstandingBatches = maxOutstandingBatches;
rnd = new Random(rndSeed);
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeSearchCursor.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeSearchCursor.java
index cad4a4a..e95bf86 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeSearchCursor.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeSearchCursor.java
@@ -89,9 +89,6 @@
public void next() throws HyracksDataException {
outputElement = outputPriorityQueue.poll();
needPush = true;
- if (outputElement == null) {
- throw new HyracksDataException("The outputPriorityQueue is empty");
- }
}
@Override
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedListBuilderFactory.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedListBuilderFactory.java
new file mode 100644
index 0000000..9cde18b
--- /dev/null
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/api/IInvertedListBuilderFactory.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api;
+
+public interface IInvertedListBuilderFactory {
+ public IInvertedListBuilder create();
+}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 2689294..f6a7002 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -299,42 +299,6 @@
}
@Override
- public ILSMIOOperation createMergeOperation(ILSMIOOperationCallback callback) throws HyracksDataException,
- IndexException {
- LSMInvertedIndexOpContext ctx = createOpContext();
- ctx.reset(IndexOp.SEARCH);
- IIndexCursor cursor = new LSMInvertedIndexRangeSearchCursor();
- RangePredicate mergePred = new RangePredicate(null, null, true, true, null, null);
-
- // Scan diskInvertedIndexes ignoring the memoryInvertedIndex.
- List<Object> mergingComponents = lsmHarness.search(cursor, mergePred, ctx, false);
- if (mergingComponents.size() <= 1) {
- cursor.close();
- return null;
- }
-
- LSMInvertedIndexComponent firstComponent = (LSMInvertedIndexComponent) mergingComponents.get(0);
- OnDiskInvertedIndex firstInvIndex = (OnDiskInvertedIndex) firstComponent.getInvIndex();
- String firstFileName = firstInvIndex.getBTree().getFileReference().getFile().getName();
-
- LSMInvertedIndexComponent lastComponent = (LSMInvertedIndexComponent) mergingComponents.get(mergingComponents
- .size() - 1);
- OnDiskInvertedIndex lastInvIndex = (OnDiskInvertedIndex) lastComponent.getInvIndex();
- String lastFileName = lastInvIndex.getBTree().getFileReference().getFile().getName();
-
- LSMInvertedIndexFileNameComponent fileNameComponent = (LSMInvertedIndexFileNameComponent) fileManager
- .getRelMergeFileName(firstFileName, lastFileName);
- FileReference dictBTreeFileRef = fileManager.createMergeFile(fileNameComponent.getDictBTreeFileName());
- FileReference deletedKeysBTreeFileRef = fileManager.createMergeFile(fileNameComponent
- .getDeletedKeysBTreeFileName());
-
- LSMInvertedIndexMergeOperation mergeOp = new LSMInvertedIndexMergeOperation(this, mergingComponents, cursor,
- dictBTreeFileRef, deletedKeysBTreeFileRef, callback);
-
- return mergeOp;
- }
-
- @Override
public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput) throws IndexException {
return new LSMInvertedIndexBulkLoader(fillFactor, verifyInput);
}
@@ -457,7 +421,6 @@
IInvertedIndex mergedDiskInvertedIndex = createDiskInvIndex(diskInvIndexFactory,
mergeOp.getDictBTreeMergeTarget(), true);
IIndexCursor cursor = mergeOp.getCursor();
- // TODO: Reset verifyInput to false.
IIndexBulkLoader invIndexBulkLoader = mergedDiskInvertedIndex.createBulkLoader(1.0f, true);
try {
while (cursor.hasNext()) {
@@ -481,6 +444,42 @@
}
@Override
+ public ILSMIOOperation createMergeOperation(ILSMIOOperationCallback callback) throws HyracksDataException,
+ IndexException {
+ LSMInvertedIndexOpContext ctx = createOpContext();
+ ctx.reset(IndexOp.SEARCH);
+ IIndexCursor cursor = new LSMInvertedIndexRangeSearchCursor();
+ RangePredicate mergePred = new RangePredicate(null, null, true, true, null, null);
+
+ // Scan diskInvertedIndexes ignoring the memoryInvertedIndex.
+ List<Object> mergingComponents = lsmHarness.search(cursor, mergePred, ctx, false);
+ if (mergingComponents.size() <= 1) {
+ cursor.close();
+ return null;
+ }
+
+ LSMInvertedIndexComponent firstComponent = (LSMInvertedIndexComponent) mergingComponents.get(0);
+ OnDiskInvertedIndex firstInvIndex = (OnDiskInvertedIndex) firstComponent.getInvIndex();
+ String firstFileName = firstInvIndex.getBTree().getFileReference().getFile().getName();
+
+ LSMInvertedIndexComponent lastComponent = (LSMInvertedIndexComponent) mergingComponents.get(mergingComponents
+ .size() - 1);
+ OnDiskInvertedIndex lastInvIndex = (OnDiskInvertedIndex) lastComponent.getInvIndex();
+ String lastFileName = lastInvIndex.getBTree().getFileReference().getFile().getName();
+
+ LSMInvertedIndexFileNameComponent fileNameComponent = (LSMInvertedIndexFileNameComponent) fileManager
+ .getRelMergeFileName(firstFileName, lastFileName);
+ FileReference dictBTreeFileRef = fileManager.createMergeFile(fileNameComponent.getDictBTreeFileName());
+ FileReference deletedKeysBTreeFileRef = fileManager.createMergeFile(fileNameComponent
+ .getDeletedKeysBTreeFileName());
+
+ LSMInvertedIndexMergeOperation mergeOp = new LSMInvertedIndexMergeOperation(this, mergingComponents, cursor,
+ dictBTreeFileRef, deletedKeysBTreeFileRef, callback);
+
+ return mergeOp;
+ }
+
+ @Override
public void addMergedComponent(Object newComponent, List<Object> mergedComponents) {
diskComponents.removeAll(mergedComponents);
diskComponents.addLast(newComponent);
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
index d32c3c4..ed8f600 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
@@ -78,7 +78,6 @@
} else {
currentOff += elementSize;
}
-
currentElementIx++;
tuple.reset(pages[currentPageIx].getBuffer().array(), currentOff);
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
index 2740970..8e15526 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
@@ -393,7 +393,6 @@
if (verifyInput && lastTupleBuilder.getSize() != 0) {
if (allCmp.compare(tuple, lastTuple) <= 0) {
- System.out.println("FOUND YOU!");
throw new UnsortedInputException(
"Input stream given to OnDiskInvertedIndex bulk load is not sorted.");
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java
index b044ebe..5401519 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java
@@ -22,14 +22,15 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.IndexFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
-import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNameMapper;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilderFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class OnDiskInvertedIndexFactory extends IndexFactory<IInvertedIndex> {
- protected final IInvertedListBuilder invListBuilder;
+ protected final IInvertedListBuilderFactory invListBuilderFactory;
protected final ITypeTraits[] invListTypeTraits;
protected final IBinaryComparatorFactory[] invListCmpFactories;
protected final ITypeTraits[] tokenTypeTraits;
@@ -37,11 +38,11 @@
protected final IInvertedIndexFileNameMapper fileNameMapper;
public OnDiskInvertedIndexFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
- IInvertedListBuilder invListBuilder, ITypeTraits[] invListTypeTraits,
+ IInvertedListBuilderFactory invListBuilderFactory, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IInvertedIndexFileNameMapper fileNameMapper) {
super(bufferCache, fileMapProvider, null);
- this.invListBuilder = invListBuilder;
+ this.invListBuilderFactory = invListBuilderFactory;
this.invListTypeTraits = invListTypeTraits;
this.invListCmpFactories = invListCmpFactories;
this.tokenTypeTraits = tokenTypeTraits;
@@ -53,6 +54,7 @@
public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws IndexException {
String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getPath());
FileReference invListsFile = new FileReference(new File(invListsFilePath));
+ IInvertedListBuilder invListBuilder = invListBuilderFactory.create();
return new OnDiskInvertedIndex(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits,
invListCmpFactories, tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile);
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
index 372c144..15bc2da 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
@@ -97,6 +97,7 @@
throw new HyracksDataException(e);
}
invListCursor.pinPages();
+ invListCursor.hasNext();
unpinNeeded = true;
concatTuple.reset();
concatTuple.addTuple(tokenTuple);
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
index fba2426..02b325a 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
@@ -40,10 +40,12 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilderFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeElementInvertedListBuilder;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeElementInvertedListBuilderFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
@@ -98,9 +100,10 @@
LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider,
onDiskDirFileRef, deletedKeysBTreeFactory);
- IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
+ IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory(
+ invListTypeTraits);
OnDiskInvertedIndexFactory invIndexFactory = new OnDiskInvertedIndexFactory(diskBufferCache,
- diskFileMapProvider, invListBuilder, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
+ diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, fileManager);
LSMInvertedIndex invIndex = new LSMInvertedIndex(memBufferCache, memFreePageManager, invIndexFactory,
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java
index 26f82a6..a22a989 100644
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java
@@ -69,14 +69,14 @@
public static final int LSM_BTREE_HYRACKS_FRAME_SIZE = 128;
// Mem configuration for Inverted Index.
- public static final int LSM_INVINDEX_DISK_PAGE_SIZE = 256;
+ public static final int LSM_INVINDEX_DISK_PAGE_SIZE = 1024;
public static final int LSM_INVINDEX_DISK_NUM_PAGES = 1000;
public static final int LSM_INVINDEX_DISK_MAX_OPEN_FILES = 1000;
- public static final int LSM_INVINDEX_MEM_PAGE_SIZE = 256;
+ public static final int LSM_INVINDEX_MEM_PAGE_SIZE = 1024;
public static final int LSM_INVINDEX_MEM_NUM_PAGES = 100;
public static final int LSM_INVINDEX_HYRACKS_FRAME_SIZE = 256;
// Test parameters.
- public static final int LSM_INVINDEX_NUM_DOCS_TO_INSERT = 10000;
+ public static final int LSM_INVINDEX_NUM_DOCS_TO_INSERT = 100;
// Used for full-fledged search test.
public static final int LSM_INVINDEX_NUM_DOC_QUERIES = 1000;
public static final int LSM_INVINDEX_NUM_RANDOM_QUERIES = 1000;
@@ -89,7 +89,7 @@
public static final int LSM_INVINDEX_NUM_DELETE_ROUNDS = 3;
// Allocate a generous size to make sure we have enough elements for all tests.
public static final int LSM_INVINDEX_SCAN_COUNT_ARRAY_SIZE = 1000000;
- public static final int LSM_INVINDEX_MULTITHREAD_NUM_OPERATIONS = 1000;
+ public static final int LSM_INVINDEX_MULTITHREAD_NUM_OPERATIONS = 10000;
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexMergeTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexMergeTest.java
index 65ff11f..2b5d8c0 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexMergeTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexMergeTest.java
@@ -53,15 +53,14 @@
} else {
LSMInvertedIndexTestUtils.insertIntoInvIndex(testCtx, tupleGen, NUM_DOCS_TO_INSERT);
}
-
- // Perform merge.
- ILSMIOOperation ioop = invIndexAccessor.createMergeOperation(NoOpIOOperationCallback.INSTANCE);
- if (ioop != null) {
- invIndexAccessor.merge(ioop);
- }
- validateAndCheckIndex(testCtx);
- runTinySearchWorkload(testCtx, tupleGen);
}
+ // Perform merge.
+ ILSMIOOperation ioop = invIndexAccessor.createMergeOperation(NoOpIOOperationCallback.INSTANCE);
+ if (ioop != null) {
+ invIndexAccessor.merge(ioop);
+ }
+ validateAndCheckIndex(testCtx);
+ runTinySearchWorkload(testCtx, tupleGen);
}
invIndex.deactivate();
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexMultiThreadTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexMultiThreadTest.java
index 28afc59..8b01a40 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexMultiThreadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexMultiThreadTest.java
@@ -83,19 +83,16 @@
protected ArrayList<TestWorkloadConf> getTestWorkloadConf() {
ArrayList<TestWorkloadConf> workloadConfs = new ArrayList<TestWorkloadConf>();
- /*
// Insert only workload.
TestOperation[] insertOnlyOps = new TestOperation[] { TestOperation.INSERT };
workloadConfs.add(new TestWorkloadConf(insertOnlyOps, ProbabilityHelper
.getUniformProbDist(insertOnlyOps.length)));
- */
// Insert and merge workload.
TestOperation[] insertMergeOps = new TestOperation[] { TestOperation.INSERT, TestOperation.MERGE };
workloadConfs.add(new TestWorkloadConf(insertMergeOps, ProbabilityHelper
.getUniformProbDist(insertMergeOps.length)));
- /*
// Inserts mixed with point searches and scans.
TestOperation[] insertSearchOnlyOps = new TestOperation[] { TestOperation.INSERT, TestOperation.POINT_SEARCH,
TestOperation.SCAN };
@@ -123,7 +120,6 @@
TestOperation[] allOps = new TestOperation[] { TestOperation.INSERT, TestOperation.DELETE,
TestOperation.POINT_SEARCH, TestOperation.SCAN, TestOperation.MERGE };
workloadConfs.add(new TestWorkloadConf(allOps, ProbabilityHelper.getUniformProbDist(allOps.length)));
- */
return workloadConfs;
}
@@ -131,7 +127,7 @@
@Test
public void wordTokensInvIndexTest() throws IOException, IndexException, InterruptedException {
String dataMsg = "Documents";
- int[] numThreads = new int[] { REGULAR_NUM_THREADS };
+ int[] numThreads = new int[] { REGULAR_NUM_THREADS, EXCESSIVE_NUM_THREADS };
for (int i = 0; i < numThreads.length; i++) {
for (TestWorkloadConf conf : workloadConfs) {
setUp();
@@ -143,4 +139,20 @@
}
}
}
+
+ @Test
+ public void hashedNGramTokensInvIndexTest() throws IOException, IndexException, InterruptedException {
+ String dataMsg = "Person Names";
+ int[] numThreads = new int[] { REGULAR_NUM_THREADS, EXCESSIVE_NUM_THREADS };
+ for (int i = 0; i < numThreads.length; i++) {
+ for (TestWorkloadConf conf : workloadConfs) {
+ setUp();
+ LSMInvertedIndexTestContext testCtx = LSMInvertedIndexTestUtils.createHashedNGramInvIndexTestContext(
+ harness, InvertedIndexType.LSM);
+ TupleGenerator tupleGen = LSMInvertedIndexTestUtils.createPersonNamesTupleGen(harness.getRandom());
+ runTest(testCtx, tupleGen, numThreads[i], conf, dataMsg);
+ tearDown();
+ }
+ }
+ }
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexTestWorker.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexTestWorker.java
index 09ceb0e..7fede4c 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexTestWorker.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexTestWorker.java
@@ -70,10 +70,7 @@
switch (op) {
case INSERT: {
- accessor.insert(tuple);
- // Add tuple to document corpus so we can delete it.
- ITupleReference copyTuple = TupleUtils.copyTuple(tuple);
- documentCorpus.add(copyTuple);
+ insert(accessor, tuple);
break;
}
@@ -87,11 +84,8 @@
documentCorpus.set(docIndex, documentCorpus.get(documentCorpus.size() - 1));
documentCorpus.remove(documentCorpus.size() - 1);
} else {
- // Treat this case as an insert.
- accessor.insert(tuple);
- // Add tuple to document corpus so we can delete it.
- ITupleReference copyTuple = TupleUtils.copyTuple(tuple);
- documentCorpus.add(copyTuple);
+ // No existing documents to delete, treat this case as an insert.
+ insert(accessor, tuple);
}
break;
}
@@ -123,12 +117,7 @@
accessor.merge(ioop);
}
} catch (LSMMergeInProgressException e) {
- e.printStackTrace();
- // Ignore ongoing merges. Do an insert instead.
- //accessor.insert(tuple);
- // Add tuple to document corpus so we can delete it.
- //ITupleReference copyTuple = TupleUtils.copyTuple(tuple);
- //documentCorpus.add(copyTuple);
+ insert(accessor, tuple);
}
break;
}
@@ -137,4 +126,12 @@
throw new HyracksDataException("Op " + op.toString() + " not supported.");
}
}
+
+ private void insert(LSMInvertedIndexAccessor accessor, ITupleReference tuple) throws HyracksDataException, IndexException {
+ // Ignore ongoing merges. Do an insert instead.
+ accessor.insert(tuple);
+ // Add tuple to document corpus so we can delete it.
+ ITupleReference copyTuple = TupleUtils.copyTuple(tuple);
+ documentCorpus.add(copyTuple);
+ }
}