Merge commit 'd71cb24eb7040875eaf9b70cc5311e87b24f149d' from release-0.9.4-pre-rc
Change-Id: I94cf63ce0207222a22585d2db6b4fe3b104027b5
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
index bb76204..2aebc59 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
@@ -178,7 +178,7 @@
private void stopIfRunning(ActiveRuntimeId runtimeId, IActiveRuntime runtime)
throws HyracksDataException, InterruptedException {
- if (runtimes.remove(runtimeId) != null) {
+ if (runtimes.containsKey(runtimeId)) {
runtime.stop();
} else {
LOGGER.info("Not stopping already stopped runtime " + runtimeId);
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
index acc3970..cc70d57 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
@@ -30,6 +30,7 @@
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.function.BiPredicate;
import java.util.stream.Stream;
@@ -70,6 +71,7 @@
joinPath(getProjectPath().toString(), "src", "test", "resources", "cc.conf");
private static final String DEFAULT_STORAGE_PATH = joinPath("target", "io", "dir");
private static String storagePath = DEFAULT_STORAGE_PATH;
+ private static final long RESULT_TTL = TimeUnit.MINUTES.toMillis(5);
static {
System.setProperty("java.util.logging.manager", org.apache.logging.log4j.jul.LogManager.class.getName());
@@ -198,7 +200,7 @@
ccConfig.setClientListenAddress(Inet4Address.getLoopbackAddress().getHostAddress());
ccConfig.setClientListenPort(DEFAULT_HYRACKS_CC_CLIENT_PORT);
ccConfig.setClusterListenPort(DEFAULT_HYRACKS_CC_CLUSTER_PORT);
- ccConfig.setResultTTL(120000L);
+ ccConfig.setResultTTL(RESULT_TTL);
ccConfig.setResultSweepThreshold(1000L);
ccConfig.setEnforceFrameWriterProtocol(true);
configManager.set(ControllerConfig.Option.DEFAULT_DIR, joinPath(getDefaultStoragePath(), "asterixdb"));
@@ -217,7 +219,7 @@
ncConfig.setDataListenAddress(Inet4Address.getLoopbackAddress().getHostAddress());
ncConfig.setResultListenAddress(Inet4Address.getLoopbackAddress().getHostAddress());
ncConfig.setMessagingListenAddress(Inet4Address.getLoopbackAddress().getHostAddress());
- ncConfig.setResultTTL(120000L);
+ ncConfig.setResultTTL(RESULT_TTL);
ncConfig.setResultSweepThreshold(1000L);
ncConfig.setVirtualNC();
configManager.set(ControllerConfig.Option.DEFAULT_DIR, joinPath(getDefaultStoragePath(), "asterixdb", ncName));
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java
index 18de540..c635411 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java
@@ -19,8 +19,6 @@
package org.apache.hyracks.storage.am.common.tuples;
-import java.util.Arrays;
-
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
/**
@@ -96,14 +94,17 @@
return tuples[tupleIndex].getFieldLength(fieldIndex);
}
+ /**
+ * Right now this class is only used by inverted index, and only contains 2 tuples.
+ * As a result, sequential search would be more efficient than binary search
+ */
private int getTupleIndex(int fIdx) {
- int tupleIndex = Arrays.binarySearch(fieldCounts, 0, numTuples - 1, fIdx);
- if (tupleIndex < 0) {
- tupleIndex = -tupleIndex - 1;
- } else {
- ++tupleIndex;
+ for (int i = 0; i < numTuples; i++) {
+ if (fIdx < fieldCounts[i]) {
+ return i;
+ }
}
- return tupleIndex;
+ throw new IllegalArgumentException("Illegal field index " + fIdx);
}
private int getFieldIndex(int tupleIndex, int fIdx) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
index 445a005..fb1fa63 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
@@ -125,7 +125,6 @@
rangeCursors[i].close();
}
}
- rangeCursors = null;
} finally {
if (lsmHarness != null) {
lsmHarness.endSearch(opCtx);
@@ -249,7 +248,7 @@
public class PriorityQueueElement {
private ITupleReference tuple;
- private int cursorIndex;
+ private final int cursorIndex;
public PriorityQueueElement(int cursorIndex) {
tuple = null;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
index 4eb7728..f2fb9af 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
@@ -95,14 +95,14 @@
opTrackerProvider.getOperationTracker(serviceCtx, this), ioScheduler, ioOpCallbackFactory,
invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields,
filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable,
- metadataPageManagerFactory);
+ metadataPageManagerFactory, serviceCtx.getTracer());
} else {
return InvertedIndexUtils.createLSMInvertedIndex(ioManager, virtualBufferCaches, typeTraits, cmpFactories,
tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache, file.getAbsolutePath(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerProvider.getOperationTracker(serviceCtx, this),
ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterTypeTraits, filterCmpFactories,
filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable,
- metadataPageManagerFactory);
+ metadataPageManagerFactory, serviceCtx.getTracer());
}
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 5fda514..e46c24a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -100,10 +100,10 @@
IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
int[] invertedIndexFields, int[] filterFields, int[] filterFieldsForNonBulkLoadOps,
- int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) throws HyracksDataException {
+ int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, ITracer tracer) throws HyracksDataException {
super(ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy,
opTracker, ioScheduler, ioOpCallbackFactory, componentFactory, componentFactory, filterFrameFactory,
- filterManager, filterFields, durable, filterHelper, invertedIndexFields, ITracer.NONE);
+ filterManager, filterFields, durable, filterHelper, invertedIndexFields, tracer);
this.tokenizerFactory = tokenizerFactory;
this.invListTypeTraits = invListTypeTraits;
this.invListCmpFactories = invListCmpFactories;
@@ -349,65 +349,65 @@
LSMInvertedIndexMergeOperation mergeOp = (LSMInvertedIndexMergeOperation) operation;
RangePredicate mergePred = new RangePredicate(null, null, true, true, null, null);
IIndexCursor cursor = mergeOp.getCursor();
- try {
- ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
- // Scan diskInvertedIndexes ignoring the memoryInvertedIndex.
- // Create an inverted index instance.
- ILSMDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(),
- mergeOp.getDeletedKeysBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
- ILSMDiskComponentBulkLoader componentBulkLoader;
- // In case we must keep the deleted-keys BTrees, then they must be merged *before* merging the inverted
- // indexes so that lsmHarness.endSearch() is called once when the inverted indexes have been merged.
- if (mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents
- .get(diskComponents.size() - 1)) {
- // Keep the deleted tuples since the oldest disk component is not included in the merge operation
- LSMInvertedIndexDeletedKeysBTreeMergeCursor btreeCursor =
- new LSMInvertedIndexDeletedKeysBTreeMergeCursor(opCtx);
- try {
- long numElements = 0L;
- for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
- numElements += ((LSMInvertedIndexDiskComponent) mergeOp.getMergingComponents().get(i))
- .getBloomFilter().getNumElements();
- }
- componentBulkLoader = component.createBulkLoader(1.0f, false, numElements, false, false, false);
- loadDeleteTuples(opCtx, btreeCursor, mergePred, componentBulkLoader);
- } finally {
- btreeCursor.destroy();
- }
- } else {
- componentBulkLoader = component.createBulkLoader(1.0f, false, 0L, false, false, false);
- }
- search(opCtx, cursor, mergePred);
+ ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
+ // Scan diskInvertedIndexes ignoring the memoryInvertedIndex.
+ // Create an inverted index instance.
+ ILSMDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(),
+ mergeOp.getDeletedKeysBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
+ ILSMDiskComponentBulkLoader componentBulkLoader;
+ // In case we must keep the deleted-keys BTrees, then they must be merged *before* merging the inverted
+ // indexes so that lsmHarness.endSearch() is called once when the inverted indexes have been merged.
+ if (mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents
+ .get(diskComponents.size() - 1)) {
+ // Keep the deleted tuples since the oldest disk component is not included in the merge operation
+ LSMInvertedIndexDeletedKeysBTreeMergeCursor btreeCursor =
+ new LSMInvertedIndexDeletedKeysBTreeMergeCursor(opCtx);
try {
- while (cursor.hasNext()) {
- cursor.next();
- ITupleReference tuple = cursor.getTuple();
- componentBulkLoader.add(tuple);
- }
- } finally {
- cursor.close();
- }
- if (component.getLSMComponentFilter() != null) {
- List<ITupleReference> filterTuples = new ArrayList<>();
+ long numElements = 0L;
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
- ITupleReference min = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple();
- ITupleReference max = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple();
- if (min != null) {
- filterTuples.add(min);
- }
- if (max != null) {
- filterTuples.add(max);
- }
+ numElements += ((LSMInvertedIndexDiskComponent) mergeOp.getMergingComponents().get(i))
+ .getBloomFilter().getNumElements();
}
- getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples,
- NoOpOperationCallback.INSTANCE);
- getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder());
+ componentBulkLoader = component.createBulkLoader(1.0f, false, numElements, false, false, false);
+ loadDeleteTuples(opCtx, btreeCursor, mergePred, componentBulkLoader);
+ } finally {
+ btreeCursor.destroy();
}
- componentBulkLoader.end();
- return component;
- } finally {
- cursor.close();
+ } else {
+ componentBulkLoader = component.createBulkLoader(1.0f, false, 0L, false, false, false);
}
+ search(opCtx, cursor, mergePred);
+ try {
+ while (cursor.hasNext()) {
+ cursor.next();
+ componentBulkLoader.add(cursor.getTuple());
+ }
+ } finally {
+ try {
+ cursor.close();
+ } finally {
+ cursor.destroy();
+ }
+ }
+ if (component.getLSMComponentFilter() != null) {
+ List<ITupleReference> filterTuples = new ArrayList<>();
+ for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
+ ITupleReference min = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple();
+ ITupleReference max = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple();
+ if (min != null) {
+ filterTuples.add(min);
+ }
+ if (max != null) {
+ filterTuples.add(max);
+ }
+ }
+ getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples,
+ NoOpOperationCallback.INSTANCE);
+ getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder());
+ }
+ componentBulkLoader.end();
+
+ return component;
}
private void loadDeleteTuples(ILSMIndexOperationContext opCtx,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDeletedKeysBTreeMergeCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDeletedKeysBTreeMergeCursor.java
index f1f5241..faa90eb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDeletedKeysBTreeMergeCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDeletedKeysBTreeMergeCursor.java
@@ -60,7 +60,6 @@
ArrayList<IIndexAccessor> btreeAccessors = lsmInitialState.getDeletedKeysBTreeAccessors();
for (int i = 0; i < numBTrees; i++) {
rangeCursors[i] = btreeAccessors.get(i).createSearchCursor(false);
- btreeAccessors.get(i).search(rangeCursors[i], btreePredicate);
}
IndexCursorUtils.open(btreeAccessors, rangeCursors, btreePredicate);
try {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
index 12dc23f..020b788 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
@@ -98,10 +98,9 @@
keysOnlyTuple.reset(checkElement.getTuple());
int end = checkElement.getCursorIndex();
for (int i = 0; i < end; i++) {
- if (bloomFilters[i] != null && bloomFilters[i].contains(keysOnlyTuple, hashes)) {
+ if (bloomFilters[i] != null && !bloomFilters[i].contains(keysOnlyTuple, hashes)) {
continue;
}
- deletedKeysBTreeCursors[i].close();
deletedKeysBTreeAccessors.get(i).search(deletedKeysBTreeCursors[i], keySearchPred);
try {
if (deletedKeysBTreeCursors[i].hasNext()) {
@@ -114,4 +113,31 @@
return false;
}
+ @Override
+ public void doClose() throws HyracksDataException {
+ try {
+ super.doClose();
+ } finally {
+ if (deletedKeysBTreeCursors != null) {
+ for (int i = 0; i < deletedKeysBTreeCursors.length; i++) {
+ deletedKeysBTreeCursors[i].close();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void doDestroy() throws HyracksDataException {
+ try {
+ super.doDestroy();
+ } finally {
+ if (deletedKeysBTreeCursors != null) {
+ for (int i = 0; i < deletedKeysBTreeCursors.length; i++) {
+ deletedKeysBTreeCursors[i].destroy();
+ }
+ deletedKeysBTreeCursors = null;
+ }
+ }
+ }
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
index d7482a8..604d374 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
@@ -40,6 +40,7 @@
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexUtils;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.util.trace.ITracer;
public class PartitionedLSMInvertedIndex extends LSMInvertedIndex {
@@ -52,12 +53,12 @@
IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
int[] invertedIndexFields, int[] filterFields, int[] filterFieldsForNonBulkLoadOps,
- int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) throws HyracksDataException {
+ int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, ITracer tracer) throws HyracksDataException {
super(ioManager, virtualBufferCaches, componentFactory, filterHelper, filterFrameFactory, filterManager,
bloomFilterFalsePositiveRate, diskBufferCache, fileManager, invListTypeTraits, invListCmpFactories,
tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler,
ioOpCallbackFactory, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps, durable);
+ invertedIndexFieldsForNonBulkLoadOps, durable, tracer);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
index da3f079..7f3d12f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
@@ -220,9 +220,11 @@
// Assumption: processing inverted list takes time; so, we don't want to keep them on the buffer cache.
// Rather, we utilize the assigned working memory (buffers).
tmpBuffer = page.getBuffer();
- tmpBuffer.rewind();
- buffers.get(currentBufferIdx).rewind();
- buffers.get(currentBufferIdx).put(tmpBuffer);
+
+ // Copies the entire content of the page to the current buffer in the working memory.
+ System.arraycopy(tmpBuffer.array(), 0, buffers.get(currentBufferIdx).array(), 0,
+ buffers.get(currentBufferIdx).capacity());
+ buffers.get(currentBufferIdx).position(buffers.get(currentBufferIdx).capacity());
currentBufferIdx++;
bufferCache.unpin(page);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java
index b033b99..2401c67 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java
@@ -53,6 +53,7 @@
protected ICachedPage page;
protected boolean pinned;
+ protected int pinnedPageId = -1;
public FixedSizeElementInvertedListScanCursor(IBufferCache bufferCache, int fileId, ITypeTraits[] invListFields)
throws HyracksDataException {
@@ -103,14 +104,18 @@
*/
@Override
public void loadPages() throws HyracksDataException {
- if (pinned) {
- unloadPages();
- }
if (currentPageId == endPageId) {
+ // inverted list exhausted, return
return;
}
currentPageId++;
+ if (pinned && pinnedPageId == currentPageId) {
+ // already pinned, return
+ return;
+ }
+ unloadPages();
page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false);
+ pinnedPageId = currentPageId;
pinned = true;
}
@@ -134,7 +139,6 @@
// Deducts 1 since the startPage would be set to bufferCurrentPageId + 1 in loadPages().
this.currentPageId = startPageId - 1;
this.numPages = endPageId - startPageId + 1;
- this.pinned = false;
}
@Override
@@ -158,16 +162,14 @@
@Override
public void doClose() throws HyracksDataException {
- if (pinned) {
- unloadPages();
- }
+ // No op
+ // We allow the inverted list cursor to hold at most one page to avoid
+ // unnecessary pins
}
@Override
public void doDestroy() throws HyracksDataException {
- if (pinned) {
- unloadPages();
- }
+ unloadPages();
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
index 2f4f1d6..14ebe46 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
@@ -204,9 +204,9 @@
try {
if (ctx.getBtreeCursor().hasNext()) {
ctx.getBtreeCursor().next();
- openInvertedListCursor(ctx.getBtreeCursor().getTuple(), listCursor);
+ openInvertedListCursor(ctx.getBtreeCursor().getTuple(), listCursor, ctx);
} else {
- LSMInvertedIndexSearchCursorInitialState initState = new LSMInvertedIndexSearchCursorInitialState();
+ LSMInvertedIndexSearchCursorInitialState initState = ctx.getCursorInitialState();
initState.setInvertedListInfo(0, 0, 0, 0);
listCursor.open(initState, null);
}
@@ -215,8 +215,8 @@
}
}
- public void openInvertedListCursor(ITupleReference btreeTuple, InvertedListCursor listCursor)
- throws HyracksDataException {
+ public void openInvertedListCursor(ITupleReference btreeTuple, InvertedListCursor listCursor,
+ OnDiskInvertedIndexOpContext opCtx) throws HyracksDataException {
int startPageId = IntegerPointable.getInteger(btreeTuple.getFieldData(invListStartPageIdField),
btreeTuple.getFieldStart(invListStartPageIdField));
int endPageId = IntegerPointable.getInteger(btreeTuple.getFieldData(invListEndPageIdField),
@@ -225,7 +225,7 @@
btreeTuple.getFieldStart(invListStartOffField));
int numElements = IntegerPointable.getInteger(btreeTuple.getFieldData(invListNumElementsField),
btreeTuple.getFieldStart(invListNumElementsField));
- LSMInvertedIndexSearchCursorInitialState initState = new LSMInvertedIndexSearchCursorInitialState();
+ LSMInvertedIndexSearchCursorInitialState initState = opCtx.getCursorInitialState();
initState.setInvertedListInfo(startPageId, endPageId, startOff, numElements);
listCursor.open(initState, null);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
index 267cc79..81bc46b9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexSearchCursorInitialState;
import org.apache.hyracks.storage.common.IIndexAccessor;
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.MultiComparator;
@@ -32,12 +33,13 @@
public class OnDiskInvertedIndexOpContext implements IIndexOperationContext {
private final RangePredicate btreePred = new RangePredicate(null, null, true, true, null, null);
- private IIndexAccessor btreeAccessor;
- private IIndexCursor btreeCursor;
- private MultiComparator searchCmp;
+ private final IIndexAccessor btreeAccessor;
+ private final IIndexCursor btreeCursor;
+ private final MultiComparator searchCmp;
// For prefix search on partitioned indexes.
private MultiComparator prefixSearchCmp;
private boolean destroyed = false;
+ private LSMInvertedIndexSearchCursorInitialState cursorInitialState;
public OnDiskInvertedIndexOpContext(BTree btree) throws HyracksDataException {
// TODO: Ignore opcallbacks for now.
@@ -96,4 +98,11 @@
btreeCursor.destroy();
}
}
+
+ public LSMInvertedIndexSearchCursorInitialState getCursorInitialState() {
+ if (cursorInitialState == null) {
+ cursorInitialState = new LSMInvertedIndexSearchCursorInitialState();
+ }
+ return cursorInitialState;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
index d9e7d34..9d99c9e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
@@ -26,7 +26,6 @@
import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
import org.apache.hyracks.storage.am.common.tuples.ConcatenatingTupleReference;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
import org.apache.hyracks.storage.common.EnforcedIndexCursor;
import org.apache.hyracks.storage.common.ICursorInitialState;
@@ -41,7 +40,7 @@
private final BTree btree;
private final IIndexAccessor btreeAccessor;
- private final IInPlaceInvertedIndex invIndex;
+ private final OnDiskInvertedIndex invIndex;
private final IIndexOperationContext opCtx;
private final InvertedListCursor invListRangeSearchCursor;
private boolean isInvListCursorOpen;
@@ -50,11 +49,11 @@
private RangePredicate btreePred;
private final PermutingTupleReference tokenTuple;
- private ConcatenatingTupleReference concatTuple;
+ private final ConcatenatingTupleReference concatTuple;
- public OnDiskInvertedIndexRangeSearchCursor(IInPlaceInvertedIndex invIndex, IIndexOperationContext opCtx)
+ public OnDiskInvertedIndexRangeSearchCursor(OnDiskInvertedIndex invIndex, IIndexOperationContext opCtx)
throws HyracksDataException {
- this.btree = ((OnDiskInvertedIndex) invIndex).getBTree();
+ this.btree = invIndex.getBTree();
this.btreeAccessor = btree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
this.invIndex = invIndex;
this.opCtx = opCtx;
@@ -87,11 +86,7 @@
return true;
}
// The current inverted-list-range-search cursor is exhausted.
- try {
- invListRangeSearchCursor.unloadPages();
- } finally {
- invListRangeSearchCursor.close();
- }
+ invListRangeSearchCursor.close();
isInvListCursorOpen = false;
openInvListRangeSearchCursor();
return isInvListCursorOpen;
@@ -109,14 +104,8 @@
@Override
public void doDestroy() throws HyracksDataException {
try {
- if (isInvListCursorOpen) {
- try {
- invListRangeSearchCursor.unloadPages();
- } finally {
- isInvListCursorOpen = false;
- invListRangeSearchCursor.destroy();
- }
- }
+ invListRangeSearchCursor.destroy();
+ isInvListCursorOpen = false;
} finally {
btreeCursor.destroy();
}
@@ -125,14 +114,8 @@
@Override
public void doClose() throws HyracksDataException {
try {
- if (isInvListCursorOpen) {
- try {
- invListRangeSearchCursor.unloadPages();
- } finally {
- invListRangeSearchCursor.close();
- }
- isInvListCursorOpen = false;
- }
+ invListRangeSearchCursor.close();
+ isInvListCursorOpen = false;
} finally {
btreeCursor.close();
}
@@ -148,7 +131,8 @@
if (btreeCursor.hasNext()) {
btreeCursor.next();
tokenTuple.reset(btreeCursor.getTuple());
- invIndex.openInvertedListCursor(invListRangeSearchCursor, tokenTuple, opCtx);
+ invIndex.openInvertedListCursor(btreeCursor.getTuple(), invListRangeSearchCursor,
+ (OnDiskInvertedIndexOpContext) opCtx);
invListRangeSearchCursor.prepareLoadPages();
invListRangeSearchCursor.loadPages();
concatTuple.reset();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
index eff4f5a..8c6b386 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
@@ -117,7 +117,7 @@
short numTokens = ShortPointable.getShort(btreeTuple.getFieldData(PARTITIONING_NUM_TOKENS_FIELD),
btreeTuple.getFieldStart(PARTITIONING_NUM_TOKENS_FIELD));
InvertedListCursor invListCursor = partSearcher.getCachedInvertedListCursor();
- openInvertedListCursor(btreeTuple, invListCursor);
+ openInvertedListCursor(btreeTuple, invListCursor, ctx);
invListPartitions.addInvertedListCursor(invListCursor, numTokens);
tokenExists = true;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
index ba54ed4..1e812b4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
@@ -62,6 +62,7 @@
import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.PartitionedOnDiskInvertedIndexFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.util.trace.ITracer;
public class InvertedIndexUtils {
@@ -129,7 +130,7 @@
ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] invertedIndexFields,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable,
- IMetadataPageManagerFactory pageManagerFactory) throws HyracksDataException {
+ IMetadataPageManagerFactory pageManagerFactory, ITracer tracer) throws HyracksDataException {
BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, invListTypeTraits,
invListCmpFactories, diskBufferCache, pageManagerFactory);
@@ -166,7 +167,7 @@
filterManager, bloomFilterFalsePositiveRate, diskBufferCache, fileManager, invListTypeTraits,
invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker,
ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps, durable);
+ invertedIndexFieldsForNonBulkLoadOps, durable, tracer);
}
public static PartitionedLSMInvertedIndex createPartitionedLSMInvertedIndex(IIOManager ioManager,
@@ -178,7 +179,7 @@
ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] invertedIndexFields,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable,
- IPageManagerFactory pageManagerFactory) throws HyracksDataException {
+ IPageManagerFactory pageManagerFactory, ITracer tracer) throws HyracksDataException {
BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, invListTypeTraits,
invListCmpFactories, diskBufferCache, pageManagerFactory);
@@ -215,6 +216,6 @@
filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, diskBufferCache, fileManager,
invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterFields,
- filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable);
+ filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable, tracer);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
index ae8713c..2435cc4 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
@@ -55,6 +55,9 @@
import org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestUtils.HyracksTaskTestContext;
import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.common.IIndexAccessParameters;
+import org.apache.hyracks.util.trace.ITraceCategoryRegistry;
+import org.apache.hyracks.util.trace.TraceCategoryRegistry;
+import org.apache.hyracks.util.trace.Tracer;
@SuppressWarnings("rawtypes")
public class LSMInvertedIndexTestContext extends OrderedIndexTestContext {
@@ -184,7 +187,9 @@
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
harness.getIOOperationCallbackFactory(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps, true, harness.getMetadataPageManagerFactory());
+ invertedIndexFieldsForNonBulkLoadOps, true, harness.getMetadataPageManagerFactory(),
+ new Tracer(LSMInvertedIndexTestContext.class.getSimpleName(),
+ ITraceCategoryRegistry.CATEGORIES_ALL, new TraceCategoryRegistry()));
break;
}
case PARTITIONED_LSM: {
@@ -195,7 +200,9 @@
harness.getOperationTracker(), harness.getIOScheduler(),
harness.getIOOperationCallbackFactory(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps, true, harness.getMetadataPageManagerFactory());
+ invertedIndexFieldsForNonBulkLoadOps, true, harness.getMetadataPageManagerFactory(),
+ new Tracer(LSMInvertedIndexTestContext.class.getSimpleName(),
+ ITraceCategoryRegistry.CATEGORIES_ALL, new TraceCategoryRegistry()));
break;
}
default: {