Fix upsert deadlock and upsert with filtered primary only
This change fixes a deadlock that happens when 3 operations
an upsert, a search and a flush happen simulteniously.
If all the memory components are full, the upsert
gets blocked, the upsert could've obtained a lock on the
search key which would block the search not allowing it
to exit the components and not allowing the components
to be cleared and reused.
In addition, the change refactors common LSM index code.
Change-Id: I93fac0f27ab0b3cc071ff38aef90d850cbbce488
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1762
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <hubailmor@gmail.com>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/NoOpFrameOperationCallbackFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/NoOpFrameOperationCallbackFactory.java
index 3952b11..5e3879f 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/NoOpFrameOperationCallbackFactory.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/NoOpFrameOperationCallbackFactory.java
@@ -40,7 +40,7 @@
private static class NoOpFrameOperationCallback implements IFrameOperationCallback {
@Override
- public void frameCompleted(boolean modified) throws HyracksDataException {
+ public void frameCompleted() throws HyracksDataException {
// No Op
}
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
index d04443a..ec7df85 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
@@ -24,12 +24,13 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.primitive.LongPointable;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
-import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.freepage.MutableArrayValueReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.apache.hyracks.storage.am.lsm.common.impls.DiskComponentMetadata;
+import org.apache.hyracks.storage.am.lsm.common.utils.ComponentMetadataUtil;
// A single LSMIOOperationCallback per LSM index used to perform actions around Flush and Merge operations
public abstract class AbstractLSMIOOperationCallback implements ILSMIOOperationCallback {
@@ -93,16 +94,14 @@
}
}
- public abstract long getComponentLSN(List<? extends ILSMComponent> oldComponents) throws HyracksDataException;
-
public void putLSNIntoMetadata(ILSMDiskComponent index, List<ILSMComponent> oldComponents)
throws HyracksDataException {
index.getMetadata().put(LSN_KEY, LongPointable.FACTORY.createPointable(getComponentLSN(oldComponents)));
}
- public static long getTreeIndexLSN(ITreeIndex treeIndex) throws HyracksDataException {
+ public static long getTreeIndexLSN(DiskComponentMetadata md) throws HyracksDataException {
LongPointable pointable = new LongPointable();
- IMetadataPageManager metadataPageManager = (IMetadataPageManager) treeIndex.getPageManager();
+ IMetadataPageManager metadataPageManager = md.getMetadataPageManager();
metadataPageManager.get(metadataPageManager.createMetadataFrame(), LSN_KEY, pointable);
return pointable.getLength() == 0 ? INVALID : pointable.longValue();
}
@@ -139,6 +138,40 @@
return false;
}
+ @Override
+ public final void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents,
+ ILSMDiskComponent newComponent) throws HyracksDataException {
+ //TODO: Copying Filters and all content of the metadata pages for flush operation should be done here
+ if (newComponent != null) {
+ putLSNIntoMetadata(newComponent, oldComponents);
+ if (opType == LSMOperationType.MERGE) {
+ LongPointable markerLsn = LongPointable.FACTORY
+ .createPointable(ComponentMetadataUtil.getLong(oldComponents.get(0).getMetadata(),
+ ComponentMetadataUtil.MARKER_LSN_KEY, ComponentMetadataUtil.NOT_FOUND));
+ newComponent.getMetadata().put(ComponentMetadataUtil.MARKER_LSN_KEY, markerLsn);
+ }
+
+ }
+ }
+
+ public long getComponentLSN(List<? extends ILSMComponent> diskComponents) throws HyracksDataException {
+ if (diskComponents == null) {
+ // Implies a flush IO operation. --> moves the flush pointer
+ // Flush operation of an LSM index are executed sequentially.
+ synchronized (this) {
+ long lsn = mutableLastLSNs[readIndex];
+ return lsn;
+ }
+ }
+ // Get max LSN from the diskComponents. Implies a merge IO operation or Recovery operation.
+ long maxLSN = -1L;
+ for (ILSMComponent c : diskComponents) {
+ DiskComponentMetadata md = ((ILSMDiskComponent) c).getMetadata();
+ maxLSN = Math.max(getTreeIndexLSN(md), maxLSN);
+ }
+ return maxLSN;
+ }
+
/**
* @param component
* @param componentFilePath
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java
index 173c962..b9442dc 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java
@@ -19,18 +19,11 @@
package org.apache.asterix.common.ioopcallbacks;
-import java.util.List;
-
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.primitive.LongPointable;
-import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeDiskComponent;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
-import org.apache.hyracks.storage.am.lsm.common.utils.ComponentMetadataUtil;
public class LSMBTreeIOOperationCallback extends AbstractLSMIOOperationCallback {
@@ -39,42 +32,6 @@
}
@Override
- public void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents,
- ILSMDiskComponent newComponent) throws HyracksDataException {
- //TODO: Copying Filters and all content of the metadata pages for flush operation should be done here
- if (newComponent != null) {
- LSMBTreeDiskComponent btreeComponent = (LSMBTreeDiskComponent) newComponent;
- putLSNIntoMetadata(btreeComponent, oldComponents);
- if (opType == LSMOperationType.MERGE) {
- LongPointable markerLsn = LongPointable.FACTORY
- .createPointable(ComponentMetadataUtil.getLong(oldComponents.get(0).getMetadata(),
- ComponentMetadataUtil.MARKER_LSN_KEY, ComponentMetadataUtil.NOT_FOUND));
- btreeComponent.getMetadata().put(ComponentMetadataUtil.MARKER_LSN_KEY, markerLsn);
- }
-
- }
- }
-
- @Override
- public long getComponentLSN(List<? extends ILSMComponent> diskComponents) throws HyracksDataException {
- if (diskComponents == null) {
- // Implies a flush IO operation. --> moves the flush pointer
- // Flush operation of an LSM index are executed sequentially.
- synchronized (this) {
- long lsn = mutableLastLSNs[readIndex];
- return lsn;
- }
- }
- // Get max LSN from the diskComponents. Implies a merge IO operation or Recovery operation.
- long maxLSN = -1L;
- for (ILSMComponent c : diskComponents) {
- BTree btree = ((LSMBTreeDiskComponent) c).getBTree();
- maxLSN = Math.max(AbstractLSMIOOperationCallback.getTreeIndexLSN(btree), maxLSN);
- }
- return maxLSN;
- }
-
- @Override
public long getComponentFileLSNOffset(ILSMDiskComponent diskComponent, String diskComponentFilePath)
throws HyracksDataException {
if (diskComponentFilePath.endsWith(LSMBTreeFileManager.BTREE_STRING)) {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java
index 6c987d6..0e80397 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java
@@ -18,46 +18,15 @@
*/
package org.apache.asterix.common.ioopcallbacks;
-import java.util.List;
-
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyDiskComponent;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyFileManager;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
public class LSMBTreeWithBuddyIOOperationCallback extends AbstractLSMIOOperationCallback {
@Override
- public void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents,
- ILSMDiskComponent newComponent) throws HyracksDataException {
- if (newComponent != null) {
- LSMBTreeWithBuddyDiskComponent btreeComponent = (LSMBTreeWithBuddyDiskComponent) newComponent;
- putLSNIntoMetadata(btreeComponent, oldComponents);
- }
- }
-
- @Override
- public long getComponentLSN(List<? extends ILSMComponent> diskComponents) throws HyracksDataException {
- if (diskComponents == null) {
- // Implies a flush IO operation <Will never happen currently as Btree with buddy btree is only used with external datasets>
- synchronized (this) {
- long lsn = mutableLastLSNs[readIndex];
- return lsn;
- }
- }
- // Get max LSN from the diskComponents. Implies a merge IO operation or Recovery operation.
- long maxLSN = -1;
- for (Object o : diskComponents) {
- LSMBTreeWithBuddyDiskComponent btreeComponent = (LSMBTreeWithBuddyDiskComponent) o;
- maxLSN = Math.max(AbstractLSMIOOperationCallback.getTreeIndexLSN(btreeComponent.getBTree()), maxLSN);
- }
- return maxLSN;
- }
-
- @Override
public long getComponentFileLSNOffset(ILSMDiskComponent diskComponent, String diskComponentFilePath)
throws HyracksDataException {
if (diskComponentFilePath.endsWith(LSMBTreeWithBuddyFileManager.BTREE_STRING)) {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java
index 657d908..69286d5c 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java
@@ -19,13 +19,9 @@
package org.apache.asterix.common.ioopcallbacks;
-import java.util.List;
-
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
import org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexDiskComponent;
import org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexFileManager;
@@ -36,34 +32,6 @@
}
@Override
- public void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents,
- ILSMDiskComponent newComponent) throws HyracksDataException {
- if (newComponent != null) {
- LSMInvertedIndexDiskComponent invIndexComponent = (LSMInvertedIndexDiskComponent) newComponent;
- putLSNIntoMetadata(invIndexComponent, oldComponents);
- }
- }
-
- @Override
- public long getComponentLSN(List<? extends ILSMComponent> diskComponents) throws HyracksDataException {
- if (diskComponents == null) {
- // Implies a flush IO operation.
- synchronized (this) {
- long lsn = mutableLastLSNs[readIndex];
- return lsn;
- }
- }
- // Get max LSN from the diskComponents. Implies a merge IO operation or Recovery operation.
- long maxLSN = -1;
- for (Object o : diskComponents) {
- LSMInvertedIndexDiskComponent invIndexComponent = (LSMInvertedIndexDiskComponent) o;
- maxLSN = Math.max(AbstractLSMIOOperationCallback.getTreeIndexLSN(invIndexComponent.getDeletedKeysBTree()),
- maxLSN);
- }
- return maxLSN;
- }
-
- @Override
public long getComponentFileLSNOffset(ILSMDiskComponent diskComponent, String diskComponentFilePath)
throws HyracksDataException {
if (diskComponentFilePath.endsWith(LSMInvertedIndexFileManager.DELETED_KEYS_BTREE_SUFFIX)) {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java
index 2dc06f7..4908c08 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java
@@ -19,13 +19,9 @@
package org.apache.asterix.common.ioopcallbacks;
-import java.util.List;
-
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeDiskComponent;
import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeFileManager;
@@ -36,33 +32,6 @@
}
@Override
- public void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents,
- ILSMDiskComponent newComponent) throws HyracksDataException {
- if (newComponent != null) {
- LSMRTreeDiskComponent rtreeComponent = (LSMRTreeDiskComponent) newComponent;
- putLSNIntoMetadata(rtreeComponent, oldComponents);
- }
- }
-
- @Override
- public long getComponentLSN(List<? extends ILSMComponent> diskComponents) throws HyracksDataException {
- if (diskComponents == null) {
- // Implies a flush IO operation.
- synchronized (this) {
- long lsn = mutableLastLSNs[readIndex];
- return lsn;
- }
- }
- // Get max LSN from the diskComponents. Implies a merge IO operation or Recovery operation.
- long maxLSN = INVALID;
- for (Object o : diskComponents) {
- LSMRTreeDiskComponent rtreeComponent = (LSMRTreeDiskComponent) o;
- maxLSN = Math.max(AbstractLSMIOOperationCallback.getTreeIndexLSN(rtreeComponent.getRTree()), maxLSN);
- }
- return maxLSN;
- }
-
- @Override
public long getComponentFileLSNOffset(ILSMDiskComponent diskComponent, String diskComponentFilePath)
throws HyracksDataException {
if (diskComponentFilePath.endsWith(LSMRTreeFileManager.RTREE_STRING)) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalFilesIndexModificationOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalFilesIndexModificationOperatorDescriptor.java
index f4fb455..94ef285 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalFilesIndexModificationOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalFilesIndexModificationOperatorDescriptor.java
@@ -70,7 +70,7 @@
LSMTwoPCBTreeBulkLoader bulkLoader = null;
try {
bulkLoader = (LSMTwoPCBTreeBulkLoader) ((ExternalBTree) index)
- .createTransactionBulkLoader(BTree.DEFAULT_FILL_FACTOR, false, files.size(), false);
+ .createTransactionBulkLoader(BTree.DEFAULT_FILL_FACTOR, false, files.size());
// Load files
// The files must be ordered according to their numbers
for (ExternalFile file : files) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorDescriptor.java
index 52230e0..60f1721 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorDescriptor.java
@@ -33,9 +33,8 @@
public ExternalIndexBulkModifyOperatorDescriptor(IOperatorDescriptorRegistry spec,
IIndexDataflowHelperFactory dataflowHelperFactory, int[] deletedFiles, int[] fieldPermutation,
- float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmpty) {
- super(spec, null, fieldPermutation, fillFactor, verifyInput, numElementsHint, checkIfEmpty,
- dataflowHelperFactory);
+ float fillFactor, boolean verifyInput, long numElementsHint) {
+ super(spec, null, fieldPermutation, fillFactor, verifyInput, numElementsHint, false, dataflowHelperFactory);
this.deletedFiles = deletedFiles;
}
@@ -43,7 +42,7 @@
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new ExternalIndexBulkModifyOperatorNodePushable(indexHelperFactory, ctx, partition, fieldPermutation,
- fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex,
+ fillFactor, verifyInput, numElementsHint,
recordDescProvider.getInputRecordDescriptor(getActivityId(), 0), deletedFiles);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorNodePushable.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorNodePushable.java
index e47bb07..45d424e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorNodePushable.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorNodePushable.java
@@ -43,10 +43,9 @@
public ExternalIndexBulkModifyOperatorNodePushable(IIndexDataflowHelperFactory indexHelperFactory,
IHyracksTaskContext ctx, int partition, int[] fieldPermutation, float fillFactor, boolean verifyInput,
- long numElementsHint, boolean checkIfEmpty, RecordDescriptor inputRecDesc, int[] deletedFiles)
- throws HyracksDataException {
- super(indexHelperFactory, ctx, partition, fieldPermutation, fillFactor, verifyInput, numElementsHint,
- checkIfEmpty, inputRecDesc);
+ long numElementsHint, RecordDescriptor inputRecDesc, int[] deletedFiles) throws HyracksDataException {
+ super(indexHelperFactory, ctx, partition, fieldPermutation, fillFactor, verifyInput, numElementsHint, false,
+ inputRecDesc);
this.deletedFiles = deletedFiles;
}
@@ -61,8 +60,8 @@
try {
writer.open();
// Transactional BulkLoader
- bulkLoader = ((ITwoPCIndex) index).createTransactionBulkLoader(fillFactor, verifyInput, deletedFiles.length,
- checkIfEmptyIndex);
+ bulkLoader =
+ ((ITwoPCIndex) index).createTransactionBulkLoader(fillFactor, verifyInput, deletedFiles.length);
// Delete files
for (int i = 0; i < deletedFiles.length; i++) {
fileNumber.setValue(deletedFiles[i]);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index fa519e1..86e3911 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -515,7 +515,7 @@
}
}
ExternalIndexBulkModifyOperatorDescriptor treeIndexBulkLoadOp = new ExternalIndexBulkModifyOperatorDescriptor(
- spec, dataflowHelperFactory, deletedFiles, fieldPermutation, fillFactor, false, numElementsHint, false);
+ spec, dataflowHelperFactory, deletedFiles, fieldPermutation, fillFactor, false, numElementsHint);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, treeIndexBulkLoadOp,
secondaryPartitionConstraint);
return treeIndexBulkLoadOp;
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
index bff3ca6..40635c8 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
@@ -214,8 +214,8 @@
accessor.reset(buffer);
LSMTreeIndexAccessor lsmAccessor = (LSMTreeIndexAccessor) indexAccessor;
int tupleCount = accessor.getTupleCount();
- boolean firstModification = true;
int i = 0;
+ lsmAccessor.enter();
try {
while (i < tupleCount) {
tb.reset();
@@ -223,7 +223,7 @@
boolean recordWasDeleted = false;
tuple.reset(accessor, i);
resetSearchPredicate(i);
- if (hasSecondaries) {
+ if (isFiltered || hasSecondaries) {
lsmAccessor.search(cursor, searchPred);
if (cursor.hasNext()) {
cursor.next();
@@ -243,31 +243,27 @@
if (isDeleteOperation(tuple, numOfPrimaryKeys)) {
// Only delete if it is a delete and not upsert
abstractModCallback.setOp(Operation.DELETE);
- if (firstModification) {
- lsmAccessor.delete(tuple);
- firstModification = false;
- } else {
- lsmAccessor.forceDelete(tuple);
- }
+ lsmAccessor.forceDelete(tuple);
recordWasDeleted = true;
} else {
abstractModCallback.setOp(Operation.UPSERT);
- if (firstModification) {
- lsmAccessor.upsert(tuple);
- firstModification = false;
- } else {
- lsmAccessor.forceUpsert(tuple);
- }
+ lsmAccessor.forceUpsert(tuple);
recordWasInserted = true;
}
+ if (isFiltered && prevTuple != null) {
+ // need to update the filter of the new component with the previous value
+ lsmAccessor.updateFilter(prevTuple);
+ }
writeOutput(i, recordWasInserted, recordWasDeleted);
i++;
}
// callback here before calling nextFrame on the next operator
- frameOpCallback.frameCompleted(!firstModification);
+ frameOpCallback.frameCompleted();
appender.write(writer, true);
} catch (Exception e) {
throw HyracksDataException.create(e);
+ } finally {
+ lsmAccessor.exit();
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
index e321a1d..7eaeafb 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
@@ -82,6 +82,12 @@
public static final int UNSORTED_LOAD_INPUT = 46;
public static final int OPERATION_EXCEEDED_MAX_RESTARTS = 47;
public static final int DUPLICATE_LOAD_INPUT = 48;
+ public static final int CANNOT_CREATE_ACTIVE_INDEX = 49;
+ public static final int CANNOT_ACTIVATE_ACTIVE_INDEX = 50;
+ public static final int CANNOT_DEACTIVATE_INACTIVE_INDEX = 51;
+ public static final int CANNOT_DESTROY_ACTIVE_INDEX = 52;
+ public static final int CANNOT_CLEAR_INACTIVE_INDEX = 53;
+ public static final int CANNOT_ALLOCATE_MEMORY_FOR_INACTIVE_INDEX = 54;
// Compilation error codes.
public static final int RULECOLLECTION_NOT_INSTANCE_OF_LIST = 10000;
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
index c3c2596..e95e7f4 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
@@ -67,5 +67,11 @@
46 = Unsorted load input
47 = Operation exceeded the maximum number of restarts %1$s
48 = Loading duplicate keys into the primary storage
+49 = Failed to create the index since it is active
+50 = Failed to activate the index since it is active
+51 = Failed to deactivate the index since it is inactive
+52 = Failed to destroy the index since it is active
+53 = Failed to clear the index since it is inactive
+54 = Failed to allocate memory components for the index since it is inactive
10000 = The given rule collection %1$s is not an instance of the List class.
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
index e98db9b..7f57524 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
@@ -535,7 +535,7 @@
}
private final boolean acquireLatch(ICachedPage node, BTreeOpContext ctx, boolean isLeaf) {
- if (!isLeaf || (ctx.getOperation() == IndexOperation.SEARCH && !ctx.getCursor().exclusiveLatchNodes())) {
+ if (!isLeaf || (ctx.getOperation() == IndexOperation.SEARCH && !ctx.getCursor().isExclusiveLatchNodes())) {
node.acquireReadLatch();
return true;
} else {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
index 92f1ae9..d7084bb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
@@ -244,11 +244,6 @@
}
@Override
- public ICachedPage getPage() {
- return page;
- }
-
- @Override
public void setBufferCache(IBufferCache bufferCache) {
this.bufferCache = bufferCache;
}
@@ -259,17 +254,7 @@
}
@Override
- public boolean exclusiveLatchNodes() {
+ public boolean isExclusiveLatchNodes() {
return exclusiveLatchNodes;
}
-
- @Override
- public void markCurrentTupleAsUpdated() throws HyracksDataException {
- if (exclusiveLatchNodes) {
- isPageDirty = true;
- } else {
- throw new HyracksDataException("This cursor has not been created with the intention to allow updates.");
- }
- }
-
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
index 64e7399..32cc7ee 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
@@ -112,11 +112,6 @@
return null;
}
- @Override
- public ICachedPage getPage() {
- return page;
- }
-
public int getTupleOffset() {
return frame.getTupleOffset(tupleIndex - 1);
}
@@ -312,19 +307,10 @@
}
@Override
- public boolean exclusiveLatchNodes() {
+ public boolean isExclusiveLatchNodes() {
return exclusiveLatchNodes;
}
- @Override
- public void markCurrentTupleAsUpdated() throws HyracksDataException {
- if (exclusiveLatchNodes) {
- isPageDirty = true;
- } else {
- throw new HyracksDataException("This cursor has not been created with the intention to allow updates.");
- }
- }
-
public boolean isBloomFilterAware() {
return false;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexCursor.java
index d0771bb..bdaf3aa 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexCursor.java
@@ -19,21 +19,15 @@
package org.apache.hyracks.storage.am.common.api;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
public interface ITreeIndexCursor extends IIndexCursor {
- public ICachedPage getPage();
+ void setBufferCache(IBufferCache bufferCache);
- public void setBufferCache(IBufferCache bufferCache);
-
- public void setFileId(int fileId);
+ void setFileId(int fileId);
// For allowing updates.
- public boolean exclusiveLatchNodes();
-
- public void markCurrentTupleAsUpdated() throws HyracksDataException;
+ boolean isExclusiveLatchNodes();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
index 0ed5f19..163a3d1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
@@ -23,6 +23,7 @@
import java.util.List;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
@@ -84,26 +85,11 @@
@Override
public synchronized void create() throws HyracksDataException {
if (isActive) {
- throw new HyracksDataException("Failed to create the index since it is activated.");
+ throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_ACTIVE_INDEX);
}
synchronized (fileMapProvider) {
- boolean fileIsMapped = fileMapProvider.isMapped(file);
- if (!fileIsMapped) {
- bufferCache.createFile(file);
- }
- fileId = fileMapProvider.lookupFileId(file);
- try {
- // Also creates the file if it doesn't exist yet.
- bufferCache.openFile(fileId);
- } catch (HyracksDataException e) {
- // Revert state of buffer cache since file failed to open.
- if (!fileIsMapped) {
- bufferCache.deleteFile(fileId, false);
- }
- throw e;
- }
+ fileId = createAndOpen(bufferCache, fileMapProvider, file);
}
-
freePageManager.open(fileId);
freePageManager.init(interiorFrameFactory, leafFrameFactory);
setRootPage();
@@ -111,6 +97,27 @@
bufferCache.closeFile(fileId);
}
+ public static int createAndOpen(IBufferCache bufferCache, IFileMapProvider fileMapProvider, FileReference file)
+ throws HyracksDataException {
+ int fileId;
+ boolean fileIsMapped = fileMapProvider.isMapped(file);
+ if (!fileIsMapped) {
+ bufferCache.createFile(file);
+ }
+ fileId = fileMapProvider.lookupFileId(file);
+ try {
+ // Also creates the file if it doesn't exist yet.
+ bufferCache.openFile(fileId);
+ } catch (HyracksDataException e) {
+ // Revert state of buffer cache since file failed to open.
+ if (!fileIsMapped) {
+ bufferCache.deleteFile(fileId, false);
+ }
+ throw e;
+ }
+ return fileId;
+ }
+
private void setRootPage() throws HyracksDataException {
rootPage = freePageManager.getRootPageId();
bulkloadLeafStart = freePageManager.getBulkLoadLeaf();
@@ -119,7 +126,7 @@
@Override
public synchronized void activate() throws HyracksDataException {
if (isActive) {
- throw new HyracksDataException("Failed to activate the index since it is already activated.");
+ throw HyracksDataException.create(ErrorCode.CANNOT_ACTIVATE_ACTIVE_INDEX);
}
boolean fileIsMapped = false;
synchronized (fileMapProvider) {
@@ -150,7 +157,7 @@
@Override
public synchronized void deactivate() throws HyracksDataException {
if (!isActive && hasEverBeenActivated) {
- throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
+ throw HyracksDataException.create(ErrorCode.CANNOT_DEACTIVATE_INACTIVE_INDEX);
}
if (isActive) {
freePageManager.close();
@@ -169,7 +176,7 @@
@Override
public synchronized void destroy() throws HyracksDataException {
if (isActive) {
- throw new HyracksDataException("Failed to destroy the index since it is activated.");
+ throw HyracksDataException.create(ErrorCode.CANNOT_DESTROY_ACTIVE_INDEX);
}
if (fileId == -1) {
@@ -183,7 +190,7 @@
@Override
public synchronized void clear() throws HyracksDataException {
if (!isActive) {
- throw new HyracksDataException("Failed to clear the index since it is not activated.");
+ throw HyracksDataException.create(ErrorCode.CANNOT_CLEAR_INACTIVE_INDEX);
}
freePageManager.init(interiorFrameFactory, leafFrameFactory);
setRootPage();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java
index 88c9e51..0b8b64c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java
@@ -69,11 +69,6 @@
return null;
}
- @Override
- public ICachedPage getPage() {
- return page;
- }
-
private boolean positionToNextLeaf(boolean skipCurrent) throws HyracksDataException {
while (frame.getLevel() != 0 || skipCurrent || frame.getTupleCount() == 0) {
if (++currentPageId > maxPageId) {
@@ -161,12 +156,7 @@
}
@Override
- public boolean exclusiveLatchNodes() {
+ public boolean isExclusiveLatchNodes() {
return false;
}
-
- @Override
- public void markCurrentTupleAsUpdated() throws HyracksDataException {
- throw new HyracksDataException("Updating tuples is not supported with this cursor.");
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/DualTupleReference.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/DualTupleReference.java
index f6ae8c1..ca38853 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/DualTupleReference.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/DualTupleReference.java
@@ -54,7 +54,7 @@
permutingTuple.reset(tuple);
}
- public ITupleReference getPermutingTuple() {
+ public PermutingTupleReference getPermutingTuple() {
return permutingTuple;
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index 1025cf9..1d8de79 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -24,7 +24,6 @@
import java.util.List;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
@@ -59,6 +58,8 @@
import org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
import org.apache.hyracks.storage.am.lsm.common.impls.ExternalIndexHarness;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor.ICursorFactory;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
import org.apache.hyracks.storage.common.IIndexCursor;
@@ -76,6 +77,7 @@
*/
public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
+ private static final ICursorFactory cursorFactory = opCtx -> new LSMBTreeSearchCursor(opCtx);
// This component factory has to be different since it uses different tuple
// writer in it's leaf frames to support inserting both
// regular and delete tuples
@@ -108,6 +110,11 @@
this.interiorFrameFactory = interiorFrameFactory;
}
+ @Override
+ public ExternalIndexHarness getLsmHarness() {
+ return (ExternalIndexHarness) super.getLsmHarness();
+ }
+
// The subsume merged components is overridden to account for:
// Maintaining two versions of the index
@Override
@@ -163,7 +170,7 @@
// This method creates the appropriate opContext for the targeted version
public ExternalBTreeOpContext createOpContext(ISearchOperationCallback searchCallback, int targetVersion) {
return new ExternalBTreeOpContext(insertLeafFrameFactory, deleteLeafFrameFactory, searchCallback,
- componentFactory.getBloomFilterKeyFields().length, cmpFactories, targetVersion, lsmHarness);
+ componentFactory.getBloomFilterKeyFields().length, cmpFactories, targetVersion, getLsmHarness());
}
// The only reason to override the following method is that it uses a different context object
@@ -193,7 +200,7 @@
FileReference lastFile = lastBTree.getFileReference();
LSMComponentFileReferences relMergeFileRefs =
fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
- ILSMIndexAccessor accessor = new LSMBTreeAccessor(lsmHarness, opCtx);
+ ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getLsmHarness(), opCtx, cursorFactory);
ioScheduler.scheduleOperation(new LSMBTreeMergeOperation(accessor, mergingComponents, cursor,
relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(),
callback, fileManager.getBaseDir()));
@@ -244,7 +251,7 @@
@Override
public synchronized void activate() throws HyracksDataException {
- if (isActivated) {
+ if (isActive) {
throw new HyracksDataException("Failed to activate the index since it is already activated.");
}
if (diskComponents.size() == 0 && secondDiskComponents.size() == 0) {
@@ -259,7 +266,7 @@
diskComponents.add(component);
secondDiskComponents.add(component);
}
- ((ExternalIndexHarness) lsmHarness).indexFirstTimeActivated();
+ getLsmHarness().indexFirstTimeActivated();
} else {
// This index has been opened before
for (ILSMComponent c : diskComponents) {
@@ -280,7 +287,7 @@
}
}
}
- isActivated = true;
+ isActive = true;
}
@Override
@@ -291,7 +298,7 @@
@Override
public void deactivate(boolean flushOnExit) throws HyracksDataException {
- if (!isActivated) {
+ if (!isActive) {
throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
}
if (flushOnExit) {
@@ -315,17 +322,17 @@
bloomFilter.deactivate();
}
}
- isActivated = false;
+ isActive = false;
}
// The clear method is not used anywhere in AsterixDB! we override it anyway
// to exit components first and deal with the two lists
@Override
public void clear() throws HyracksDataException {
- if (!isActivated) {
+ if (!isActive) {
throw new HyracksDataException("Failed to clear the index since it is not activated.");
}
- ((ExternalIndexHarness) lsmHarness).indexClear();
+ getLsmHarness().indexClear();
for (ILSMComponent c : diskComponents) {
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
@@ -351,7 +358,7 @@
@Override
public void destroy() throws HyracksDataException {
- if (isActivated) {
+ if (isActive) {
throw new HyracksDataException("Failed to destroy the index since it is activated.");
}
for (ILSMComponent c : diskComponents) {
@@ -372,20 +379,6 @@
version = 0;
}
- @Override
- public void validate() throws HyracksDataException {
- for (ILSMComponent c : diskComponents) {
- BTree btree = ((LSMBTreeDiskComponent) c).getBTree();
- btree.validate();
- }
- for (ILSMComponent c : secondDiskComponents) {
- if (!diskComponents.contains(c)) {
- BTree btree = ((LSMBTreeDiskComponent) c).getBTree();
- btree.validate();
- }
- }
- }
-
// Not supported
@Override
public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException {
@@ -442,16 +435,16 @@
// For initial load
@Override
- public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- return new LSMTwoPCBTreeBulkLoader(fillLevel, verifyInput, numElementsHint, checkIfEmptyIndex, false);
+ public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
+ return new LSMTwoPCBTreeBulkLoader(fillLevel, verifyInput, numElementsHint, false);
}
// For transaction bulk load <- could consolidate with the above method ->
@Override
- public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- return new LSMTwoPCBTreeBulkLoader(fillLevel, verifyInput, numElementsHint, checkIfEmptyIndex, true);
+ public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
+ return new LSMTwoPCBTreeBulkLoader(fillLevel, verifyInput, numElementsHint, true);
}
// The bulk loader used for both initial loading and transaction
@@ -467,15 +460,12 @@
private final ITreeIndexTupleWriterFactory frameTupleWriterFactory;
public LSMTwoPCBTreeBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex, boolean isTransaction) throws HyracksDataException {
+ boolean isTransaction) throws HyracksDataException {
this.isTransaction = isTransaction;
// Create the appropriate target
if (isTransaction) {
component = createTransactionTarget();
} else {
- if (checkIfEmptyIndex && !isEmptyIndex()) {
- throw HyracksDataException.create(ErrorCode.LOAD_NON_EMPTY_INDEX);
- }
component = createBulkLoadTarget();
}
@@ -551,7 +541,7 @@
btree.deactivate();
bloomFilter.deactivate();
} else {
- lsmHarness.addBulkLoadedComponent(component);
+ getLsmHarness().addBulkLoadedComponent(component);
}
}
}
@@ -606,13 +596,15 @@
@Override
public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
- return new LSMBTreeAccessor(lsmHarness, createOpContext(searchCallback, version));
+ ExternalBTreeOpContext opCtx = createOpContext(searchCallback, version);
+ return new LSMTreeIndexAccessor(getLsmHarness(), opCtx, cursorFactory);
}
@Override
public ILSMIndexAccessor createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion)
throws HyracksDataException {
- return new LSMBTreeAccessor(lsmHarness, createOpContext(searchCallback, targetIndexVersion));
+ ExternalBTreeOpContext opCtx = createOpContext(searchCallback, targetIndexVersion);
+ return new LSMTreeIndexAccessor(getLsmHarness(), opCtx, cursorFactory);
}
@Override
@@ -668,7 +660,7 @@
component = createDiskComponent(componentFactory, componentFileRefrences.getInsertIndexFileReference(),
componentFileRefrences.getBloomFilterFileReference(), false);
}
- ((ExternalIndexHarness) lsmHarness).addTransactionComponents(component);
+ getLsmHarness().addTransactionComponents(component);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeOpContext.java
index 69c633c..f94c38a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeOpContext.java
@@ -18,40 +18,27 @@
*/
package org.apache.hyracks.storage.am.lsm.btree.impls;
-import java.util.LinkedList;
-import java.util.List;
-
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
-import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
-import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
public class ExternalBTreeOpContext extends AbstractLSMIndexOperationContext {
private IBTreeLeafFrame insertLeafFrame;
private IBTreeLeafFrame deleteLeafFrame;
- private IndexOperation op;
private final MultiComparator cmp;
private final MultiComparator bloomFilterCmp;
- private final ISearchOperationCallback searchCallback;
- private final List<ILSMComponent> componentHolder;
- private final List<ILSMDiskComponent> componentsToBeMerged;
- private final List<ILSMDiskComponent> componentsToBeReplicated;
private final int targetIndexVersion;
- private ISearchPredicate searchPredicate;
private LSMBTreeCursorInitialState searchInitialState;
public ExternalBTreeOpContext(ITreeIndexFrameFactory insertLeafFrameFactory,
ITreeIndexFrameFactory deleteLeafFrameFactory, ISearchOperationCallback searchCallback,
int numBloomFilterKeyFields, IBinaryComparatorFactory[] cmpFactories, int targetIndexVersion,
ILSMHarness lsmHarness) {
+ super(null, null, null, searchCallback, null);
if (cmpFactories != null) {
this.cmp = MultiComparator.create(cmpFactories);
} else {
@@ -66,80 +53,21 @@
if (deleteLeafFrame != null && this.cmp != null) {
deleteLeafFrame.setMultiComparator(cmp);
}
- this.componentHolder = new LinkedList<>();
- this.componentsToBeMerged = new LinkedList<>();
- this.componentsToBeReplicated = new LinkedList<>();
- this.searchCallback = searchCallback;
this.targetIndexVersion = targetIndexVersion;
searchInitialState = new LSMBTreeCursorInitialState(insertLeafFrameFactory, cmp, bloomFilterCmp, lsmHarness,
null, searchCallback, null);
}
@Override
- public void setOperation(IndexOperation newOp) {
- reset();
- this.op = newOp;
- }
-
- @Override
- public void reset() {
- super.reset();
- componentHolder.clear();
- componentsToBeMerged.clear();
- componentsToBeReplicated.clear();
- }
-
- @Override
- public IndexOperation getOperation() {
- return op;
- }
-
- @Override
- public List<ILSMComponent> getComponentHolder() {
- return componentHolder;
- }
-
- @Override
- public ISearchOperationCallback getSearchOperationCallback() {
- return searchCallback;
- }
-
- // Disk only index should never needs a modification callback
- @Override
- public IModificationOperationCallback getModificationCallback() {
- return null;
- }
-
- @Override
public void setCurrentMutableComponentId(int currentMutableComponentId) {
// Do nothing: this method should never be called for this class
}
- @Override
- public List<ILSMDiskComponent> getComponentsToBeMerged() {
- return componentsToBeMerged;
- }
-
// Used by indexes with global transaction
public int getTargetIndexVersion() {
return targetIndexVersion;
}
- @Override
- public void setSearchPredicate(ISearchPredicate searchPredicate) {
- this.searchPredicate = searchPredicate;
- }
-
- @Override
- public ISearchPredicate getSearchPredicate() {
- return searchPredicate;
- }
-
- @Override
- public List<ILSMDiskComponent> getComponentsToBeReplicated() {
- return componentsToBeReplicated;
- }
-
public LSMBTreeCursorInitialState getSearchInitialState() {
return searchInitialState;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index 6684e68..fed4588 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -48,19 +48,19 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.ITwoPCIndex;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
-import org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.impls.ExternalIndexHarness;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
@@ -70,7 +70,6 @@
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
-import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
@@ -121,7 +120,7 @@
@Override
public void create() throws HyracksDataException {
- if (isActivated) {
+ if (isActive) {
throw new HyracksDataException("Failed to create the index since it is activated.");
}
fileManager.deleteDirs();
@@ -131,8 +130,13 @@
}
@Override
+ protected ILSMDiskComponent loadComponent(LSMComponentFileReferences refs) throws HyracksDataException {
+ return null;
+ }
+
+ @Override
public void activate() throws HyracksDataException {
- if (isActivated) {
+ if (isActive) {
throw new HyracksDataException("Failed to activate the index since it is already activated.");
}
@@ -149,7 +153,7 @@
diskComponents.add(component);
secondDiskComponents.add(component);
}
- ((ExternalIndexHarness) lsmHarness).indexFirstTimeActivated();
+ ((ExternalIndexHarness) getLsmHarness()).indexFirstTimeActivated();
} else {
// This index has been opened before or is brand new with no
// components. It should also maintain the version pointer
@@ -175,66 +179,51 @@
}
}
}
- isActivated = true;
+ isActive = true;
}
@Override
public void clear() throws HyracksDataException {
- if (!isActivated) {
+ if (!isActive) {
throw new HyracksDataException("Failed to clear the index since it is not activated.");
}
- ((ExternalIndexHarness) lsmHarness).indexClear();
-
- for (ILSMComponent c : diskComponents) {
- LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
- component.getBTree().deactivate();
- component.getBuddyBTree().deactivate();
- component.getBloomFilter().deactivate();
- component.getBTree().destroy();
- component.getBloomFilter().destroy();
- component.getBuddyBTree().destroy();
+ ((ExternalIndexHarness) getLsmHarness()).indexClear();
+ for (ILSMDiskComponent c : diskComponents) {
+ clearDiskComponent(c);
// Remove from second list to avoid destroying twice
secondDiskComponents.remove(c);
}
-
- for (ILSMComponent c : secondDiskComponents) {
- LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
- component.getBTree().deactivate();
- component.getBloomFilter().deactivate();
- component.getBuddyBTree().deactivate();
- component.getBTree().destroy();
- component.getBloomFilter().destroy();
- component.getBuddyBTree().destroy();
+ for (ILSMDiskComponent c : secondDiskComponents) {
+ clearDiskComponent(c);
}
-
diskComponents.clear();
secondDiskComponents.clear();
version = 0;
}
@Override
- public void deactivate() throws HyracksDataException {
- deactivate(true);
+ protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
+ component.getBTree().deactivate();
+ component.getBuddyBTree().deactivate();
+ component.getBloomFilter().deactivate();
+ component.getBTree().destroy();
+ component.getBloomFilter().destroy();
+ component.getBuddyBTree().destroy();
}
@Override
public void destroy() throws HyracksDataException {
- if (isActivated) {
+ if (isActive) {
throw new HyracksDataException("Failed to destroy the index since it is activated.");
}
- for (ILSMComponent c : diskComponents) {
- LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
- component.getBTree().destroy();
- component.getBuddyBTree().destroy();
- component.getBloomFilter().destroy();
+ for (ILSMDiskComponent c : diskComponents) {
+ destroyDiskComponent(c);
// Remove from second list to avoid destroying twice
secondDiskComponents.remove(c);
}
- for (ILSMComponent c : secondDiskComponents) {
- LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
- component.getBTree().destroy();
- component.getBuddyBTree().destroy();
- component.getBloomFilter().destroy();
+ for (ILSMDiskComponent c : secondDiskComponents) {
+ destroyDiskComponent(c);
}
diskComponents.clear();
secondDiskComponents.clear();
@@ -243,19 +232,18 @@
}
@Override
+ protected void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
+ component.getBTree().destroy();
+ component.getBuddyBTree().destroy();
+ component.getBloomFilter().destroy();
+ }
+
+ @Override
public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) throws HyracksDataException {
- return new LSMBTreeWithBuddyAccessor(lsmHarness, createOpContext(searchCallback, version));
- }
-
- @Override
- public void validate() throws HyracksDataException {
- throw new UnsupportedOperationException("Validation not implemented for LSM B-Trees with Buddy B-Tree.");
- }
-
- @Override
- public long getMemoryAllocationSize() {
- return 0;
+ return new LSMTreeIndexAccessor(getLsmHarness(), createOpContext(searchCallback, version),
+ ctx -> new LSMBTreeWithBuddySearchCursor(ctx, buddyBTreeFields));
}
// The subsume merged components is overridden to account for:
@@ -288,16 +276,16 @@
// For initial load
@Override
- public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- return new LSMTwoPCBTreeWithBuddyBulkLoader(fillLevel, verifyInput, 0, checkIfEmptyIndex, false);
+ public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
+ return new LSMTwoPCBTreeWithBuddyBulkLoader(fillLevel, verifyInput, 0, false);
}
// For transaction bulk load <- could consolidate with the above method ->
@Override
- public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- return new LSMTwoPCBTreeWithBuddyBulkLoader(fillLevel, verifyInput, numElementsHint, checkIfEmptyIndex, true);
+ public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
+ return new LSMTwoPCBTreeWithBuddyBulkLoader(fillLevel, verifyInput, numElementsHint, true);
}
@Override
@@ -345,7 +333,8 @@
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
ITreeIndexCursor cursor = new LSMBTreeWithBuddySortedCursor(bctx, buddyBTreeFields);
LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
- ILSMIndexAccessor accessor = new LSMBTreeWithBuddyAccessor(lsmHarness, bctx);
+ ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getLsmHarness(), bctx,
+ opCtx -> new LSMBTreeWithBuddySearchCursor(opCtx, buddyBTreeFields));
// Since we have two lists of components, to tell whether we need to
// keep deleted tuples, we need to know
@@ -367,22 +356,21 @@
// This method creates the appropriate opContext for the targeted version
public ExternalBTreeWithBuddyOpContext createOpContext(ISearchOperationCallback searchCallback, int targetVersion) {
return new ExternalBTreeWithBuddyOpContext(btreeCmpFactories, buddyBtreeCmpFactories, searchCallback,
- targetVersion, lsmHarness, btreeInteriorFrameFactory, btreeLeafFrameFactory,
+ targetVersion, getLsmHarness(), btreeInteriorFrameFactory, btreeLeafFrameFactory,
buddyBtreeLeafFrameFactory);
}
@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
LSMBTreeWithBuddyMergeOperation mergeOp = (LSMBTreeWithBuddyMergeOperation) operation;
- ITreeIndexCursor cursor = mergeOp.getCursor();
+ IIndexCursor cursor = mergeOp.getCursor();
ISearchPredicate btreeSearchPred = new RangePredicate(null, null, true, true, null, null);
ILSMIndexOperationContext opCtx = ((LSMBTreeWithBuddySortedCursor) cursor).getOpCtx();
opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
search(opCtx, cursor, btreeSearchPred);
- LSMBTreeWithBuddyDiskComponent mergedComponent =
- createDiskComponent(componentFactory, mergeOp.getBTreeMergeTarget(), mergeOp.getBuddyBTreeMergeTarget(),
- mergeOp.getBloomFilterMergeTarget(), true);
+ LSMBTreeWithBuddyDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(),
+ mergeOp.getBuddyBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
// In case we must keep the deleted-keys BuddyBTrees, then they must be
// merged *before* merging the b-trees so that
@@ -516,40 +504,25 @@
@Override
public void deactivate(boolean flushOnExit) throws HyracksDataException {
- if (!isActivated) {
+ if (!isActive) {
throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
}
-
if (flushOnExit) {
- BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
- cb.afterFinalize(LSMOperationType.FLUSH, null);
+ ioOpCallback.afterFinalize(LSMOperationType.FLUSH, null);
}
// Even though, we deactivate the index, we don't exit components or
// modify any of the lists to make sure they
// are there if the index was opened again
-
- for (ILSMComponent c : diskComponents) {
- LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
- BTree btree = component.getBTree();
- BTree buddyBtree = component.getBuddyBTree();
- BloomFilter bloomFilter = component.getBloomFilter();
- btree.deactivateCloseHandle();
- buddyBtree.deactivateCloseHandle();
- bloomFilter.deactivate();
+ for (ILSMDiskComponent c : diskComponents) {
+ deactivateDiskComponent(c);
}
- for (ILSMComponent c : secondDiskComponents) {
+ for (ILSMDiskComponent c : secondDiskComponents) {
// Only deactivate non shared components
if (!diskComponents.contains(c)) {
- LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
- BTree btree = component.getBTree();
- BTree buddyBtree = component.getBuddyBTree();
- BloomFilter bloomFilter = component.getBloomFilter();
- btree.deactivateCloseHandle();
- buddyBtree.deactivateCloseHandle();
- bloomFilter.deactivate();
+ deactivateDiskComponent(c);
}
}
- isActivated = false;
+ isActive = false;
}
@Override
@@ -611,26 +584,6 @@
// accessor will try to do it
// we could throw the exception here but we don't. it will eventually be
// thrown by the index itself
- public class LSMBTreeWithBuddyAccessor extends LSMTreeIndexAccessor {
- public LSMBTreeWithBuddyAccessor(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) {
- super(lsmHarness, ctx);
- }
-
- @Override
- public ITreeIndexCursor createSearchCursor(boolean exclusive) {
- return new LSMBTreeWithBuddySearchCursor(ctx, buddyBTreeFields);
- }
-
- public MultiComparator getBTreeMultiComparator() {
- ExternalBTreeWithBuddyOpContext concreteCtx = (ExternalBTreeWithBuddyOpContext) ctx;
- return concreteCtx.getBTreeMultiComparator();
- }
-
- public MultiComparator getBodyBTreeMultiComparator() {
- ExternalBTreeWithBuddyOpContext concreteCtx = (ExternalBTreeWithBuddyOpContext) ctx;
- return concreteCtx.getBuddyBTreeMultiComparator();
- }
- }
// The bulk loader used for both initial loading and transaction
// modifications
@@ -645,15 +598,12 @@
private final boolean isTransaction;
public LSMTwoPCBTreeWithBuddyBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex, boolean isTransaction) throws HyracksDataException {
+ boolean isTransaction) throws HyracksDataException {
this.isTransaction = isTransaction;
// Create the appropriate target
if (isTransaction) {
component = createTransactionTarget();
} else {
- if (checkIfEmptyIndex && !isEmptyIndex()) {
- throw HyracksDataException.create(ErrorCode.LOAD_NON_EMPTY_INDEX);
- }
component = createBulkLoadTarget();
}
@@ -730,7 +680,7 @@
buddyBtree.deactivate();
bloomFilter.deactivate();
} else {
- lsmHarness.addBulkLoadedComponent(component);
+ getLsmHarness().addBulkLoadedComponent(component);
}
}
}
@@ -781,7 +731,8 @@
@Override
public ILSMIndexAccessor createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion)
throws HyracksDataException {
- return new LSMBTreeWithBuddyAccessor(lsmHarness, createOpContext(searchCallback, targetIndexVersion));
+ return new LSMTreeIndexAccessor(getLsmHarness(), createOpContext(searchCallback, targetIndexVersion),
+ ctx -> new LSMBTreeWithBuddySearchCursor(ctx, buddyBTreeFields));
}
// This function in an instance of this index is only used after a bulk load
@@ -825,7 +776,7 @@
componentFileRefrences.getDeleteIndexFileReference(),
componentFileRefrences.getBloomFilterFileReference(), false);
}
- ((ExternalIndexHarness) lsmHarness).addTransactionComponents(component);
+ ((ExternalIndexHarness) getLsmHarness()).addTransactionComponents(component);
}
@Override
@@ -859,7 +810,74 @@
}
@Override
- public void allocateMemoryComponents() throws HyracksDataException {
+ protected void deactivateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
//do nothing since external index never use memory components
}
+
+ @Override
+ protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
+ BTree btree = component.getBTree();
+ BTree buddyBtree = component.getBuddyBTree();
+ BloomFilter bloomFilter = component.getBloomFilter();
+ btree.deactivateCloseHandle();
+ buddyBtree.deactivateCloseHandle();
+ bloomFilter.deactivate();
+ }
+
+ @Override
+ protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ //do nothing since external index never use memory components
+ }
+
+ @Override
+ protected void clearMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ //do nothing since external index never use memory components
+ }
+
+ @Override
+ protected long getMemoryComponentSize(ILSMMemoryComponent c) {
+ return 0;
+ }
+
+ @Override
+ protected void validateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ throw new UnsupportedOperationException("Validation not implemented for LSM B-Trees with Buddy B-Tree.");
+ }
+
+ @Override
+ protected void validateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ throw new UnsupportedOperationException("Validation not implemented for LSM B-Trees with Buddy B-Tree.");
+ }
+
+ @Override
+ protected void allocateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ //do nothing since external index never use memory components
+ }
+
+ @Override
+ protected LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent,
+ ILSMDiskComponent lastComponent) throws HyracksDataException {
+ return null;
+ }
+
+ @Override
+ protected AbstractLSMIndexOperationContext createOpContext(IModificationOperationCallback modificationCallback,
+ ISearchOperationCallback searchCallback) {
+ return null;
+ }
+
+ @Override
+ protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx,
+ ILSMMemoryComponent flushingComponent, LSMComponentFileReferences componentFileRefs,
+ ILSMIOOperationCallback callback) throws HyracksDataException {
+ return null;
+ }
+
+ @Override
+ protected ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx,
+ List<ILSMComponent> mergingComponents, LSMComponentFileReferences mergeFileRefs,
+ ILSMIOOperationCallback callback) throws HyracksDataException {
+ return null;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddyOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddyOpContext.java
index ce18b20..d5cd2e2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddyOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddyOpContext.java
@@ -18,42 +18,26 @@
*/
package org.apache.hyracks.storage.am.lsm.btree.impls;
-import java.util.LinkedList;
-import java.util.List;
-
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
-import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
-import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
public class ExternalBTreeWithBuddyOpContext extends AbstractLSMIndexOperationContext {
- private IndexOperation op;
private MultiComparator bTreeCmp;
private MultiComparator buddyBTreeCmp;
- private final List<ILSMComponent> componentHolder;
- private final List<ILSMDiskComponent> componentsToBeMerged;
- private final List<ILSMDiskComponent> componentsToBeReplicated;
- private final ISearchOperationCallback searchCallback;
private final int targetIndexVersion;
- private ISearchPredicate searchPredicate;
private LSMBTreeWithBuddyCursorInitialState searchInitialState;
public ExternalBTreeWithBuddyOpContext(IBinaryComparatorFactory[] btreeCmpFactories,
IBinaryComparatorFactory[] buddyBtreeCmpFactories, ISearchOperationCallback searchCallback,
int targetIndexVersion, ILSMHarness lsmHarness, ITreeIndexFrameFactory btreeInteriorFrameFactory,
ITreeIndexFrameFactory btreeLeafFrameFactory, ITreeIndexFrameFactory buddyBtreeLeafFrameFactory) {
- this.componentHolder = new LinkedList<>();
- this.componentsToBeMerged = new LinkedList<>();
- this.componentsToBeReplicated = new LinkedList<>();
- this.searchCallback = searchCallback;
+ super(null, null, null, searchCallback, null);
this.targetIndexVersion = targetIndexVersion;
this.bTreeCmp = MultiComparator.create(btreeCmpFactories);
this.buddyBTreeCmp = MultiComparator.create(buddyBtreeCmpFactories);
@@ -63,29 +47,10 @@
}
@Override
- public void setOperation(IndexOperation newOp) {
- reset();
- this.op = newOp;
- }
-
- @Override
public void setCurrentMutableComponentId(int currentMutableComponentId) {
// Do nothing. this should never be called for disk only indexes
}
- @Override
- public void reset() {
- super.reset();
- componentHolder.clear();
- componentsToBeMerged.clear();
- componentsToBeReplicated.clear();
- }
-
- @Override
- public IndexOperation getOperation() {
- return op;
- }
-
public MultiComparator getBTreeMultiComparator() {
return bTreeCmp;
}
@@ -94,46 +59,16 @@
return buddyBTreeCmp;
}
- @Override
- public List<ILSMComponent> getComponentHolder() {
- return componentHolder;
- }
-
- @Override
- public ISearchOperationCallback getSearchOperationCallback() {
- return searchCallback;
- }
-
// This should never be needed for disk only indexes
@Override
public IModificationOperationCallback getModificationCallback() {
return null;
}
- @Override
- public List<ILSMDiskComponent> getComponentsToBeMerged() {
- return componentsToBeMerged;
- }
-
public int getTargetIndexVersion() {
return targetIndexVersion;
}
- @Override
- public void setSearchPredicate(ISearchPredicate searchPredicate) {
- this.searchPredicate = searchPredicate;
- }
-
- @Override
- public ISearchPredicate getSearchPredicate() {
- return searchPredicate;
- }
-
- @Override
- public List<ILSMDiskComponent> getComponentsToBeReplicated() {
- return componentsToBeReplicated;
- }
-
public LSMBTreeWithBuddyCursorInitialState getSearchInitialState() {
return searchInitialState;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 1c99d5a..29532d4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -36,40 +36,36 @@
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification;
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
-import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeBulkLoader;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.common.api.IPageManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.impls.AbstractSearchPredicate;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
-import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
-import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
-import org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor.ICursorFactory;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.common.IIndexAccessor;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
@@ -77,12 +73,12 @@
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
-import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
+ private static final ICursorFactory cursorFactory = opCtx -> new LSMBTreeSearchCursor(opCtx);
// For creating BTree's used in flush and merge.
protected final LSMBTreeDiskComponentFactory componentFactory;
// For creating BTree's used in bulk load. Different from diskBTreeFactory
@@ -95,7 +91,7 @@
protected final IBinaryComparatorFactory[] cmpFactories;
private final boolean needKeyDupCheck;
- private final int[] btreeFields;
+
// Primary LSMBTree has a Bloomfilter, but Secondary one doesn't have.
private final boolean hasBloomFilter;
@@ -111,7 +107,7 @@
int[] btreeFields, int[] filterFields, boolean durable) throws HyracksDataException {
super(ioManager, virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
- filterManager, filterFields, durable);
+ filterManager, filterFields, durable, filterFactory, btreeFields);
this.insertLeafFrameFactory = insertLeafFrameFactory;
this.deleteLeafFrameFactory = deleteLeafFrameFactory;
this.cmpFactories = cmpFactories;
@@ -131,7 +127,6 @@
bulkLoadComponentFactory =
new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory, filterFactory);
this.needKeyDupCheck = needKeyDupCheck;
- this.btreeFields = btreeFields;
this.hasBloomFilter = needKeyDupCheck;
}
@@ -151,172 +146,62 @@
componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, null);
bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory, null);
this.needKeyDupCheck = needKeyDupCheck;
- this.btreeFields = null;
//TODO remove BloomFilter from external dataset's secondary LSMBTree index
this.hasBloomFilter = true;
}
- @Override
- public synchronized void create() throws HyracksDataException {
- if (isActivated) {
- throw new HyracksDataException("Failed to create the index since it is activated.");
- }
- // Why delete is part of the create??
- fileManager.deleteDirs();
- fileManager.createDirs();
- diskComponents.clear();
+ public boolean hasBloomFilter() {
+ return hasBloomFilter;
}
@Override
- public synchronized void activate() throws HyracksDataException {
- if (isActivated) {
- throw new HyracksDataException("Failed to activate the index since it is already activated.");
- }
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- immutableComponents.clear();
- List<LSMComponentFileReferences> validFileReferences;
- validFileReferences = fileManager.cleanupAndGetValidFiles();
- for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
- LSMBTreeDiskComponent component;
- component = createDiskComponent(componentFactory, lsmComonentFileReference.getInsertIndexFileReference(),
- lsmComonentFileReference.getBloomFilterFileReference(), false);
- immutableComponents.add(component);
- }
- isActivated = true;
+ public boolean isPrimaryIndex() {
+ return needKeyDupCheck;
}
@Override
- public synchronized void deactivate(boolean flushOnExit) throws HyracksDataException {
- if (!isActivated) {
- throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
- }
-
- if (flushOnExit) {
- BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
- ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- accessor.scheduleFlush(cb);
- try {
- cb.waitForIO();
- } catch (InterruptedException e) {
- throw new HyracksDataException(e);
- }
- }
-
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
- component.getBTree().deactivateCloseHandle();
- if (hasBloomFilter) {
- component.getBloomFilter().deactivate();
- }
- }
- deallocateMemoryComponents();
- isActivated = false;
+ public IBinaryComparatorFactory[] getComparatorFactories() {
+ return cmpFactories;
}
@Override
- public synchronized void deactivate() throws HyracksDataException {
- deactivate(true);
+ protected ILSMDiskComponent loadComponent(LSMComponentFileReferences lsmComonentFileReferences)
+ throws HyracksDataException {
+ return createDiskComponent(componentFactory, lsmComonentFileReferences.getInsertIndexFileReference(),
+ lsmComonentFileReferences.getBloomFilterFileReference(), false);
}
@Override
- public void destroy() throws HyracksDataException {
- if (isActivated) {
- throw new HyracksDataException("Failed to destroy the index since it is activated.");
+ protected void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+ component.getBTree().destroy();
+ if (hasBloomFilter) {
+ component.getBloomFilter().destroy();
}
-
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
- component.getBTree().destroy();
- if (hasBloomFilter) {
- component.getBloomFilter().destroy();
- }
- }
- for (ILSMComponent c : memoryComponents) {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- mutableComponent.getBTree().destroy();
- }
- fileManager.deleteDirs();
}
@Override
- public void clear() throws HyracksDataException {
- if (!isActivated) {
- throw new HyracksDataException("Failed to clear the index since it is not activated.");
- }
-
- clearMemoryComponents();
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMComponent c : immutableComponents) {
- LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
- if (hasBloomFilter) {
- component.getBloomFilter().deactivate();
- }
- component.getBTree().deactivate();
- if (hasBloomFilter) {
- component.getBloomFilter().destroy();
- }
- component.getBTree().destroy();
- }
- immutableComponents.clear();
+ protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+ mutableComponent.getBTree().destroy();
}
@Override
- public void getOperationalComponents(ILSMIndexOperationContext ctx) throws HyracksDataException {
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
- int cmc = currentMutableComponentId.get();
- ctx.setCurrentMutableComponentId(cmc);
- operationalComponents.clear();
- switch (ctx.getOperation()) {
- case UPDATE:
- case PHYSICALDELETE:
- case FLUSH:
- case DELETE:
- case UPSERT:
- operationalComponents.add(memoryComponents.get(cmc));
- break;
- case INSERT:
- addOperationalMutableComponents(operationalComponents);
- operationalComponents.addAll(immutableComponents);
- break;
- case SEARCH:
- if (memoryComponentsAllocated) {
- addOperationalMutableComponents(operationalComponents);
- }
- if (filterManager != null) {
- for (ILSMComponent c : immutableComponents) {
- if (c.getLSMComponentFilter().satisfy(
- ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMinFilterTuple(),
- ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMaxFilterTuple(),
- ((LSMBTreeOpContext) ctx).getFilterCmp())) {
- operationalComponents.add(c);
- }
- }
- } else {
- operationalComponents.addAll(immutableComponents);
- }
-
- break;
- case MERGE:
- operationalComponents.addAll(ctx.getComponentsToBeMerged());
- break;
- case FULL_MERGE:
- operationalComponents.addAll(immutableComponents);
- break;
- case REPLICATE:
- operationalComponents.addAll(ctx.getComponentsToBeReplicated());
- break;
- default:
- throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
+ protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+ if (hasBloomFilter) {
+ component.getBloomFilter().deactivate();
}
+ component.getBTree().deactivate();
+ if (hasBloomFilter) {
+ component.getBloomFilter().destroy();
+ }
+ component.getBTree().destroy();
}
@Override
public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException {
LSMBTreeOpContext ctx = (LSMBTreeOpContext) ictx;
-
ITupleReference indexTuple;
if (ctx.getIndexTuple() != null) {
ctx.getIndexTuple().reset(tuple);
@@ -336,11 +221,7 @@
ctx.getCurrentMutableBTreeAccessor().upsert(indexTuple);
break;
}
- if (ctx.getFilterTuple() != null) {
- ctx.getFilterTuple().reset(tuple);
- memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter().update(ctx.getFilterTuple(),
- ctx.getFilterCmp());
- }
+ updateFilter(ctx, tuple);
}
private boolean insert(ITupleReference tuple, LSMBTreeOpContext ctx) throws HyracksDataException {
@@ -403,21 +284,6 @@
}
@Override
- public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
- throws HyracksDataException {
- ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
- LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- LSMBTreeOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- assert ctx.getComponentHolder().size() == 1;
- opCtx.setOperation(IndexOperation.FLUSH);
- opCtx.getComponentHolder().add(flushingComponent);
- ILSMIndexAccessor flushAccessor = new LSMBTreeAccessor(lsmHarness, opCtx);
- ioScheduler.scheduleOperation(new LSMBTreeFlushOperation(flushAccessor, flushingComponent,
- componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getBloomFilterFileReference(),
- callback, fileManager.getBaseDir()));
- }
-
- @Override
public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException {
LSMBTreeFlushOperation flushOp = (LSMBTreeFlushOperation) operation;
LSMBTreeMemoryComponent flushingComponent = (LSMBTreeMemoryComponent) flushOp.getFlushingComponent();
@@ -446,8 +312,8 @@
bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate);
}
- LSMBTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getBTreeFlushTarget(),
- flushOp.getBloomFilterFlushTarget(), true);
+ LSMBTreeDiskComponent component =
+ createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getBloomFilterTarget(), true);
IIndexBulkLoader bulkLoader = component.getBTree().createBulkLoader(1.0f, false, numElements, false);
IIndexBulkLoader builder = null;
if (hasBloomFilter) {
@@ -476,8 +342,8 @@
List<ITupleReference> filterTuples = new ArrayList<>();
filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
- filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilter(component.getLSMComponentFilter(), component.getBTree());
+ getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
+ getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getBTree());
}
// Write metadata from memory component to disk
// Q. what about the merge operation? how do we resolve conflicts
@@ -492,33 +358,9 @@
}
@Override
- public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
- throws HyracksDataException {
- LSMBTreeOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- opCtx.setOperation(IndexOperation.MERGE);
- List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
- boolean returnDeletedTuples = false;
- if (ctx.getComponentHolder().get(ctx.getComponentHolder().size() - 1) != diskComponents
- .get(diskComponents.size() - 1)) {
- returnDeletedTuples = true;
- }
- ITreeIndexCursor cursor = new LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples);
- BTree firstBTree = ((LSMBTreeDiskComponent) mergingComponents.get(0)).getBTree();
- BTree lastBTree = ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1)).getBTree();
- FileReference firstFile = firstBTree.getFileReference();
- FileReference lastFile = lastBTree.getFileReference();
- LSMComponentFileReferences relMergeFileRefs =
- fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
- ILSMIndexAccessor accessor = new LSMBTreeAccessor(lsmHarness, opCtx);
- ioScheduler.scheduleOperation(new LSMBTreeMergeOperation(accessor, mergingComponents, cursor,
- relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(),
- callback, fileManager.getBaseDir()));
- }
-
- @Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
LSMBTreeMergeOperation mergeOp = (LSMBTreeMergeOperation) operation;
- ITreeIndexCursor cursor = mergeOp.getCursor();
+ IIndexCursor cursor = mergeOp.getCursor();
RangePredicate rangePred = new RangePredicate(null, null, true, true, null, null);
ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
@@ -535,8 +377,8 @@
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate);
}
- LSMBTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getBTreeMergeTarget(),
- mergeOp.getBloomFilterMergeTarget(), true);
+ LSMBTreeDiskComponent mergedComponent =
+ createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBloomFilterTarget(), true);
IIndexBulkLoader bulkLoader = mergedComponent.getBTree().createBulkLoader(1.0f, false, numElements, false);
IIndexBulkLoader builder = null;
@@ -565,8 +407,8 @@
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
}
- filterManager.updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getBTree());
+ getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
+ getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getBTree());
}
bulkLoader.end();
return mergedComponent;
@@ -590,15 +432,15 @@
component.getBloomFilter().activate();
}
if (component.getLSMComponentFilter() != null && !createComponent) {
- filterManager.readFilter(component.getLSMComponentFilter(), component.getBTree());
+ getFilterManager().readFilter(component.getLSMComponentFilter(), component.getBTree());
}
return component;
}
@Override
- public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- return new LSMBTreeBulkLoader(fillLevel, verifyInput, numElementsHint, checkIfEmptyIndex);
+ public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
+ return new LSMBTreeBulkLoader(this, fillLevel, verifyInput, numElementsHint);
}
protected ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException {
@@ -618,168 +460,32 @@
markAsValidInternal(component.getBTree());
}
- public class LSMBTreeBulkLoader implements IIndexBulkLoader {
- private final ILSMDiskComponent component;
- private final BTreeBulkLoader bulkLoader;
- private final IIndexBulkLoader builder;
- private boolean cleanedUpArtifacts = false;
- private boolean isEmptyComponent = true;
- private boolean endedBloomFilterLoad = false;
- public final PermutingTupleReference indexTuple;
- public final PermutingTupleReference filterTuple;
- public final MultiComparator filterCmp;
-
- public LSMBTreeBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- if (checkIfEmptyIndex && !isEmptyIndex()) {
- throw HyracksDataException.create(ErrorCode.LOAD_NON_EMPTY_INDEX);
- }
- component = createBulkLoadTarget();
- bulkLoader = (BTreeBulkLoader) ((LSMBTreeDiskComponent) component).getBTree().createBulkLoader(fillFactor,
- verifyInput, numElementsHint, false);
-
- if (hasBloomFilter) {
- int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
- BloomFilterSpecification bloomFilterSpec =
- BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate);
- builder = ((LSMBTreeDiskComponent) component).getBloomFilter().createBuilder(numElementsHint,
- bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
- } else {
- builder = null;
- }
-
- if (filterFields != null) {
- indexTuple = new PermutingTupleReference(btreeFields);
- filterCmp = MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories());
- filterTuple = new PermutingTupleReference(filterFields);
- } else {
- indexTuple = null;
- filterCmp = null;
- filterTuple = null;
- }
- }
-
- @Override
- public void add(ITupleReference tuple) throws HyracksDataException {
- try {
- ITupleReference t;
- if (indexTuple != null) {
- indexTuple.reset(tuple);
- t = indexTuple;
- } else {
- t = tuple;
- }
-
- bulkLoader.add(t);
- if (hasBloomFilter) {
- builder.add(t);
- }
-
- if (filterTuple != null) {
- filterTuple.reset(tuple);
- component.getLSMComponentFilter().update(filterTuple, filterCmp);
- }
- } catch (Exception e) {
- cleanupArtifacts();
- throw e;
- }
- if (isEmptyComponent) {
- isEmptyComponent = false;
- }
- }
-
- protected void cleanupArtifacts() throws HyracksDataException {
- if (!cleanedUpArtifacts) {
- cleanedUpArtifacts = true;
- if (hasBloomFilter && !endedBloomFilterLoad) {
- builder.abort();
- endedBloomFilterLoad = true;
- }
- ((LSMBTreeDiskComponent) component).getBTree().deactivate();
- ((LSMBTreeDiskComponent) component).getBTree().destroy();
- if (hasBloomFilter) {
- ((LSMBTreeDiskComponent) component).getBloomFilter().deactivate();
- ((LSMBTreeDiskComponent) component).getBloomFilter().destroy();
- }
- }
- }
-
- @Override
- public void end() throws HyracksDataException {
- if (!cleanedUpArtifacts) {
- if (hasBloomFilter && !endedBloomFilterLoad) {
- builder.end();
- endedBloomFilterLoad = true;
- }
-
- if (component.getLSMComponentFilter() != null) {
- filterManager.writeFilter(component.getLSMComponentFilter(),
- ((LSMBTreeDiskComponent) component).getBTree());
- }
- bulkLoader.end();
-
- if (isEmptyComponent) {
- cleanupArtifacts();
- } else {
- //TODO(amoudi): Ensure Bulk load follow the same lifecycle Other Operations (Flush, Merge, etc).
- //then after operation should be called from harness as well
- //https://issues.apache.org/jira/browse/ASTERIXDB-1764
- ioOpCallback.afterOperation(LSMOperationType.FLUSH, null, component);
- lsmHarness.addBulkLoadedComponent(component);
- }
- }
- }
-
- @Override
- public void abort() throws HyracksDataException {
- if (bulkLoader != null) {
- bulkLoader.abort();
- }
-
- if (builder != null) {
- builder.abort();
- }
-
- }
+ @Override
+ protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx,
+ ILSMMemoryComponent flushingComponent, LSMComponentFileReferences componentFileRefs,
+ ILSMIOOperationCallback callback) {
+ ILSMIndexAccessor accessor = createAccessor(opCtx);
+ return new LSMBTreeFlushOperation(accessor, flushingComponent, componentFileRefs.getInsertIndexFileReference(),
+ componentFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir());
}
+ @Override
public LSMBTreeOpContext createOpContext(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
int numBloomFilterKeyFields = hasBloomFilter ? componentFactory.getBloomFilterKeyFields().length : 0;
return new LSMBTreeOpContext(memoryComponents, insertLeafFrameFactory, deleteLeafFrameFactory,
- modificationCallback, searchCallback, numBloomFilterKeyFields, btreeFields, filterFields, lsmHarness);
+ modificationCallback, searchCallback, numBloomFilterKeyFields, getTreeFields(), getFilterFields(),
+ getLsmHarness(), getFilterCmpFactories());
}
@Override
public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
- return new LSMBTreeAccessor(lsmHarness, createOpContext(modificationCallback, searchCallback));
+ return createAccessor(createOpContext(modificationCallback, searchCallback));
}
- public class LSMBTreeAccessor extends LSMTreeIndexAccessor {
- public LSMBTreeAccessor(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) {
- super(lsmHarness, ctx);
- }
-
- @Override
- public IIndexCursor createSearchCursor(boolean exclusive) {
- return new LSMBTreeSearchCursor(ctx);
- }
-
- public MultiComparator getMultiComparator() {
- LSMBTreeOpContext concreteCtx = (LSMBTreeOpContext) ctx;
- return concreteCtx.getCmp();
- }
- }
-
- @Override
- public IBufferCache getBufferCache() {
- return diskBufferCache;
- }
-
- @Override
- public IBinaryComparatorFactory[] getComparatorFactories() {
- return cmpFactories;
+ public ILSMIndexAccessor createAccessor(AbstractLSMIndexOperationContext opCtx) {
+ return new LSMTreeIndexAccessor(getLsmHarness(), opCtx, cursorFactory);
}
@Override
@@ -818,17 +524,6 @@
}
@Override
- public long getMemoryAllocationSize() {
- long size = 0;
- for (ILSMComponent c : memoryComponents) {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- IBufferCache virtualBufferCache = mutableComponent.getBTree().getBufferCache();
- size += virtualBufferCache.getNumPages() * virtualBufferCache.getPageSize();
- }
- return size;
- }
-
- @Override
public int getRootPageId() {
LSMBTreeMemoryComponent mutableComponent =
(LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
@@ -836,13 +531,10 @@
}
@Override
- public void validate() throws HyracksDataException {
- validateMemoryComponents();
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- BTree btree = ((LSMBTreeDiskComponent) c).getBTree();
- btree.validate();
- }
+ protected long getMemoryComponentSize(ILSMMemoryComponent c) {
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+ IBufferCache virtualBufferCache = mutableComponent.getBTree().getBufferCache();
+ return virtualBufferCache.getNumPages() * (long) virtualBufferCache.getPageSize();
}
@Override
@@ -851,11 +543,6 @@
}
@Override
- public boolean isPrimaryIndex() {
- return needKeyDupCheck;
- }
-
- @Override
public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
Set<String> files = new HashSet<>();
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) lsmComponent;
@@ -867,65 +554,71 @@
}
@Override
- public synchronized void allocateMemoryComponents() throws HyracksDataException {
- if (!isActivated) {
- throw new HyracksDataException("Failed to allocate memory components since the index is not active");
- }
- if (memoryComponentsAllocated) {
- return;
- }
- for (ILSMComponent c : memoryComponents) {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).open();
- mutableComponent.getBTree().create();
- mutableComponent.getBTree().activate();
- }
- memoryComponentsAllocated = true;
+ protected void clearMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+ mutableComponent.getBTree().clear();
+ mutableComponent.reset();
}
- private void addOperationalMutableComponents(List<ILSMComponent> operationalComponents) {
- int cmc = currentMutableComponentId.get();
- int numMutableComponents = memoryComponents.size();
- for (int i = 0; i < numMutableComponents - 1; i++) {
- ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- if (mutableComponent.isReadable()) {
- // Make sure newest components are added first
- operationalComponents.add(0, mutableComponent);
- }
- }
- // The current mutable component is always added
- operationalComponents.add(0, memoryComponents.get(cmc));
+ @Override
+ protected void validateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+ mutableComponent.getBTree().validate();
}
- private synchronized void clearMemoryComponents() throws HyracksDataException {
- if (memoryComponentsAllocated) {
- for (ILSMComponent c : memoryComponents) {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- mutableComponent.getBTree().clear();
- mutableComponent.reset();
- }
+ @Override
+ protected void validateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ BTree btree = ((LSMBTreeDiskComponent) c).getBTree();
+ btree.validate();
+ }
+
+ @Override
+ protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+ component.getBTree().deactivateCloseHandle();
+ if (hasBloomFilter) {
+ component.getBloomFilter().deactivate();
}
}
- private synchronized void validateMemoryComponents() throws HyracksDataException {
- if (memoryComponentsAllocated) {
- for (ILSMComponent c : memoryComponents) {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- mutableComponent.getBTree().validate();
- }
- }
+ @Override
+ protected void deactivateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+ mutableComponent.getBTree().deactivate();
+ mutableComponent.getBTree().destroy();
+ ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).close();
}
- private synchronized void deallocateMemoryComponents() throws HyracksDataException {
- if (memoryComponentsAllocated) {
- for (ILSMComponent c : memoryComponents) {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- mutableComponent.getBTree().deactivate();
- mutableComponent.getBTree().destroy();
- ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).close();
- }
- memoryComponentsAllocated = false;
+ @Override
+ protected void allocateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+ ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).open();
+ mutableComponent.getBTree().create();
+ mutableComponent.getBTree().activate();
+ }
+
+ @Override
+ protected LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent,
+ ILSMDiskComponent lastComponent) throws HyracksDataException {
+ BTree firstBTree = ((LSMBTreeDiskComponent) firstComponent).getBTree();
+ BTree lastBTree = ((LSMBTreeDiskComponent) lastComponent).getBTree();
+ FileReference firstFile = firstBTree.getFileReference();
+ FileReference lastFile = lastBTree.getFileReference();
+ return fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
+ }
+
+ @Override
+ protected ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx,
+ List<ILSMComponent> mergingComponents, LSMComponentFileReferences mergeFileRefs,
+ ILSMIOOperationCallback callback) {
+ boolean returnDeletedTuples = false;
+ ILSMIndexAccessor accessor = createAccessor(opCtx);
+ if (mergingComponents.get(mergingComponents.size() - 1) != diskComponents.get(diskComponents.size() - 1)) {
+ returnDeletedTuples = true;
}
+ ITreeIndexCursor cursor = new LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples);
+ return new LSMBTreeMergeOperation(accessor, mergingComponents, cursor,
+ mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getBloomFilterFileReference(), callback,
+ fileManager.getBaseDir());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java
new file mode 100644
index 0000000..247b3de
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.impls;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
+import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification;
+import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeBulkLoader;
+import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.apache.hyracks.storage.common.IIndexBulkLoader;
+import org.apache.hyracks.storage.common.MultiComparator;
+
+public class LSMBTreeBulkLoader implements IIndexBulkLoader {
+ private final LSMBTree lsmIndex;
+ private final ILSMDiskComponent component;
+ private final BTreeBulkLoader bulkLoader;
+ private final IIndexBulkLoader builder;
+ private boolean cleanedUpArtifacts = false;
+ private boolean isEmptyComponent = true;
+ private boolean endedBloomFilterLoad = false;
+ public final PermutingTupleReference indexTuple;
+ public final PermutingTupleReference filterTuple;
+ public final MultiComparator filterCmp;
+
+ public LSMBTreeBulkLoader(LSMBTree lsmIndex, float fillFactor, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
+ this.lsmIndex = lsmIndex;
+ component = lsmIndex.createBulkLoadTarget();
+ bulkLoader = (BTreeBulkLoader) ((LSMBTreeDiskComponent) component).getBTree().createBulkLoader(fillFactor,
+ verifyInput, numElementsHint, false);
+
+ if (lsmIndex.hasBloomFilter()) {
+ int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
+ BloomFilterSpecification bloomFilterSpec =
+ BloomCalculations.computeBloomSpec(maxBucketsPerElement, lsmIndex.bloomFilterFalsePositiveRate());
+ builder = ((LSMBTreeDiskComponent) component).getBloomFilter().createBuilder(numElementsHint,
+ bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
+ } else {
+ builder = null;
+ }
+
+ if (lsmIndex.getFilterFields() != null) {
+ indexTuple = new PermutingTupleReference(lsmIndex.getTreeFields());
+ filterCmp = MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories());
+ filterTuple = new PermutingTupleReference(lsmIndex.getFilterFields());
+ } else {
+ indexTuple = null;
+ filterCmp = null;
+ filterTuple = null;
+ }
+ }
+
+ @Override
+ public void add(ITupleReference tuple) throws HyracksDataException {
+ try {
+ ITupleReference t;
+ if (indexTuple != null) {
+ indexTuple.reset(tuple);
+ t = indexTuple;
+ } else {
+ t = tuple;
+ }
+
+ bulkLoader.add(t);
+ if (lsmIndex.hasBloomFilter()) {
+ builder.add(t);
+ }
+
+ if (filterTuple != null) {
+ filterTuple.reset(tuple);
+ component.getLSMComponentFilter().update(filterTuple, filterCmp);
+ }
+ } catch (Exception e) {
+ cleanupArtifacts();
+ throw e;
+ }
+ if (isEmptyComponent) {
+ isEmptyComponent = false;
+ }
+ }
+
+ private void cleanupArtifacts() throws HyracksDataException {
+ if (!cleanedUpArtifacts) {
+ cleanedUpArtifacts = true;
+ if (lsmIndex.hasBloomFilter() && !endedBloomFilterLoad) {
+ builder.abort();
+ endedBloomFilterLoad = true;
+ }
+ ((LSMBTreeDiskComponent) component).getBTree().deactivate();
+ ((LSMBTreeDiskComponent) component).getBTree().destroy();
+ if (lsmIndex.hasBloomFilter()) {
+ ((LSMBTreeDiskComponent) component).getBloomFilter().deactivate();
+ ((LSMBTreeDiskComponent) component).getBloomFilter().destroy();
+ }
+ }
+ }
+
+ @Override
+ public void end() throws HyracksDataException {
+ if (!cleanedUpArtifacts) {
+ if (lsmIndex.hasBloomFilter() && !endedBloomFilterLoad) {
+ builder.end();
+ endedBloomFilterLoad = true;
+ }
+
+ if (component.getLSMComponentFilter() != null) {
+ lsmIndex.getFilterManager().writeFilter(component.getLSMComponentFilter(),
+ ((LSMBTreeDiskComponent) component).getBTree());
+ }
+ bulkLoader.end();
+
+ if (isEmptyComponent) {
+ cleanupArtifacts();
+ } else {
+ //TODO(amoudi): Ensure Bulk load follow the same lifecycle Other Operations (Flush, Merge, etc).
+ //then after operation should be called from harness as well
+ //https://issues.apache.org/jira/browse/ASTERIXDB-1764
+ lsmIndex.getIOOperationCallback().afterOperation(LSMOperationType.FLUSH, null, component);
+ lsmIndex.getLsmHarness().addBulkLoadedComponent(component);
+ }
+ }
+ }
+
+ @Override
+ public void abort() throws HyracksDataException {
+ if (bulkLoader != null) {
+ bulkLoader.abort();
+ }
+
+ if (builder != null) {
+ builder.abort();
+ }
+
+ }
+}
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java
index c57c35f..4a06778 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java
@@ -18,92 +18,23 @@
*/
package org.apache.hyracks.storage.am.lsm.btree.impls;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IODeviceHandle;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.FlushOperation;
-public class LSMBTreeFlushOperation implements ILSMIOOperation, Comparable<LSMBTreeFlushOperation> {
-
- private final ILSMIndexAccessor accessor;
- private final ILSMComponent flushingComponent;
- private final FileReference btreeFlushTarget;
+public class LSMBTreeFlushOperation extends FlushOperation {
private final FileReference bloomFilterFlushTarget;
- private final ILSMIOOperationCallback callback;
- private final String indexIdentifier;
- public LSMBTreeFlushOperation(ILSMIndexAccessor accessor, ILSMComponent flushingComponent,
- FileReference btreeFlushTarget, FileReference bloomFilterFlushTarget, ILSMIOOperationCallback callback,
+ public LSMBTreeFlushOperation(ILSMIndexAccessor accessor, ILSMMemoryComponent flushingComponent,
+ FileReference flushTarget, FileReference bloomFilterFlushTarget, ILSMIOOperationCallback callback,
String indexIdentifier) {
- this.accessor = accessor;
- this.flushingComponent = flushingComponent;
- this.btreeFlushTarget = btreeFlushTarget;
+ super(accessor, flushingComponent, flushTarget, callback, indexIdentifier);
this.bloomFilterFlushTarget = bloomFilterFlushTarget;
- this.callback = callback;
- this.indexIdentifier = indexIdentifier;
}
- @Override
- public Set<IODeviceHandle> getReadDevices() {
- return Collections.emptySet();
- }
-
- @Override
- public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<>();
- devs.add(btreeFlushTarget.getDeviceHandle());
- if (bloomFilterFlushTarget != null) {
- devs.add(bloomFilterFlushTarget.getDeviceHandle());
- }
- return devs;
- }
-
- @Override
- public Boolean call() throws HyracksDataException {
- accessor.flush(this);
- return true;
- }
-
- @Override
- public ILSMIOOperationCallback getCallback() {
- return callback;
- }
-
- public FileReference getBTreeFlushTarget() {
- return btreeFlushTarget;
- }
-
- public FileReference getBloomFilterFlushTarget() {
+ public FileReference getBloomFilterTarget() {
return bloomFilterFlushTarget;
}
-
- public ILSMIndexAccessor getAccessor() {
- return accessor;
- }
-
- public ILSMComponent getFlushingComponent() {
- return flushingComponent;
- }
-
- @Override
- public String getIndexUniqueIdentifier() {
- return indexIdentifier;
- }
-
- @Override
- public LSMIOOpertionType getIOOpertionType() {
- return LSMIOOpertionType.FLUSH;
- }
-
- @Override
- public int compareTo(LSMBTreeFlushOperation o) {
- return btreeFlushTarget.getFile().getName().compareTo(o.getBTreeFlushTarget().getFile().getName());
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java
index fbf6c5c..0cc76f2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java
@@ -19,98 +19,27 @@
package org.apache.hyracks.storage.am.lsm.btree.impls;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation;
-public class LSMBTreeMergeOperation implements ILSMIOOperation {
+public class LSMBTreeMergeOperation extends MergeOperation {
- private final ILSMIndexAccessor accessor;
- private final List<ILSMComponent> mergingComponents;
- private final ITreeIndexCursor cursor;
- private final FileReference btreeMergeTarget;
private final FileReference bloomFilterMergeTarget;
- private final ILSMIOOperationCallback callback;
- private final String indexIdentifier;
public LSMBTreeMergeOperation(ILSMIndexAccessor accessor, List<ILSMComponent> mergingComponents,
- ITreeIndexCursor cursor, FileReference btreeMergeTarget, FileReference bloomFilterMergeTarget,
+ ITreeIndexCursor cursor, FileReference target, FileReference bloomFilterMergeTarget,
ILSMIOOperationCallback callback, String indexIdentifier) {
- this.accessor = accessor;
- this.mergingComponents = mergingComponents;
- this.cursor = cursor;
- this.btreeMergeTarget = btreeMergeTarget;
+ super(accessor, target, callback, indexIdentifier, mergingComponents, cursor);
this.bloomFilterMergeTarget = bloomFilterMergeTarget;
- this.callback = callback;
- this.indexIdentifier = indexIdentifier;
}
- @Override
- public Set<IODeviceHandle> getReadDevices() {
- Set<IODeviceHandle> devs = new HashSet<>();
- for (ILSMComponent o : mergingComponents) {
- LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) o;
- devs.add(component.getBTree().getFileReference().getDeviceHandle());
- if (bloomFilterMergeTarget != null) {
- devs.add(component.getBloomFilter().getFileReference().getDeviceHandle());
- }
- }
- return devs;
- }
-
- @Override
- public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<>();
- devs.add(btreeMergeTarget.getDeviceHandle());
- if (bloomFilterMergeTarget != null) {
- devs.add(bloomFilterMergeTarget.getDeviceHandle());
- }
- return devs;
- }
-
- @Override
- public Boolean call() throws HyracksDataException {
- accessor.merge(this);
- return true;
- }
-
- @Override
- public ILSMIOOperationCallback getCallback() {
- return callback;
- }
-
- public FileReference getBTreeMergeTarget() {
- return btreeMergeTarget;
- }
-
- public FileReference getBloomFilterMergeTarget() {
+ public FileReference getBloomFilterTarget() {
return bloomFilterMergeTarget;
}
-
- public ITreeIndexCursor getCursor() {
- return cursor;
- }
-
- public List<ILSMComponent> getMergingComponents() {
- return mergingComponents;
- }
-
- @Override
- public String getIndexUniqueIdentifier() {
- return indexIdentifier;
- }
-
- @Override
- public LSMIOOpertionType getIOOpertionType() {
- return LSMIOOpertionType.MERGE;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
index 8b7a8a8..77a8aeb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.lsm.btree.impls;
-import java.util.LinkedList;
import java.util.List;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -31,15 +30,11 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
-import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
public final class LSMBTreeOpContext extends AbstractLSMIndexOperationContext {
@@ -56,14 +51,6 @@
private final BTreeOpContext[] mutableBTreeOpCtxs;
private final MultiComparator cmp;
private final MultiComparator bloomFilterCmp;
- private final IModificationOperationCallback modificationCallback;
- private final ISearchOperationCallback searchCallback;
- private final List<ILSMComponent> componentHolder;
- private final List<ILSMDiskComponent> componentsToBeMerged;
- private final List<ILSMDiskComponent> componentsToBeReplicated;
- private final PermutingTupleReference indexTuple;
- private final MultiComparator filterCmp;
- private final PermutingTupleReference filterTuple;
private final BTreeRangeSearchCursor memCursor;
private final LSMBTreeCursorInitialState searchInitialState;
private final LSMBTreePointSearchCursor insertSearchCursor;
@@ -72,13 +59,12 @@
*/
private BTree.BTreeAccessor currentMutableBTreeAccessor;
private BTreeOpContext currentMutableBTreeOpCtx;
- private IndexOperation op;
- private ISearchPredicate searchPredicate;
public LSMBTreeOpContext(List<ILSMMemoryComponent> mutableComponents, ITreeIndexFrameFactory insertLeafFrameFactory,
ITreeIndexFrameFactory deleteLeafFrameFactory, IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback, int numBloomFilterKeyFields, int[] btreeFields, int[] filterFields,
- ILSMHarness lsmHarness) {
+ ILSMHarness lsmHarness, IBinaryComparatorFactory[] filterCmpFactories) {
+ super(btreeFields, filterFields, filterCmpFactories, searchCallback, modificationCallback);
LSMBTreeMemoryComponent c = (LSMBTreeMemoryComponent) mutableComponents.get(0);
IBinaryComparatorFactory cmpFactories[] = c.getBTree().getComparatorFactories();
if (cmpFactories[0] != null) {
@@ -100,7 +86,6 @@
NoOpOperationCallback.INSTANCE);
mutableBTreeOpCtxs[i] = mutableBTreeAccessors[i].getOpContext();
}
-
this.insertLeafFrameFactory = insertLeafFrameFactory;
this.deleteLeafFrameFactory = deleteLeafFrameFactory;
this.insertLeafFrame = (IBTreeLeafFrame) insertLeafFrameFactory.createFrame();
@@ -111,21 +96,6 @@
if (deleteLeafFrame != null && this.getCmp() != null) {
deleteLeafFrame.setMultiComparator(getCmp());
}
- this.componentHolder = new LinkedList<>();
- this.componentsToBeMerged = new LinkedList<>();
- this.componentsToBeReplicated = new LinkedList<>();
- this.modificationCallback = modificationCallback;
- this.searchCallback = searchCallback;
-
- if (filterFields != null) {
- indexTuple = new PermutingTupleReference(btreeFields);
- filterCmp = MultiComparator.create(c.getLSMComponentFilter().getFilterCmpFactories());
- filterTuple = new PermutingTupleReference(filterFields);
- } else {
- indexTuple = null;
- filterCmp = null;
- filterTuple = null;
- }
searchPredicate = new RangePredicate(null, null, true, true, getCmp(), getCmp());
memCursor = (insertLeafFrame != null) ? new BTreeRangeSearchCursor(insertLeafFrame, false) : null;
searchInitialState = new LSMBTreeCursorInitialState(insertLeafFrameFactory, getCmp(), bloomFilterCmp,
@@ -150,34 +120,6 @@
}
@Override
- public void reset() {
- super.reset();
- componentHolder.clear();
- componentsToBeMerged.clear();
- componentsToBeReplicated.clear();
- }
-
- @Override
- public IndexOperation getOperation() {
- return op;
- }
-
- @Override
- public List<ILSMComponent> getComponentHolder() {
- return componentHolder;
- }
-
- @Override
- public ISearchOperationCallback getSearchOperationCallback() {
- return searchCallback;
- }
-
- @Override
- public IModificationOperationCallback getModificationCallback() {
- return modificationCallback;
- }
-
- @Override
public void setCurrentMutableComponentId(int currentMutableComponentId) {
setCurrentMutableBTreeAccessor(mutableBTreeAccessors[currentMutableComponentId]);
currentMutableBTreeOpCtx = mutableBTreeOpCtxs[currentMutableComponentId];
@@ -200,34 +142,6 @@
}
}
- @Override
- public List<ILSMDiskComponent> getComponentsToBeMerged() {
- return componentsToBeMerged;
- }
-
- @Override
- public void setSearchPredicate(ISearchPredicate searchPredicate) {
- this.searchPredicate = searchPredicate;
- }
-
- @Override
- public ISearchPredicate getSearchPredicate() {
- return searchPredicate;
- }
-
- @Override
- public List<ILSMDiskComponent> getComponentsToBeReplicated() {
- return componentsToBeReplicated;
- }
-
- public MultiComparator getFilterCmp() {
- return filterCmp;
- }
-
- public PermutingTupleReference getIndexTuple() {
- return indexTuple;
- }
-
public BTree.BTreeAccessor getCurrentMutableBTreeAccessor() {
return currentMutableBTreeAccessor;
}
@@ -236,10 +150,6 @@
this.currentMutableBTreeAccessor = currentMutableBTreeAccessor;
}
- public PermutingTupleReference getFilterTuple() {
- return filterTuple;
- }
-
public LSMBTreePointSearchCursor getInsertSearchCursor() {
return insertSearchCursor;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
index 31c47ea..6126c53 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
@@ -30,14 +30,17 @@
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
-import org.apache.hyracks.storage.am.lsm.common.api.*;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleReference;
import org.apache.hyracks.storage.am.lsm.common.impls.BloomFilterAwareBTreePointSearchCursor;
import org.apache.hyracks.storage.common.ICursorInitialState;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
public class LSMBTreePointSearchCursor implements ITreeIndexCursor {
@@ -248,12 +251,6 @@
}
@Override
- public ICachedPage getPage() {
- // do nothing
- return null;
- }
-
- @Override
public void setBufferCache(IBufferCache bufferCache) {
// do nothing
}
@@ -264,12 +261,8 @@
}
@Override
- public boolean exclusiveLatchNodes() {
+ public boolean isExclusiveLatchNodes() {
return false;
}
- @Override
- public void markCurrentTupleAsUpdated() throws HyracksDataException {
- throw new HyracksDataException("Updating tuples is not supported with this cursor.");
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeSearchCursor.java
index 9a19b39..c9bee31 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeSearchCursor.java
@@ -27,7 +27,6 @@
import org.apache.hyracks.storage.common.ICursorInitialState;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
public class LSMBTreeSearchCursor implements ITreeIndexCursor {
@@ -96,11 +95,6 @@
}
@Override
- public ICachedPage getPage() {
- return currentCursor.getPage();
- }
-
- @Override
public void setBufferCache(IBufferCache bufferCache) {
currentCursor.setBufferCache(bufferCache);
}
@@ -112,12 +106,8 @@
}
@Override
- public boolean exclusiveLatchNodes() {
- return currentCursor.exclusiveLatchNodes();
+ public boolean isExclusiveLatchNodes() {
+ return currentCursor.isExclusiveLatchNodes();
}
- @Override
- public void markCurrentTupleAsUpdated() throws HyracksDataException {
- throw new HyracksDataException("Updating tuples is not supported with this cursor.");
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java
index 4a91de8..03d28d0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java
@@ -38,7 +38,6 @@
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
public abstract class LSMBTreeWithBuddyAbstractCursor implements ITreeIndexCursor {
@@ -164,12 +163,6 @@
}
@Override
- public ICachedPage getPage() {
- // Do nothing
- return null;
- }
-
- @Override
public void setBufferCache(IBufferCache bufferCache) {
// Do nothing
}
@@ -180,13 +173,8 @@
}
@Override
- public boolean exclusiveLatchNodes() {
+ public boolean isExclusiveLatchNodes() {
return false;
}
- @Override
- public void markCurrentTupleAsUpdated() throws HyracksDataException {
- throw new HyracksDataException("Updating tuples is not supported with this cursor.");
- }
-
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java
index 0806834..2817f3a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java
@@ -18,112 +18,39 @@
*/
package org.apache.hyracks.storage.am.lsm.btree.impls;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation;
-public class LSMBTreeWithBuddyMergeOperation implements ILSMIOOperation {
+public class LSMBTreeWithBuddyMergeOperation extends MergeOperation {
- private final ILSMIndexAccessor accessor;
- private final List<ILSMComponent> mergingComponents;
- private final ITreeIndexCursor cursor;
- private final FileReference btreeMergeTarget;
private final FileReference buddyBtreeMergeTarget;
private final FileReference bloomFilterMergeTarget;
- private final ILSMIOOperationCallback callback;
- private final String indexIdentifier;
private final boolean keepDeletedTuples;
public LSMBTreeWithBuddyMergeOperation(ILSMIndexAccessor accessor, List<ILSMComponent> mergingComponents,
- ITreeIndexCursor cursor, FileReference btreeMergeTarget, FileReference buddyBtreeMergeTarget,
+ ITreeIndexCursor cursor, FileReference target, FileReference buddyBtreeMergeTarget,
FileReference bloomFilterMergeTarget, ILSMIOOperationCallback callback, String indexIdentifier,
boolean keepDeletedTuples) {
- this.accessor = accessor;
- this.mergingComponents = mergingComponents;
- this.cursor = cursor;
- this.btreeMergeTarget = btreeMergeTarget;
+ super(accessor, target, callback, indexIdentifier, mergingComponents, cursor);
this.buddyBtreeMergeTarget = buddyBtreeMergeTarget;
this.bloomFilterMergeTarget = bloomFilterMergeTarget;
- this.callback = callback;
- this.indexIdentifier = indexIdentifier;
this.keepDeletedTuples = keepDeletedTuples;
}
- @Override
- public Set<IODeviceHandle> getReadDevices() {
- Set<IODeviceHandle> devs = new HashSet<>();
- for (ILSMComponent o : mergingComponents) {
- LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) o;
- devs.add(component.getBTree().getFileReference().getDeviceHandle());
-
- devs.add(component.getBuddyBTree().getFileReference().getDeviceHandle());
- devs.add(component.getBloomFilter().getFileReference().getDeviceHandle());
-
- }
- return devs;
- }
-
- @Override
- public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<>();
- devs.add(btreeMergeTarget.getDeviceHandle());
-
- devs.add(buddyBtreeMergeTarget.getDeviceHandle());
- devs.add(bloomFilterMergeTarget.getDeviceHandle());
-
- return devs;
- }
-
- @Override
- public Boolean call() throws HyracksDataException {
- accessor.merge(this);
- return true;
- }
-
- @Override
- public ILSMIOOperationCallback getCallback() {
- return callback;
- }
-
- @Override
- public String getIndexUniqueIdentifier() {
- return indexIdentifier;
- }
-
- @Override
- public LSMIOOpertionType getIOOpertionType() {
- return LSMIOOpertionType.MERGE;
- }
-
- public FileReference getBTreeMergeTarget() {
- return btreeMergeTarget;
- }
-
- public FileReference getBuddyBTreeMergeTarget() {
+ public FileReference getBuddyBTreeTarget() {
return buddyBtreeMergeTarget;
}
- public FileReference getBloomFilterMergeTarget() {
+ public FileReference getBloomFilterTarget() {
return bloomFilterMergeTarget;
}
- public ITreeIndexCursor getCursor() {
- return cursor;
- }
-
- public List<ILSMComponent> getMergingComponents() {
- return mergingComponents;
- }
-
public boolean isKeepDeletedTuples() {
return keepDeletedTuples;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
index de72690..96eb559 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
@@ -29,9 +29,7 @@
* Called once processing the frame is done before calling nextFrame on the next IFrameWriter in
* the pipeline
*
- * @param modified
- * true if the index was modified during the processing of the frame, false otherwise
* @throws HyracksDataException
*/
- void frameCompleted(boolean modified) throws HyracksDataException;
+ void frameCompleted() throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFactory.java
index 64562bd..6284d4e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFactory.java
@@ -18,8 +18,17 @@
*/
package org.apache.hyracks.storage.am.lsm.common.api;
-@FunctionalInterface
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+
public interface ILSMComponentFilterFactory {
+ /**
+ * @return an instance of ILSMComponentFilter
+ */
ILSMComponentFilter createFilter();
+ /**
+ * @return the array of binary comparator factories for the filter
+ */
+ IBinaryComparatorFactory[] getFilterCmpFactories();
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java
index e57acab..f9a4e80 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java
@@ -68,8 +68,7 @@
* @throws HyracksDataException
* @throws IndexException
*/
- void search(ILSMIndexOperationContext ctx, IIndexCursor cursor, ISearchPredicate pred)
- throws HyracksDataException;
+ void search(ILSMIndexOperationContext ctx, IIndexCursor cursor, ISearchPredicate pred) throws HyracksDataException;
/**
* End the search
@@ -87,8 +86,7 @@
* @throws HyracksDataException
* @throws IndexException
*/
- void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
- throws HyracksDataException;
+ void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback) throws HyracksDataException;
/**
* Schedule full merge
@@ -98,8 +96,7 @@
* @throws HyracksDataException
* @throws IndexException
*/
- void scheduleFullMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
- throws HyracksDataException;
+ void scheduleFullMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback) throws HyracksDataException;
/**
* Perform a merge operation
@@ -198,4 +195,28 @@
*/
void forceUpdateMeta(ILSMIndexOperationContext ctx, IValueReference key, IValueReference value)
throws HyracksDataException;
+
+ /**
+ * Enter components for the operation
+ *
+ * @param ctx
+ * @throws HyracksDataException
+ */
+ void enter(ILSMIndexOperationContext ctx) throws HyracksDataException;
+
+ /**
+ * Exits components for the operation
+ *
+ * @param ctx
+ * @throws HyracksDataException
+ */
+ void exit(ILSMIndexOperationContext ctx) throws HyracksDataException;
+
+ /**
+ * Update the filter with the value in the passed tuple
+ *
+ * @param ctx
+ * @throws HyracksDataException
+ */
+ void updateFilter(ILSMIndexOperationContext ctx, ITupleReference tuple) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
index 829523c..79360d5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
@@ -18,7 +18,6 @@
*/
package org.apache.hyracks.storage.am.lsm.common.api;
-import java.util.Set;
import java.util.concurrent.Callable;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -26,21 +25,34 @@
public interface ILSMIOOperation extends Callable<Boolean> {
+ /**
+ * Represents the io operation type
+ */
enum LSMIOOpertionType {
FLUSH,
MERGE
}
- Set<IODeviceHandle> getReadDevices();
+ /**
+ * @return the device on which the operation is running
+ */
+ IODeviceHandle getDevice();
- Set<IODeviceHandle> getWriteDevices();
+ /**
+ * @return the operation callback
+ */
+ ILSMIOOperationCallback getCallback();
+
+ /**
+ * @return the index id
+ */
+ String getIndexIdentifier();
+
+ /**
+ * @return the operation type
+ */
+ LSMIOOpertionType getIOOpertionType();
@Override
Boolean call() throws HyracksDataException;
-
- ILSMIOOperationCallback getCallback();
-
- String getIndexUniqueIdentifier();
-
- LSMIOOpertionType getIOOpertionType();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
index 9fb48c4..2a9186b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
@@ -42,7 +42,7 @@
*/
public interface ILSMIndex extends IIndex {
- void deactivate(boolean flushOnExit) throws HyracksDataException;
+ void deactivate(boolean flush) throws HyracksDataException;
@Override
ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
@@ -130,4 +130,13 @@
* @return true if the index is durable. Otherwise false.
*/
boolean isDurable();
+
+ /**
+ * Update the filter with the passed tuple
+ *
+ * @param ictx
+ * @param tuple
+ * @throws HyracksDataException
+ */
+ void updateFilter(ILSMIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
index 5473670..f846c80 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
@@ -33,6 +33,17 @@
* concurrent operations).
*/
public interface ILSMIndexAccessor extends IIndexAccessor {
+
+ /**
+ * Enter the memory component for modification
+ */
+ void enter() throws HyracksDataException;
+
+ /**
+ * Exit the memory component
+ */
+ void exit() throws HyracksDataException;
+
/**
* Schedule a flush operation
*
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java
index 8bdf5fa..5b0378a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java
@@ -21,9 +21,11 @@
import java.util.List;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
+import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.MultiComparator;
public interface ILSMIndexOperationContext extends IIndexOperationContext {
List<ILSMComponent> getComponentHolder();
@@ -48,4 +50,10 @@
boolean isAccessingComponents();
void setAccessingComponents(boolean accessingComponents);
+
+ PermutingTupleReference getIndexTuple();
+
+ PermutingTupleReference getFilterTuple();
+
+ MultiComparator getFilterCmp();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ITwoPCIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ITwoPCIndex.java
index df03b91..fc1d5f4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ITwoPCIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ITwoPCIndex.java
@@ -30,8 +30,8 @@
* This function is used to create a BulkLoader for a transaction that is capable of insertions and deletions
* and the bulk loaded component is hidden from the index
*/
- public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException;
+ public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException;
/**
* This function is used to commit the previous transaction if it was resulted in creating any components
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java
new file mode 100644
index 0000000..09e236e
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.common.impls;
+
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IODeviceHandle;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+
+public abstract class AbstractIoOperation implements ILSMIOOperation {
+
+ protected final ILSMIndexAccessor accessor;
+ protected final FileReference target;
+ protected final ILSMIOOperationCallback callback;
+ protected final String indexIdentifier;
+
+ public AbstractIoOperation(ILSMIndexAccessor accessor, FileReference target, ILSMIOOperationCallback callback,
+ String indexIdentifier) {
+ this.accessor = accessor;
+ this.target = target;
+ this.callback = callback;
+ this.indexIdentifier = indexIdentifier;
+ }
+
+ @Override
+ public IODeviceHandle getDevice() {
+ return target.getDeviceHandle();
+ }
+
+ @Override
+ public ILSMIOOperationCallback getCallback() {
+ return callback;
+ }
+
+ public FileReference getTarget() {
+ return target;
+ }
+
+ public ILSMIndexAccessor getAccessor() {
+ return accessor;
+ }
+
+ @Override
+ public String getIndexIdentifier() {
+ return indexIdentifier;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index df42972..7c34326 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -28,20 +28,29 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationExecutionType;
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
+import org.apache.hyracks.storage.am.common.impls.AbstractSearchPredicate;
+import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.ComponentState;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
@@ -49,6 +58,9 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.apache.hyracks.storage.common.IIndexBulkLoader;
+import org.apache.hyracks.storage.common.IModificationOperationCallback;
+import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
@@ -71,11 +83,13 @@
protected final List<ILSMDiskComponent> diskComponents;
protected final List<ILSMDiskComponent> inactiveDiskComponents;
protected final double bloomFilterFalsePositiveRate;
+ protected final ILSMComponentFilterFactory filterFactory;
protected final ILSMComponentFilterFrameFactory filterFrameFactory;
protected final LSMComponentFilterManager filterManager;
+ protected final int[] treeFields;
protected final int[] filterFields;
protected final boolean durable;
- protected boolean isActivated;
+ protected boolean isActive;
protected final AtomicBoolean[] flushRequests;
protected boolean memoryComponentsAllocated = false;
@@ -84,7 +98,7 @@
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
- int[] filterFields, boolean durable) {
+ int[] filterFields, boolean durable, ILSMComponentFilterFactory filterFactory, int[] treeFields) {
this.ioManager = ioManager;
this.virtualBufferCaches = virtualBufferCaches;
this.diskBufferCache = diskBufferCache;
@@ -94,13 +108,15 @@
this.ioScheduler = ioScheduler;
this.ioOpCallback = ioOpCallback;
this.ioOpCallback.setNumOfMutableComponents(virtualBufferCaches.size());
+ this.filterFactory = filterFactory;
this.filterFrameFactory = filterFrameFactory;
this.filterManager = filterManager;
+ this.treeFields = treeFields;
this.filterFields = filterFields;
this.inactiveDiskComponents = new LinkedList<>();
this.durable = durable;
lsmHarness = new LSMHarness(this, mergePolicy, opTracker, diskBufferCache.isReplicationEnabled());
- isActivated = false;
+ isActive = false;
diskComponents = new ArrayList<>();
memoryComponents = new ArrayList<>();
currentMutableComponentId = new AtomicInteger();
@@ -124,7 +140,7 @@
this.ioOpCallback = ioOpCallback;
this.durable = durable;
lsmHarness = new ExternalIndexHarness(this, mergePolicy, opTracker, diskBufferCache.isReplicationEnabled());
- isActivated = false;
+ isActive = false;
diskComponents = new LinkedList<>();
this.inactiveDiskComponents = new LinkedList<>();
// Memory related objects are nulled
@@ -132,11 +148,251 @@
memoryComponents = null;
currentMutableComponentId = null;
flushRequests = null;
+ filterFactory = null;
filterFrameFactory = null;
filterManager = null;
+ treeFields = null;
filterFields = null;
}
+ @Override
+ public synchronized void create() throws HyracksDataException {
+ if (isActive) {
+ throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_ACTIVE_INDEX);
+ }
+ fileManager.deleteDirs();
+ fileManager.createDirs();
+ diskComponents.clear();
+ }
+
+ @Override
+ public synchronized void activate() throws HyracksDataException {
+ if (isActive) {
+ throw HyracksDataException.create(ErrorCode.CANNOT_ACTIVATE_ACTIVE_INDEX);
+ }
+ loadDiskComponents();
+ isActive = true;
+ }
+
+ protected void loadDiskComponents() throws HyracksDataException {
+ diskComponents.clear();
+ List<LSMComponentFileReferences> validFileReferences = fileManager.cleanupAndGetValidFiles();
+ for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
+ ILSMDiskComponent component = loadComponent(lsmComonentFileReference);
+ diskComponents.add(component);
+ }
+ }
+
+ @Override
+ public final synchronized void deactivate() throws HyracksDataException {
+ deactivate(true);
+ }
+
+ @Override
+ public synchronized void deactivate(boolean flush) throws HyracksDataException {
+ if (!isActive) {
+ throw HyracksDataException.create(ErrorCode.CANNOT_DEACTIVATE_INACTIVE_INDEX);
+ }
+ if (flush) {
+ flushMemoryComponents();
+ }
+ deactivateDiskComponents();
+ deactivateMemoryComponents();
+ isActive = false;
+ }
+
+ // What if more than one memory component needs flushing??
+ protected void flushMemoryComponents() throws HyracksDataException {
+ BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
+ ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ accessor.scheduleFlush(cb);
+ try {
+ cb.waitForIO();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ protected void deactivateDiskComponents() throws HyracksDataException {
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
+ for (ILSMDiskComponent c : immutableComponents) {
+ deactivateDiskComponent(c);
+ }
+ }
+
+ protected void deactivateMemoryComponents() throws HyracksDataException {
+ if (memoryComponentsAllocated) {
+ for (ILSMMemoryComponent c : memoryComponents) {
+ deactivateMemoryComponent(c);
+ }
+ memoryComponentsAllocated = false;
+ }
+ }
+
+ @Override
+ public void destroy() throws HyracksDataException {
+ if (isActive) {
+ throw HyracksDataException.create(ErrorCode.CANNOT_DESTROY_ACTIVE_INDEX);
+ }
+ destroyDiskComponents();
+ destroyMemoryComponents();
+ fileManager.deleteDirs();
+ }
+
+ protected void destroyMemoryComponents() throws HyracksDataException {
+ for (ILSMMemoryComponent c : memoryComponents) {
+ destroyMemoryComponent(c);
+ }
+ }
+
+ protected void destroyDiskComponents() throws HyracksDataException {
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
+ for (ILSMDiskComponent c : immutableComponents) {
+ destroyDiskComponent(c);
+ }
+ }
+
+ @Override
+ public void clear() throws HyracksDataException {
+ if (!isActive) {
+ throw HyracksDataException.create(ErrorCode.CANNOT_CLEAR_INACTIVE_INDEX);
+ }
+ clearMemoryComponents();
+ clearDiskComponents();
+ }
+
+ private void clearDiskComponents() throws HyracksDataException {
+ for (ILSMDiskComponent c : diskComponents) {
+ clearDiskComponent(c);
+ }
+ diskComponents.clear();
+ }
+
+ protected void clearMemoryComponents() throws HyracksDataException {
+ if (memoryComponentsAllocated) {
+ for (ILSMMemoryComponent c : memoryComponents) {
+ clearMemoryComponent(c);
+ }
+ }
+ }
+
+ @Override
+ public void getOperationalComponents(ILSMIndexOperationContext ctx) throws HyracksDataException {
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
+ List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
+ int cmc = currentMutableComponentId.get();
+ ctx.setCurrentMutableComponentId(cmc);
+ operationalComponents.clear();
+ switch (ctx.getOperation()) {
+ case UPDATE:
+ case PHYSICALDELETE:
+ case FLUSH:
+ case DELETE:
+ case UPSERT:
+ operationalComponents.add(memoryComponents.get(cmc));
+ break;
+ case INSERT:
+ addOperationalMutableComponents(operationalComponents);
+ operationalComponents.addAll(immutableComponents);
+ break;
+ case SEARCH:
+ if (memoryComponentsAllocated) {
+ addOperationalMutableComponents(operationalComponents);
+ }
+ if (filterManager != null) {
+ for (ILSMComponent c : immutableComponents) {
+ if (c.getLSMComponentFilter().satisfy(
+ ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMinFilterTuple(),
+ ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMaxFilterTuple(),
+ ctx.getFilterCmp())) {
+ operationalComponents.add(c);
+ }
+ }
+ } else {
+ operationalComponents.addAll(immutableComponents);
+ }
+
+ break;
+ case MERGE:
+ operationalComponents.addAll(ctx.getComponentsToBeMerged());
+ break;
+ case FULL_MERGE:
+ operationalComponents.addAll(immutableComponents);
+ break;
+ case REPLICATE:
+ operationalComponents.addAll(ctx.getComponentsToBeReplicated());
+ break;
+ default:
+ throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
+ }
+ }
+
+ @Override
+ public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ throws HyracksDataException {
+ ILSMMemoryComponent flushingComponent = (ILSMMemoryComponent) ctx.getComponentHolder().get(0);
+ LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
+ AbstractLSMIndexOperationContext opCtx =
+ createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ opCtx.setOperation(IndexOperation.FLUSH);
+ opCtx.getComponentHolder().add(flushingComponent);
+ ILSMIOOperation flushOp = createFlushOperation(opCtx, flushingComponent, componentFileRefs, callback);
+ ioScheduler.scheduleOperation(flushOp);
+ }
+
+ @Override
+ public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ throws HyracksDataException {
+ // merge must create a different op ctx
+ AbstractLSMIndexOperationContext opCtx =
+ createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ opCtx.setOperation(IndexOperation.MERGE);
+ List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
+ ILSMDiskComponent firstComponent = (ILSMDiskComponent) mergingComponents.get(0);
+ ILSMDiskComponent lastComponent = (ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1);
+ LSMComponentFileReferences mergeFileRefs = getMergeFileReferences(firstComponent, lastComponent);
+ ILSMIOOperation mergeOp = createMergeOperation(opCtx, mergingComponents, mergeFileRefs, callback);
+ ioScheduler.scheduleOperation(mergeOp);
+ }
+
+ private void addOperationalMutableComponents(List<ILSMComponent> operationalComponents) {
+ int cmc = currentMutableComponentId.get();
+ int numMutableComponents = memoryComponents.size();
+ for (int i = 0; i < numMutableComponents - 1; i++) {
+ ILSMMemoryComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
+ if (c.isReadable()) {
+ // Make sure newest components are added first
+ operationalComponents.add(0, c);
+ }
+ }
+ // The current mutable component is always added
+ operationalComponents.add(0, memoryComponents.get(cmc));
+ }
+
+ @Override
+ public final IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
+ boolean checkIfEmptyIndex) throws HyracksDataException {
+ if (checkIfEmptyIndex && !isEmptyIndex()) {
+ throw HyracksDataException.create(ErrorCode.LOAD_NON_EMPTY_INDEX);
+ }
+ return createBulkLoader(fillLevel, verifyInput, numElementsHint);
+ }
+
+ @Override
+ public final synchronized void allocateMemoryComponents() throws HyracksDataException {
+ if (!isActive) {
+ throw HyracksDataException.create(ErrorCode.CANNOT_ALLOCATE_MEMORY_FOR_INACTIVE_INDEX);
+ }
+ if (memoryComponentsAllocated) {
+ return;
+ }
+ for (ILSMMemoryComponent c : memoryComponents) {
+ allocateMemoryComponent(c);
+ }
+ memoryComponentsAllocated = true;
+ }
+
protected void markAsValidInternal(ITreeIndex treeIndex) throws HyracksDataException {
int fileId = treeIndex.getFileId();
IBufferCache bufferCache = treeIndex.getBufferCache();
@@ -260,8 +516,6 @@
inactiveDiskComponents.add(diskComponent);
}
- public abstract Set<String> getLSMComponentPhysicalFiles(ILSMComponent newComponent);
-
@Override
public void scheduleReplication(ILSMIndexOperationContext ctx, List<ILSMDiskComponent> lsmComponents,
boolean bulkload, ReplicationOperation operation, LSMOperationType opType) throws HyracksDataException {
@@ -286,14 +540,11 @@
try {
diskBufferCache.getIOReplicationManager().submitJob(job);
} catch (IOException e) {
- throw new HyracksDataException(e);
+ throw HyracksDataException.create(e);
}
}
@Override
- public abstract void allocateMemoryComponents() throws HyracksDataException;
-
- @Override
public boolean isMemoryComponentsAllocated() {
return memoryComponentsAllocated;
}
@@ -318,8 +569,105 @@
return memoryComponents;
}
+ protected IBinaryComparatorFactory[] getFilterCmpFactories() {
+ return filterFactory == null ? null : filterFactory.getFilterCmpFactories();
+ }
+
@Override
public int getNumOfFilterFields() {
return filterFields == null ? 0 : filterFields.length;
}
+
+ public double bloomFilterFalsePositiveRate() {
+ return bloomFilterFalsePositiveRate;
+ }
+
+ @Override
+ public void updateFilter(ILSMIndexOperationContext ctx, ITupleReference tuple) throws HyracksDataException {
+ if (ctx.getFilterTuple() != null) {
+ ctx.getFilterTuple().reset(tuple);
+ memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter().update(ctx.getFilterTuple(),
+ ctx.getFilterCmp());
+ }
+ }
+
+ public int[] getFilterFields() {
+ return filterFields;
+ }
+
+ public int[] getTreeFields() {
+ return treeFields;
+ }
+
+ public LSMComponentFilterManager getFilterManager() {
+ return filterManager;
+ }
+
+ public ILSMHarness getLsmHarness() {
+ return lsmHarness;
+ }
+
+ @Override
+ public final void validate() throws HyracksDataException {
+ if (memoryComponentsAllocated) {
+ for (ILSMMemoryComponent c : memoryComponents) {
+ validateMemoryComponent(c);
+ }
+ }
+ for (ILSMDiskComponent c : diskComponents) {
+ validateDiskComponent(c);
+ }
+ }
+
+ @Override
+ public long getMemoryAllocationSize() {
+ long size = 0;
+ for (ILSMMemoryComponent c : memoryComponents) {
+ size += getMemoryComponentSize(c);
+ }
+ return size;
+ }
+
+ public abstract Set<String> getLSMComponentPhysicalFiles(ILSMComponent newComponent);
+
+ protected abstract void allocateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException;
+
+ protected abstract IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException;
+
+ protected abstract ILSMDiskComponent loadComponent(LSMComponentFileReferences refs) throws HyracksDataException;
+
+ protected abstract void deactivateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException;
+
+ protected abstract void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException;
+
+ protected abstract void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException;
+
+ protected abstract void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException;
+
+ protected abstract void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException;
+
+ protected abstract void clearMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException;
+
+ protected abstract void validateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException;
+
+ protected abstract void validateDiskComponent(ILSMDiskComponent c) throws HyracksDataException;
+
+ protected abstract long getMemoryComponentSize(ILSMMemoryComponent c);
+
+ protected abstract LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent,
+ ILSMDiskComponent lastComponent) throws HyracksDataException;
+
+ protected abstract AbstractLSMIndexOperationContext createOpContext(
+ IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback)
+ throws HyracksDataException;
+
+ protected abstract ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx,
+ ILSMMemoryComponent flushingComponent, LSMComponentFileReferences componentFileRefs,
+ ILSMIOOperationCallback callback) throws HyracksDataException;
+
+ protected abstract ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx,
+ List<ILSMComponent> mergingComponents, LSMComponentFileReferences mergeFileRefs,
+ ILSMIOOperationCallback callback) throws HyracksDataException;
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexOperationContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexOperationContext.java
index 3b907c3..19d09af 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexOperationContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexOperationContext.java
@@ -18,11 +18,53 @@
*/
package org.apache.hyracks.storage.am.lsm.common.impls;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.common.IModificationOperationCallback;
+import org.apache.hyracks.storage.common.ISearchOperationCallback;
+import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.MultiComparator;
public abstract class AbstractLSMIndexOperationContext implements ILSMIndexOperationContext {
- private boolean accessingComponents = false;
+ protected final PermutingTupleReference indexTuple;
+ protected final MultiComparator filterCmp;
+ protected final PermutingTupleReference filterTuple;
+ protected final List<ILSMComponent> componentHolder;
+ protected final List<ILSMDiskComponent> componentsToBeMerged;
+ protected final List<ILSMDiskComponent> componentsToBeReplicated;
+ protected final ISearchOperationCallback searchCallback;
+ protected final IModificationOperationCallback modificationCallback;
+ protected IndexOperation op;
+ protected boolean accessingComponents = false;
+ protected ISearchPredicate searchPredicate;
+
+ public AbstractLSMIndexOperationContext(int[] treeFields, int[] filterFields,
+ IBinaryComparatorFactory[] filterCmpFactories, ISearchOperationCallback searchCallback,
+ IModificationOperationCallback modificationCallback) {
+ this.searchCallback = searchCallback;
+ this.modificationCallback = modificationCallback;
+ this.componentHolder = new LinkedList<>();
+ this.componentsToBeMerged = new LinkedList<>();
+ this.componentsToBeReplicated = new LinkedList<>();
+ if (filterFields != null) {
+ indexTuple = new PermutingTupleReference(treeFields);
+ filterCmp = MultiComparator.create(filterCmpFactories);
+ filterTuple = new PermutingTupleReference(filterFields);
+ } else {
+ indexTuple = null;
+ filterCmp = null;
+ filterTuple = null;
+ }
+ }
@Override
public boolean isAccessingComponents() {
@@ -35,7 +77,71 @@
}
@Override
+ public final PermutingTupleReference getIndexTuple() {
+ return indexTuple;
+ }
+
+ @Override
+ public final PermutingTupleReference getFilterTuple() {
+ return filterTuple;
+ }
+
+ @Override
+ public final MultiComparator getFilterCmp() {
+ return filterCmp;
+ }
+
+ @Override
public void reset() {
accessingComponents = false;
+ componentHolder.clear();
+ componentsToBeMerged.clear();
+ componentsToBeReplicated.clear();
+ }
+
+ @Override
+ public IndexOperation getOperation() {
+ return op;
+ }
+
+ @Override
+ public List<ILSMComponent> getComponentHolder() {
+ return componentHolder;
+ }
+
+ @Override
+ public ISearchOperationCallback getSearchOperationCallback() {
+ return searchCallback;
+ }
+
+ @Override
+ public IModificationOperationCallback getModificationCallback() {
+ return modificationCallback;
+ }
+
+ @Override
+ public List<ILSMDiskComponent> getComponentsToBeMerged() {
+ return componentsToBeMerged;
+ }
+
+ @Override
+ public List<ILSMDiskComponent> getComponentsToBeReplicated() {
+ return componentsToBeReplicated;
+ }
+
+ @Override
+ public void setOperation(IndexOperation newOp) throws HyracksDataException {
+ reset();
+ op = newOp;
+ }
+
+ @Override
+ public void setSearchPredicate(ISearchPredicate searchPredicate) {
+ this.searchPredicate = searchPredicate;
+ }
+
+ @Override
+ public ISearchPredicate getSearchPredicate() {
+ return searchPredicate;
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java
index 2899849..50e240e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java
@@ -59,7 +59,7 @@
super.afterExecute(r, t);
LSMIOOperationTask<Boolean> task = (LSMIOOperationTask<Boolean>) r;
ILSMIOOperation executedOp = task.getOperation();
- String id = executedOp.getIndexUniqueIdentifier();
+ String id = executedOp.getIndexIdentifier();
synchronized (this) {
runningFlushOperations.remove(id);
if (waitingFlushOperations.containsKey(id)) {
@@ -84,7 +84,7 @@
if (operation.getIOOpertionType() == LSMIOOpertionType.MERGE) {
executor.submit(operation);
} else {
- String id = operation.getIndexUniqueIdentifier();
+ String id = operation.getIndexIdentifier();
synchronized (executor) {
if (runningFlushOperations.containsKey(id)) {
if (waitingFlushOperations.containsKey(id)) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/DiskComponentMetadata.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/DiskComponentMetadata.java
index 2da954d..b7d2ea3 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/DiskComponentMetadata.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/DiskComponentMetadata.java
@@ -53,4 +53,8 @@
public void put(MemoryComponentMetadata metadata) throws HyracksDataException {
metadata.copy(mdpManager);
}
+
+ public IMetadataPageManager getMetadataPageManager() {
+ return mdpManager;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java
new file mode 100644
index 0000000..cafb008
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.common.impls;
+
+import java.util.Objects;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
+
+public class FlushOperation extends AbstractIoOperation implements Comparable<FlushOperation> {
+
+ protected final ILSMMemoryComponent flushingComponent;
+
+ public FlushOperation(ILSMIndexAccessor accessor, ILSMMemoryComponent flushingComponent, FileReference target,
+ ILSMIOOperationCallback callback, String indexIdentifier) {
+ super(accessor, target, callback, indexIdentifier);
+ this.flushingComponent = flushingComponent;
+ }
+
+ @Override
+ public Boolean call() throws HyracksDataException {
+ accessor.flush(this);
+ return true;
+ }
+
+ @Override
+ public ILSMIOOperationCallback getCallback() {
+ return callback;
+ }
+
+ @Override
+ public FileReference getTarget() {
+ return target;
+ }
+
+ @Override
+ public ILSMIndexAccessor getAccessor() {
+ return accessor;
+ }
+
+ public ILSMComponent getFlushingComponent() {
+ return flushingComponent;
+ }
+
+ @Override
+ public String getIndexIdentifier() {
+ return indexIdentifier;
+ }
+
+ @Override
+ public LSMIOOpertionType getIOOpertionType() {
+ return LSMIOOpertionType.FLUSH;
+ }
+
+ @Override
+ public int compareTo(FlushOperation o) {
+ return target.getFile().getName().compareTo(o.getTarget().getFile().getName());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof FlushOperation)) {
+ return false;
+ }
+ return Objects.equals(target.getFile().getName(), ((FlushOperation) o).target.getFile().getName());
+ }
+
+ @Override
+ public int hashCode() {
+ return target.getFile().getName().hashCode();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterFactory.java
index 46a629a..390321f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterFactory.java
@@ -38,4 +38,9 @@
public ILSMComponentFilter createFilter() {
return new LSMComponentFilter(tupleWriterFactory.createTupleWriter(), filterCmpFactories);
}
+
+ @Override
+ public IBinaryComparatorFactory[] getFilterCmpFactories() {
+ return filterCmpFactories;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 94fd2bb..e315858 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -577,4 +577,35 @@
throw new HyracksDataException("Opeartion already has access to components of index " + lsmIndex);
}
}
+
+ @Override
+ public void updateFilter(ILSMIndexOperationContext ctx, ITupleReference tuple) throws HyracksDataException {
+ if (!lsmIndex.isMemoryComponentsAllocated()) {
+ lsmIndex.allocateMemoryComponents();
+ }
+ lsmIndex.updateFilter(ctx, tuple);
+ }
+
+ @Override
+ public void enter(ILSMIndexOperationContext ctx) throws HyracksDataException {
+ if (!lsmIndex.isMemoryComponentsAllocated()) {
+ lsmIndex.allocateMemoryComponents();
+ }
+ getAndEnterComponents(ctx, LSMOperationType.MODIFICATION, false);
+ }
+
+ @Override
+ public void exit(ILSMIndexOperationContext ctx) throws HyracksDataException {
+ getAndExitComponentsAndComplete(ctx, LSMOperationType.MODIFICATION);
+ }
+
+ private void getAndExitComponentsAndComplete(ILSMIndexOperationContext ctx, LSMOperationType op)
+ throws HyracksDataException {
+ validateOperationEnterComponentsState(ctx);
+ synchronized (opTracker) {
+ lsmIndex.getOperationalComponents(ctx);
+ ctx.setAccessingComponents(true);
+ exitAndComplete(ctx, op);
+ }
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexReplicationJob.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexReplicationJob.java
index 18d25a8..a35d00e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexReplicationJob.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexReplicationJob.java
@@ -45,7 +45,7 @@
@Override
public void endReplication() throws HyracksDataException {
if (operationContext != null) {
- ((AbstractLSMIndex) (lsmIndex)).lsmHarness.endReplication(operationContext);
+ ((AbstractLSMIndex) (lsmIndex)).getLsmHarness().endReplication(operationContext);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
index d645876..f99a859 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
@@ -34,7 +34,6 @@
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
public abstract class LSMIndexSearchCursor implements ITreeIndexCursor {
protected final ILSMIndexOperationContext opCtx;
@@ -133,12 +132,6 @@
}
@Override
- public ICachedPage getPage() {
- // do nothing
- return null;
- }
-
- @Override
public void close() throws HyracksDataException {
try {
if (outputPriorityQueue != null) {
@@ -244,7 +237,7 @@
}
@Override
- public boolean exclusiveLatchNodes() {
+ public boolean isExclusiveLatchNodes() {
return false;
}
@@ -313,8 +306,4 @@
return cmp.compare(tupleA, tupleB);
}
- @Override
- public void markCurrentTupleAsUpdated() throws HyracksDataException {
- throw new HyracksDataException("Updating tuples is not supported with this cursor.");
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
index f117e63..a41d2c0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
@@ -35,13 +36,24 @@
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.ISearchPredicate;
-public abstract class LSMTreeIndexAccessor implements ILSMIndexAccessor {
- protected ILSMHarness lsmHarness;
- protected ILSMIndexOperationContext ctx;
+public class LSMTreeIndexAccessor implements ILSMIndexAccessor {
+ @FunctionalInterface
+ public interface ICursorFactory {
+ ITreeIndexCursor create(ILSMIndexOperationContext ctx);
+ }
- public LSMTreeIndexAccessor(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) {
+ protected final ILSMHarness lsmHarness;
+ protected final ILSMIndexOperationContext ctx;
+ protected final ICursorFactory cursorFactory;
+
+ public LSMTreeIndexAccessor(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx, ICursorFactory cursorFactory) {
this.lsmHarness = lsmHarness;
this.ctx = ctx;
+ this.cursorFactory = cursorFactory;
+ }
+
+ public ILSMIndexOperationContext getCtx() {
+ return ctx;
}
@Override
@@ -173,15 +185,35 @@
@Override
public void updateMeta(IValueReference key, IValueReference value) throws HyracksDataException {
- // a hack because delete only gets the memory component
- ctx.setOperation(IndexOperation.DELETE);
+ ctx.setOperation(IndexOperation.UPSERT);
lsmHarness.updateMeta(ctx, key, value);
}
@Override
public void forceUpdateMeta(IValueReference key, IValueReference value) throws HyracksDataException {
- // a hack because delete only gets the memory component
- ctx.setOperation(IndexOperation.DELETE);
+ ctx.setOperation(IndexOperation.UPSERT);
lsmHarness.forceUpdateMeta(ctx, key, value);
}
+
+ @Override
+ public ITreeIndexCursor createSearchCursor(boolean exclusive) {
+ return cursorFactory.create(ctx);
+ }
+
+ @Override
+ public void enter() throws HyracksDataException {
+ ctx.setOperation(IndexOperation.UPSERT);
+ lsmHarness.enter(ctx);
+ }
+
+ @Override
+ public void exit() throws HyracksDataException {
+ ctx.setOperation(IndexOperation.UPSERT);
+ lsmHarness.exit(ctx);
+ }
+
+ public void updateFilter(ITupleReference tuple) throws HyracksDataException {
+ ctx.setOperation(IndexOperation.UPSERT);
+ lsmHarness.updateFilter(ctx, tuple);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MergeOperation.java
new file mode 100644
index 0000000..3540b84
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MergeOperation.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.common.impls;
+
+import java.util.List;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.common.IIndexCursor;
+
+public class MergeOperation extends AbstractIoOperation {
+
+ protected final List<ILSMComponent> mergingComponents;
+ protected final IIndexCursor cursor;
+
+ public MergeOperation(ILSMIndexAccessor accessor, FileReference target, ILSMIOOperationCallback callback,
+ String indexIdentifier, List<ILSMComponent> mergingComponents, IIndexCursor cursor) {
+ super(accessor, target, callback, indexIdentifier);
+ this.mergingComponents = mergingComponents;
+ this.cursor = cursor;
+ }
+
+ public List<ILSMComponent> getMergingComponents() {
+ return mergingComponents;
+ }
+
+ @Override
+ public Boolean call() throws HyracksDataException {
+ accessor.merge(this);
+ return true;
+ }
+
+ @Override
+ public LSMIOOpertionType getIOOpertionType() {
+ return LSMIOOpertionType.MERGE;
+ }
+
+ public IIndexCursor getCursor() {
+ return cursor;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 3d7b716..89d8f0b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -42,9 +42,7 @@
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
import org.apache.hyracks.storage.am.btree.util.BTreeUtils;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
-import org.apache.hyracks.storage.am.common.impls.AbstractSearchPredicate;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
-import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
@@ -58,14 +56,15 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory;
-import org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor;
@@ -103,7 +102,6 @@
protected final IBinaryComparatorFactory[] invListCmpFactories;
protected final ITypeTraits[] tokenTypeTraits;
protected final IBinaryComparatorFactory[] tokenCmpFactories;
- private final int[] invertedIndexFields;
private final int[] filterFieldsForNonBulkLoadOps;
private final int[] invertedIndexFieldsForNonBulkLoadOps;
@@ -120,14 +118,13 @@
boolean durable) throws HyracksDataException {
super(ioManager, virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
- filterManager, filterFields, durable);
+ filterManager, filterFields, durable, filterFactory, invertedIndexFields);
this.tokenizerFactory = tokenizerFactory;
this.invListTypeTraits = invListTypeTraits;
this.invListCmpFactories = invListCmpFactories;
this.tokenTypeTraits = tokenTypeTraits;
this.tokenCmpFactories = tokenCmpFactories;
- this.invertedIndexFields = invertedIndexFields;
this.filterFieldsForNonBulkLoadOps = filterFieldsForNonBulkLoadOps;
this.invertedIndexFieldsForNonBulkLoadOps = invertedIndexFieldsForNonBulkLoadOps;
@@ -151,151 +148,53 @@
}
@Override
- public synchronized void create() throws HyracksDataException {
- if (isActivated) {
- throw new HyracksDataException("Failed to create the index since it is activated.");
- }
-
- fileManager.deleteDirs();
- fileManager.createDirs();
- diskComponents.clear();
+ protected ILSMDiskComponent loadComponent(LSMComponentFileReferences refs) throws HyracksDataException {
+ return createDiskInvIndexComponent(componentFactory, refs.getInsertIndexFileReference(),
+ refs.getDeleteIndexFileReference(), refs.getBloomFilterFileReference(), false);
}
@Override
- public synchronized void activate() throws HyracksDataException {
- if (isActivated) {
- throw new HyracksDataException("Failed to activate the index since it is already activated.");
- }
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- immutableComponents.clear();
- List<LSMComponentFileReferences> validFileReferences = fileManager.cleanupAndGetValidFiles();
- for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
- LSMInvertedIndexDiskComponent component;
- component = createDiskInvIndexComponent(componentFactory,
- lsmComonentFileReference.getInsertIndexFileReference(),
- lsmComonentFileReference.getDeleteIndexFileReference(),
- lsmComonentFileReference.getBloomFilterFileReference(), false);
- immutableComponents.add(component);
- }
- isActivated = true;
+ protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
+ component.getBloomFilter().deactivate();
+ component.getInvIndex().deactivate();
+ component.getDeletedKeysBTree().deactivate();
+ component.getBloomFilter().destroy();
+ component.getInvIndex().destroy();
+ component.getDeletedKeysBTree().destroy();
}
@Override
- public void clear() throws HyracksDataException {
- if (!isActivated) {
- throw new HyracksDataException("Failed to clear the index since it is not activated.");
- }
-
- clearMemoryComponents();
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
- component.getBloomFilter().deactivate();
- component.getInvIndex().deactivate();
- component.getDeletedKeysBTree().deactivate();
- component.getBloomFilter().destroy();
- component.getInvIndex().destroy();
- component.getDeletedKeysBTree().destroy();
- }
- immutableComponents.clear();
+ protected void deactivateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+ mutableComponent.getInvIndex().deactivate();
+ mutableComponent.getDeletedKeysBTree().deactivate();
+ mutableComponent.getInvIndex().destroy();
+ mutableComponent.getDeletedKeysBTree().destroy();
+ ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).close();
}
@Override
- public synchronized void deactivate(boolean flushOnExit) throws HyracksDataException {
- if (!isActivated) {
- throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
- }
- if (flushOnExit) {
- BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
- ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- accessor.scheduleFlush(cb);
- try {
- cb.waitForIO();
- } catch (InterruptedException e) {
- throw new HyracksDataException(e);
- }
- }
-
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
- component.getBloomFilter().deactivate();
- component.getInvIndex().deactivate();
- component.getDeletedKeysBTree().deactivate();
- }
- deallocateMemoryComponents();
- isActivated = false;
+ protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
+ component.getBloomFilter().deactivate();
+ component.getInvIndex().deactivate();
+ component.getDeletedKeysBTree().deactivate();
}
@Override
- public synchronized void deactivate() throws HyracksDataException {
- deactivate(true);
+ protected void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
+ component.getInvIndex().destroy();
+ component.getDeletedKeysBTree().destroy();
+ component.getBloomFilter().destroy();
}
@Override
- public synchronized void destroy() throws HyracksDataException {
- if (isActivated) {
- throw new HyracksDataException("Failed to destroy the index since it is activated.");
- }
-
- for (ILSMComponent c : memoryComponents) {
- LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
- mutableComponent.getInvIndex().destroy();
- mutableComponent.getDeletedKeysBTree().destroy();
- }
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
- component.getInvIndex().destroy();
- component.getDeletedKeysBTree().destroy();
- component.getBloomFilter().destroy();
- }
- fileManager.deleteDirs();
- }
-
- @Override
- public void getOperationalComponents(ILSMIndexOperationContext ctx) throws HyracksDataException {
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
- int cmc = currentMutableComponentId.get();
- ctx.setCurrentMutableComponentId(cmc);
- operationalComponents.clear();
- switch (ctx.getOperation()) {
- case FLUSH:
- case DELETE:
- case INSERT:
- case UPSERT:
- operationalComponents.add(memoryComponents.get(cmc));
- break;
- case SEARCH:
- if (memoryComponentsAllocated) {
- addOperationalMutableComponents(operationalComponents);
- }
- if (filterManager != null) {
- for (ILSMComponent c : immutableComponents) {
- if (c.getLSMComponentFilter().satisfy(
- ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMinFilterTuple(),
- ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMaxFilterTuple(),
- ((LSMInvertedIndexOpContext) ctx).getFilterCmp())) {
- operationalComponents.add(c);
- }
- }
- } else {
- operationalComponents.addAll(immutableComponents);
- }
- break;
- case MERGE:
- operationalComponents.addAll(ctx.getComponentsToBeMerged());
- break;
- case FULL_MERGE:
- operationalComponents.addAll(immutableComponents);
- break;
- case REPLICATE:
- operationalComponents.addAll(ctx.getComponentsToBeReplicated());
- break;
- default:
- throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
- }
+ protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+ mutableComponent.getInvIndex().destroy();
+ mutableComponent.getDeletedKeysBTree().destroy();
}
/**
@@ -333,12 +232,11 @@
ctx.getModificationCallback().before(indexTuple);
ctx.getModificationCallback().found(null, indexTuple);
switch (ctx.getOperation()) {
- case INSERT: {
+ case INSERT:
// Insert into the in-memory inverted index.
ctx.getCurrentMutableInvIndexAccessors().insert(indexTuple);
break;
- }
- case DELETE: {
+ case DELETE:
// First remove all entries in the in-memory inverted index (if any).
ctx.getCurrentMutableInvIndexAccessors().delete(indexTuple);
// Insert key into the deleted-keys BTree.
@@ -353,16 +251,10 @@
}
}
break;
- }
- default: {
+ default:
throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
- }
}
- if (ctx.getFilterTuple() != null) {
- ctx.getFilterTuple().reset(tuple);
- memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter().update(ctx.getFilterTuple(),
- ctx.getFilterCmp());
- }
+ updateFilter(ctx, tuple);
}
@Override
@@ -402,7 +294,7 @@
private ICursorInitialState createCursorInitialState(ISearchPredicate pred, IIndexOperationContext ictx,
boolean includeMutableComponent, ArrayList<IIndexAccessor> indexAccessors,
ArrayList<IIndexAccessor> deletedKeysBTreeAccessors, List<ILSMComponent> operationalComponents) {
- ICursorInitialState initState = null;
+ ICursorInitialState initState;
PermutingTupleReference keysOnlyTuple = createKeysOnlyTupleReference();
MultiComparator keyCmp = MultiComparator.create(invListCmpFactories);
@@ -413,7 +305,7 @@
deletedKeysBTreeAccessors,
((LSMInvertedIndexMemoryComponent) memoryComponents.get(currentMutableComponentId.get()))
.getDeletedKeysBTree().getLeafFrameFactory(),
- ictx, includeMutableComponent, lsmHarness, operationalComponents);
+ ictx, includeMutableComponent, getLsmHarness(), operationalComponents);
} else {
LSMInvertedIndexMemoryComponent mutableComponent =
(LSMInvertedIndexMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
@@ -422,7 +314,7 @@
initState = new LSMInvertedIndexRangeSearchCursorInitialState(tokensAndKeysCmp, keyCmp, keysOnlyTuple,
((LSMInvertedIndexMemoryComponent) memoryComponents.get(currentMutableComponentId.get()))
.getDeletedKeysBTree().getLeafFrameFactory(),
- includeMutableComponent, lsmHarness, indexAccessors, deletedKeysBTreeAccessors, pred,
+ includeMutableComponent, getLsmHarness(), indexAccessors, deletedKeysBTreeAccessors, pred,
operationalComponents);
}
return initState;
@@ -443,28 +335,12 @@
}
@Override
- public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
- throws HyracksDataException {
- ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
- LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- LSMInvertedIndexOpContext opCtx =
- createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- opCtx.setOperation(IndexOperation.FLUSH);
- opCtx.getComponentHolder().add(flushingComponent);
- ioScheduler.scheduleOperation(new LSMInvertedIndexFlushOperation(
- new LSMInvertedIndexAccessor(lsmHarness, opCtx), flushingComponent,
- componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(),
- componentFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir()));
- }
-
- @Override
public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException {
LSMInvertedIndexFlushOperation flushOp = (LSMInvertedIndexFlushOperation) operation;
// Create an inverted index instance to be bulk loaded.
- LSMInvertedIndexDiskComponent component =
- createDiskInvIndexComponent(componentFactory, flushOp.getDictBTreeFlushTarget(),
- flushOp.getDeletedKeysBTreeFlushTarget(), flushOp.getBloomFilterFlushTarget(), true);
+ LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory, flushOp.getTarget(),
+ flushOp.getDeletedKeysBTreeTarget(), flushOp.getBloomFilterTarget(), true);
IInvertedIndex diskInvertedIndex = component.getInvIndex();
// Create a scan cursor on the BTree underlying the in-memory inverted index.
@@ -491,8 +367,8 @@
List<ITupleReference> filterTuples = new ArrayList<>();
filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
- filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilter(component.getLSMComponentFilter(),
+ getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
+ getFilterManager().writeFilter(component.getLSMComponentFilter(),
((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
}
flushingComponent.getMetadata().copy(component.getMetadata());
@@ -545,31 +421,6 @@
}
@Override
- public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
- throws HyracksDataException {
- LSMInvertedIndexOpContext ictx =
- createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- ictx.setOperation(IndexOperation.MERGE);
- List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
- IIndexCursor cursor = new LSMInvertedIndexRangeSearchCursor(ictx);
-
- LSMInvertedIndexDiskComponent firstComponent = (LSMInvertedIndexDiskComponent) mergingComponents.get(0);
- OnDiskInvertedIndex firstInvIndex = (OnDiskInvertedIndex) firstComponent.getInvIndex();
- String firstFileName = firstInvIndex.getBTree().getFileReference().getFile().getName();
-
- LSMInvertedIndexDiskComponent lastComponent =
- (LSMInvertedIndexDiskComponent) mergingComponents.get(mergingComponents.size() - 1);
- OnDiskInvertedIndex lastInvIndex = (OnDiskInvertedIndex) lastComponent.getInvIndex();
- String lastFileName = lastInvIndex.getBTree().getFileReference().getFile().getName();
-
- LSMComponentFileReferences relMergeFileRefs = fileManager.getRelMergeFileReference(firstFileName, lastFileName);
- ILSMIndexAccessor accessor = new LSMInvertedIndexAccessor(lsmHarness, ctx);
- ioScheduler.scheduleOperation(new LSMInvertedIndexMergeOperation(accessor, mergingComponents, cursor,
- relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(),
- relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir()));
- }
-
- @Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
LSMInvertedIndexMergeOperation mergeOp = (LSMInvertedIndexMergeOperation) operation;
IIndexCursor cursor = mergeOp.getCursor();
@@ -581,9 +432,8 @@
search(opCtx, cursor, mergePred);
// Create an inverted index instance.
- LSMInvertedIndexDiskComponent component =
- createDiskInvIndexComponent(componentFactory, mergeOp.getDictBTreeMergeTarget(),
- mergeOp.getDeletedKeysBTreeMergeTarget(), mergeOp.getBloomFilterMergeTarget(), true);
+ LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory, mergeOp.getTarget(),
+ mergeOp.getDeletedKeysBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
IInvertedIndex mergedDiskInvertedIndex = component.getInvIndex();
@@ -651,8 +501,8 @@
filterTuples.add(max);
}
}
- filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilter(component.getLSMComponentFilter(),
+ getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
+ getFilterManager().writeFilter(component.getLSMComponentFilter(),
((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
}
invIndexBulkLoader.end();
@@ -661,9 +511,9 @@
}
@Override
- public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- return new LSMInvertedIndexBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex);
+ public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
+ return new LSMInvertedIndexBulkLoader(fillFactor, verifyInput, numElementsHint);
}
public class LSMInvertedIndexBulkLoader implements IIndexBulkLoader {
@@ -676,11 +526,8 @@
public final PermutingTupleReference filterTuple;
public final MultiComparator filterCmp;
- public LSMInvertedIndexBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- if (checkIfEmptyIndex && !isEmptyIndex()) {
- throw HyracksDataException.create(ErrorCode.LOAD_NON_EMPTY_INDEX);
- }
+ public LSMInvertedIndexBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
// Note that by using a flush target file name, we state that the
// new bulk loaded tree is "newer" than any other merged tree.
component = createBulkLoadTarget();
@@ -691,10 +538,10 @@
deletedKeysBTreeBulkLoader = ((LSMInvertedIndexDiskComponent) component).getDeletedKeysBTree()
.createBulkLoader(fillFactor, verifyInput, numElementsHint, false);
- if (filterFields != null) {
- indexTuple = new PermutingTupleReference(invertedIndexFields);
+ if (getFilterFields() != null) {
+ indexTuple = new PermutingTupleReference(getTreeFields());
filterCmp = MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories());
- filterTuple = new PermutingTupleReference(filterFields);
+ filterTuple = new PermutingTupleReference(getFilterFields());
} else {
indexTuple = null;
filterCmp = null;
@@ -745,9 +592,9 @@
public void end() throws HyracksDataException {
if (!cleanedUpArtifacts) {
if (component.getLSMComponentFilter() != null) {
- filterManager.writeFilter(component.getLSMComponentFilter(),
- (((OnDiskInvertedIndex) ((LSMInvertedIndexDiskComponent) component).getInvIndex())
- .getBTree()));
+ getFilterManager().writeFilter(component.getLSMComponentFilter(),
+ ((OnDiskInvertedIndex) ((LSMInvertedIndexDiskComponent) component).getInvIndex())
+ .getBTree());
}
invIndexBulkLoader.end();
deletedKeysBTreeBulkLoader.end();
@@ -756,7 +603,7 @@
cleanupArtifacts();
} else {
ioOpCallback.afterOperation(LSMOperationType.FLUSH, null, component);
- lsmHarness.addBulkLoadedComponent(component);
+ getLsmHarness().addBulkLoadedComponent(component);
}
}
}
@@ -802,7 +649,7 @@
component.getBloomFilter().activate();
// Will be closed during cleanup of merge().
if (component.getLSMComponentFilter() != null && !create) {
- filterManager.readFilter(component.getLSMComponentFilter(),
+ getFilterManager().readFilter(component.getLSMComponentFilter(),
((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
}
return component;
@@ -811,13 +658,14 @@
@Override
public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) throws HyracksDataException {
- return new LSMInvertedIndexAccessor(lsmHarness, createOpContext(modificationCallback, searchCallback));
+ return new LSMInvertedIndexAccessor(getLsmHarness(), createOpContext(modificationCallback, searchCallback));
}
- private LSMInvertedIndexOpContext createOpContext(IModificationOperationCallback modificationCallback,
+ @Override
+ protected LSMInvertedIndexOpContext createOpContext(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) throws HyracksDataException {
return new LSMInvertedIndexOpContext(memoryComponents, modificationCallback, searchCallback,
- invertedIndexFieldsForNonBulkLoadOps, filterFieldsForNonBulkLoadOps);
+ invertedIndexFieldsForNonBulkLoadOps, filterFieldsForNonBulkLoadOps, getFilterCmpFactories());
}
@Override
@@ -842,17 +690,6 @@
}
@Override
- public long getMemoryAllocationSize() {
- long size = 0;
- for (ILSMComponent c : memoryComponents) {
- LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
- IBufferCache virtualBufferCache = mutableComponent.getInvIndex().getBufferCache();
- size += virtualBufferCache.getNumPages() * virtualBufferCache.getPageSize();
- }
- return size;
- }
-
- @Override
public ITypeTraits[] getTokenTypeTraits() {
return tokenTypeTraits;
}
@@ -882,27 +719,11 @@
}
@Override
- public void validate() throws HyracksDataException {
- validateMemoryComponents();
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
- component.getInvIndex().validate();
- component.getDeletedKeysBTree().validate();
- }
- }
-
- @Override
public String toString() {
return "LSMInvertedIndex [" + fileManager.getBaseDir() + "]";
}
@Override
- public boolean hasMemoryComponents() {
- return true;
- }
-
- @Override
public boolean isPrimaryIndex() {
return false;
}
@@ -920,71 +741,74 @@
}
@Override
- public synchronized void allocateMemoryComponents() throws HyracksDataException {
- if (!isActivated) {
- throw new HyracksDataException("Failed to allocate memory components since the index is not active.");
- }
- if (memoryComponentsAllocated) {
- return;
- }
- for (ILSMComponent c : memoryComponents) {
- LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
- ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).open();
- mutableComponent.getInvIndex().create();
- mutableComponent.getInvIndex().activate();
- mutableComponent.getDeletedKeysBTree().create();
- mutableComponent.getDeletedKeysBTree().activate();
- }
- memoryComponentsAllocated = true;
+ protected void clearMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+ mutableComponent.getInvIndex().clear();
+ mutableComponent.getDeletedKeysBTree().clear();
+ mutableComponent.reset();
}
- private void addOperationalMutableComponents(List<ILSMComponent> operationalComponents) {
- int cmc = currentMutableComponentId.get();
- int numMutableComponents = memoryComponents.size();
- for (int i = 0; i < numMutableComponents - 1; i++) {
- ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
- LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
- if (mutableComponent.isReadable()) {
- // Make sure newest components are added first
- operationalComponents.add(0, mutableComponent);
- }
- }
- // The current mutable component is always added
- operationalComponents.add(0, memoryComponents.get(cmc));
+ @Override
+ protected long getMemoryComponentSize(ILSMMemoryComponent c) {
+ LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+ IBufferCache virtualBufferCache = mutableComponent.getInvIndex().getBufferCache();
+ return ((long) virtualBufferCache.getNumPages()) * virtualBufferCache.getPageSize();
}
- private synchronized void clearMemoryComponents() throws HyracksDataException {
- if (memoryComponentsAllocated) {
- for (ILSMComponent c : memoryComponents) {
- LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
- mutableComponent.getInvIndex().clear();
- mutableComponent.getDeletedKeysBTree().clear();
- mutableComponent.reset();
- }
- }
+ @Override
+ protected void validateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+ mutableComponent.getInvIndex().validate();
+ mutableComponent.getDeletedKeysBTree().validate();
}
- private synchronized void validateMemoryComponents() throws HyracksDataException {
- if (memoryComponentsAllocated) {
- for (ILSMComponent c : memoryComponents) {
- LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
- mutableComponent.getInvIndex().validate();
- mutableComponent.getDeletedKeysBTree().validate();
- }
- }
+ @Override
+ protected void validateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
+ component.getInvIndex().validate();
+ component.getDeletedKeysBTree().validate();
}
- private synchronized void deallocateMemoryComponents() throws HyracksDataException {
- if (memoryComponentsAllocated) {
- for (ILSMComponent c : memoryComponents) {
- LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
- mutableComponent.getInvIndex().deactivate();
- mutableComponent.getDeletedKeysBTree().deactivate();
- mutableComponent.getInvIndex().destroy();
- mutableComponent.getDeletedKeysBTree().destroy();
- ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).close();
- }
- memoryComponentsAllocated = false;
- }
+ @Override
+ protected void allocateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+ ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).open();
+ mutableComponent.getInvIndex().create();
+ mutableComponent.getInvIndex().activate();
+ mutableComponent.getDeletedKeysBTree().create();
+ mutableComponent.getDeletedKeysBTree().activate();
+ }
+
+ @Override
+ protected LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent,
+ ILSMDiskComponent lastComponent) throws HyracksDataException {
+ LSMInvertedIndexDiskComponent first = (LSMInvertedIndexDiskComponent) firstComponent;
+ OnDiskInvertedIndex firstInvIndex = (OnDiskInvertedIndex) first.getInvIndex();
+ String firstFileName = firstInvIndex.getBTree().getFileReference().getFile().getName();
+ LSMInvertedIndexDiskComponent last = (LSMInvertedIndexDiskComponent) lastComponent;
+ OnDiskInvertedIndex lastInvIndex = (OnDiskInvertedIndex) last.getInvIndex();
+ String lastFileName = lastInvIndex.getBTree().getFileReference().getFile().getName();
+ return fileManager.getRelMergeFileReference(firstFileName, lastFileName);
+ }
+
+ @Override
+ protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx,
+ ILSMMemoryComponent flushingComponent, LSMComponentFileReferences componentFileRefs,
+ ILSMIOOperationCallback callback) throws HyracksDataException {
+ return new LSMInvertedIndexFlushOperation(new LSMInvertedIndexAccessor(getLsmHarness(), opCtx),
+ flushingComponent, componentFileRefs.getInsertIndexFileReference(),
+ componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(),
+ callback, fileManager.getBaseDir());
+ }
+
+ @Override
+ protected ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx,
+ List<ILSMComponent> mergingComponents, LSMComponentFileReferences mergeFileRefs,
+ ILSMIOOperationCallback callback) throws HyracksDataException {
+ ILSMIndexAccessor accessor = new LSMInvertedIndexAccessor(getLsmHarness(), opCtx);
+ IIndexCursor cursor = new LSMInvertedIndexRangeSearchCursor(opCtx);
+ return new LSMInvertedIndexMergeOperation(accessor, mergingComponents, cursor,
+ mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getDeleteIndexFileReference(),
+ mergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
index edae439..2482103 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
@@ -203,4 +203,22 @@
throw new UnsupportedOperationException("Upsert not supported by lsm inverted index.");
}
+ /**
+ * enter the memory component for modification
+ */
+ @Override
+ public void enter() throws HyracksDataException {
+ ctx.setOperation(IndexOperation.UPSERT);
+ lsmHarness.enter(ctx);
+ }
+
+ /**
+ * exit the memory component
+ */
+ @Override
+ public void exit() throws HyracksDataException {
+ ctx.setOperation(IndexOperation.UPSERT);
+ lsmHarness.exit(ctx);
+ }
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
index 26eed04..df4f095 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
@@ -19,93 +19,29 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.impls;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IODeviceHandle;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.FlushOperation;
-public class LSMInvertedIndexFlushOperation implements ILSMIOOperation, Comparable<LSMInvertedIndexFlushOperation> {
- private final ILSMIndexAccessor accessor;
- private final ILSMComponent flushingComponent;
- private final FileReference dictBTreeFlushTarget;
+public class LSMInvertedIndexFlushOperation extends FlushOperation {
private final FileReference deletedKeysBTreeFlushTarget;
private final FileReference bloomFilterFlushTarget;
- private final ILSMIOOperationCallback callback;
- private final String indexIdentifier;
- public LSMInvertedIndexFlushOperation(ILSMIndexAccessor accessor, ILSMComponent flushingComponent,
- FileReference dictBTreeFlushTarget, FileReference deletedKeysBTreeFlushTarget,
- FileReference bloomFilterFlushTarget, ILSMIOOperationCallback callback, String indexIdentifier) {
- this.accessor = accessor;
- this.flushingComponent = flushingComponent;
- this.dictBTreeFlushTarget = dictBTreeFlushTarget;
+ public LSMInvertedIndexFlushOperation(ILSMIndexAccessor accessor, ILSMMemoryComponent flushingComponent,
+ FileReference flushTarget, FileReference deletedKeysBTreeFlushTarget, FileReference bloomFilterFlushTarget,
+ ILSMIOOperationCallback callback, String indexIdentifier) {
+ super(accessor, flushingComponent, flushTarget, callback, indexIdentifier);
this.deletedKeysBTreeFlushTarget = deletedKeysBTreeFlushTarget;
this.bloomFilterFlushTarget = bloomFilterFlushTarget;
- this.callback = callback;
- this.indexIdentifier = indexIdentifier;
}
- @Override
- public Set<IODeviceHandle> getReadDevices() {
- return Collections.emptySet();
- }
-
- @Override
- public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<>();
- devs.add(dictBTreeFlushTarget.getDeviceHandle());
- devs.add(deletedKeysBTreeFlushTarget.getDeviceHandle());
- devs.add(bloomFilterFlushTarget.getDeviceHandle());
- return devs;
-
- }
-
- @Override
- public Boolean call() throws HyracksDataException {
- accessor.flush(this);
- return true;
- }
-
- @Override
- public ILSMIOOperationCallback getCallback() {
- return callback;
- }
-
- public FileReference getDictBTreeFlushTarget() {
- return dictBTreeFlushTarget;
- }
-
- public FileReference getDeletedKeysBTreeFlushTarget() {
+ public FileReference getDeletedKeysBTreeTarget() {
return deletedKeysBTreeFlushTarget;
}
- public FileReference getBloomFilterFlushTarget() {
+ public FileReference getBloomFilterTarget() {
return bloomFilterFlushTarget;
}
-
- public ILSMComponent getFlushingComponent() {
- return flushingComponent;
- }
-
- @Override
- public String getIndexUniqueIdentifier() {
- return indexIdentifier;
- }
-
- @Override
- public LSMIOOpertionType getIOOpertionType() {
- return LSMIOOpertionType.FLUSH;
- }
-
- @Override
- public int compareTo(LSMInvertedIndexFlushOperation o) {
- return dictBTreeFlushTarget.getFile().getName().compareTo(o.getDictBTreeFlushTarget().getFile().getName());
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
index c3c8c8b..da374dc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
@@ -19,103 +19,32 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.impls;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
-import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation;
import org.apache.hyracks.storage.common.IIndexCursor;
-public class LSMInvertedIndexMergeOperation implements ILSMIOOperation {
- private final ILSMIndexAccessor accessor;
- private final List<ILSMComponent> mergingComponents;
- private final IIndexCursor cursor;
- private final FileReference dictBTreeMergeTarget;
+public class LSMInvertedIndexMergeOperation extends MergeOperation {
private final FileReference deletedKeysBTreeMergeTarget;
private final FileReference bloomFilterMergeTarget;
- private final ILSMIOOperationCallback callback;
- private final String indexIdentifier;
public LSMInvertedIndexMergeOperation(ILSMIndexAccessor accessor, List<ILSMComponent> mergingComponents,
- IIndexCursor cursor, FileReference dictBTreeMergeTarget, FileReference deletedKeysBTreeMergeTarget,
+ IIndexCursor cursor, FileReference target, FileReference deletedKeysBTreeMergeTarget,
FileReference bloomFilterMergeTarget, ILSMIOOperationCallback callback, String indexIdentifier) {
- this.accessor = accessor;
- this.mergingComponents = mergingComponents;
- this.cursor = cursor;
- this.dictBTreeMergeTarget = dictBTreeMergeTarget;
+ super(accessor, target, callback, indexIdentifier, mergingComponents, cursor);
this.deletedKeysBTreeMergeTarget = deletedKeysBTreeMergeTarget;
this.bloomFilterMergeTarget = bloomFilterMergeTarget;
- this.callback = callback;
- this.indexIdentifier = indexIdentifier;
}
- @Override
- public Set<IODeviceHandle> getReadDevices() {
- Set<IODeviceHandle> devs = new HashSet<>();
- for (Object o : mergingComponents) {
- LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) o;
- OnDiskInvertedIndex invIndex = (OnDiskInvertedIndex) component.getInvIndex();
- devs.add(invIndex.getBTree().getFileReference().getDeviceHandle());
- devs.add(component.getDeletedKeysBTree().getFileReference().getDeviceHandle());
- devs.add(component.getBloomFilter().getFileReference().getDeviceHandle());
- }
- return devs;
- }
-
- @Override
- public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<>();
- devs.add(dictBTreeMergeTarget.getDeviceHandle());
- devs.add(deletedKeysBTreeMergeTarget.getDeviceHandle());
- devs.add(bloomFilterMergeTarget.getDeviceHandle());
- return devs;
- }
-
- @Override
- public Boolean call() throws HyracksDataException {
- accessor.merge(this);
- return true;
- }
-
- @Override
- public ILSMIOOperationCallback getCallback() {
- return callback;
- }
-
- public FileReference getDictBTreeMergeTarget() {
- return dictBTreeMergeTarget;
- }
-
- public FileReference getDeletedKeysBTreeMergeTarget() {
+ public FileReference getDeletedKeysBTreeTarget() {
return deletedKeysBTreeMergeTarget;
}
- public FileReference getBloomFilterMergeTarget() {
+ public FileReference getBloomFilterTarget() {
return bloomFilterMergeTarget;
}
-
- public IIndexCursor getCursor() {
- return cursor;
- }
-
- public List<ILSMComponent> getMergingComponents() {
- return mergingComponents;
- }
-
- @Override
- public String getIndexUniqueIdentifier() {
- return indexIdentifier;
- }
-
- @Override
- public LSMIOOpertionType getIOOpertionType() {
- return LSMIOOpertionType.MERGE;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
index 44c1c13..29f98b4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
@@ -19,65 +19,39 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.impls;
-import java.util.LinkedList;
import java.util.List;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
-import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor;
import org.apache.hyracks.storage.common.IIndexAccessor;
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
-import org.apache.hyracks.storage.common.ISearchPredicate;
-import org.apache.hyracks.storage.common.MultiComparator;
public class LSMInvertedIndexOpContext extends AbstractLSMIndexOperationContext {
private static final int NUM_DOCUMENT_FIELDS = 1;
- private IndexOperation op;
- private final List<ILSMComponent> componentHolder;
- private final List<ILSMDiskComponent> componentsToBeMerged;
- private final List<ILSMDiskComponent> componentsToBeReplicated;
-
- private IModificationOperationCallback modificationCallback;
- private ISearchOperationCallback searchCallback;
-
// Tuple that only has the inverted-index elements (aka keys), projecting away the document fields.
private PermutingTupleReference keysOnlyTuple;
-
// Accessor to the in-memory inverted indexes.
private IInvertedIndexAccessor[] mutableInvIndexAccessors;
// Accessor to the deleted-keys BTrees.
private IIndexAccessor[] deletedKeysBTreeAccessors;
-
private IInvertedIndexAccessor currentMutableInvIndexAccessors;
private IIndexAccessor currentDeletedKeysBTreeAccessors;
- private final PermutingTupleReference indexTuple;
- private final MultiComparator filterCmp;
- private final PermutingTupleReference filterTuple;
-
- private ISearchPredicate searchPredicate;
-
public LSMInvertedIndexOpContext(List<ILSMMemoryComponent> mutableComponents,
IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback,
- int[] invertedIndexFields, int[] filterFields) throws HyracksDataException {
- this.componentHolder = new LinkedList<>();
- this.componentsToBeMerged = new LinkedList<>();
- this.componentsToBeReplicated = new LinkedList<>();
- this.modificationCallback = modificationCallback;
- this.searchCallback = searchCallback;
-
+ int[] invertedIndexFields, int[] filterFields, IBinaryComparatorFactory[] filterComparatorFactories)
+ throws HyracksDataException {
+ super(invertedIndexFields, filterFields, filterComparatorFactories, searchCallback, modificationCallback);
mutableInvIndexAccessors = new IInvertedIndexAccessor[mutableComponents.size()];
deletedKeysBTreeAccessors = new IIndexAccessor[mutableComponents.size()];
-
for (int i = 0; i < mutableComponents.size(); i++) {
LSMInvertedIndexMemoryComponent mutableComponent =
(LSMInvertedIndexMemoryComponent) mutableComponents.get(i);
@@ -86,9 +60,6 @@
deletedKeysBTreeAccessors[i] = mutableComponent.getDeletedKeysBTree()
.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
}
-
- assert mutableComponents.size() > 0;
-
// Project away the document fields, leaving only the key fields.
LSMInvertedIndexMemoryComponent c = (LSMInvertedIndexMemoryComponent) mutableComponents.get(0);
int numKeyFields = c.getInvIndex().getInvListTypeTraits().length;
@@ -97,51 +68,6 @@
keyFieldPermutation[i] = NUM_DOCUMENT_FIELDS + i;
}
keysOnlyTuple = new PermutingTupleReference(keyFieldPermutation);
-
- if (filterFields != null) {
- indexTuple = new PermutingTupleReference(invertedIndexFields);
- filterCmp = MultiComparator.create(c.getLSMComponentFilter().getFilterCmpFactories());
- filterTuple = new PermutingTupleReference(filterFields);
- } else {
- indexTuple = null;
- filterCmp = null;
- filterTuple = null;
- }
- }
-
- @Override
- public void reset() {
- super.reset();
- componentHolder.clear();
- componentsToBeMerged.clear();
- componentsToBeReplicated.clear();
- }
-
- @Override
- // TODO: Ignore opcallback for now.
- public void setOperation(IndexOperation newOp) throws HyracksDataException {
- reset();
- op = newOp;
- }
-
- @Override
- public IndexOperation getOperation() {
- return op;
- }
-
- @Override
- public List<ILSMComponent> getComponentHolder() {
- return componentHolder;
- }
-
- @Override
- public ISearchOperationCallback getSearchOperationCallback() {
- return searchCallback;
- }
-
- @Override
- public IModificationOperationCallback getModificationCallback() {
- return modificationCallback;
}
@Override
@@ -150,34 +76,6 @@
currentDeletedKeysBTreeAccessors = deletedKeysBTreeAccessors[currentMutableComponentId];
}
- @Override
- public List<ILSMDiskComponent> getComponentsToBeMerged() {
- return componentsToBeMerged;
- }
-
- @Override
- public void setSearchPredicate(ISearchPredicate searchPredicate) {
- this.searchPredicate = searchPredicate;
- }
-
- @Override
- public ISearchPredicate getSearchPredicate() {
- return searchPredicate;
- }
-
- @Override
- public List<ILSMDiskComponent> getComponentsToBeReplicated() {
- return componentsToBeReplicated;
- }
-
- public MultiComparator getFilterCmp() {
- return filterCmp;
- }
-
- public PermutingTupleReference getIndexTuple() {
- return indexTuple;
- }
-
public IInvertedIndexAccessor getCurrentMutableInvIndexAccessors() {
return currentMutableInvIndexAccessors;
}
@@ -189,8 +87,4 @@
public IIndexAccessor getCurrentDeletedKeysBTreeAccessors() {
return currentDeletedKeysBTreeAccessors;
}
-
- public PermutingTupleReference getFilterTuple() {
- return filterTuple;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index 9c56dec..a9eca4d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -33,25 +33,21 @@
import org.apache.hyracks.storage.am.common.api.IPageManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.impls.AbstractSearchPredicate;
-import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
-import org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import org.apache.hyracks.storage.am.rtree.impls.RTree;
@@ -81,7 +77,6 @@
protected final ITreeIndexFrameFactory btreeInteriorFrameFactory;
protected final ITreeIndexFrameFactory rtreeLeafFrameFactory;
protected final ITreeIndexFrameFactory btreeLeafFrameFactory;
- protected final int[] rtreeFields;
public AbstractLSMRTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
@@ -97,7 +92,7 @@
throws HyracksDataException {
super(ioManager, virtualBufferCaches, diskBufferCache, fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
- filterManager, filterFields, durable);
+ filterManager, filterFields, durable, filterFactory, rtreeFields);
int i = 0;
for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
RTree memRTree = new RTree(virtualBufferCache, virtualBufferCache.getFileMapProvider(),
@@ -125,7 +120,6 @@
this.linearizer = linearizer;
this.comparatorFields = comparatorFields;
this.linearizerArray = linearizerArray;
- this.rtreeFields = rtreeFields;
this.isPointMBR = isPointMBR;
}
@@ -153,126 +147,16 @@
this.linearizer = linearizer;
this.comparatorFields = comparatorFields;
this.linearizerArray = linearizerArray;
- this.rtreeFields = null;
this.isPointMBR = isPointMBR;
}
@Override
- public synchronized void create() throws HyracksDataException {
- if (isActivated) {
- throw new HyracksDataException("Failed to create the index since it is activated.");
- }
-
- fileManager.deleteDirs();
- fileManager.createDirs();
- diskComponents.clear();
- }
-
- @Override
- public synchronized void activate() throws HyracksDataException {
- if (isActivated) {
- throw new HyracksDataException("Failed to activate the index since it is already activated.");
- }
- }
-
- @Override
- public synchronized void deactivate(boolean flushOnExit) throws HyracksDataException {
- if (!isActivated) {
- throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
- }
-
- if (flushOnExit) {
- BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
- ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- accessor.scheduleFlush(cb);
- try {
- cb.waitForIO();
- } catch (InterruptedException e) {
- throw new HyracksDataException(e);
- }
- }
- deallocateMemoryComponents();
- }
-
- @Override
- public synchronized void destroy() throws HyracksDataException {
- if (isActivated) {
- throw new HyracksDataException("Failed to destroy the index since it is activated.");
- }
- }
-
- @Override
- public synchronized void clear() throws HyracksDataException {
- if (!isActivated) {
- throw new HyracksDataException("Failed to clear the index since it is not activated.");
- }
- clearMemoryComponents();
- }
-
- @Override
- public void getOperationalComponents(ILSMIndexOperationContext ctx) throws HyracksDataException {
- List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- int cmc = currentMutableComponentId.get();
- ctx.setCurrentMutableComponentId(cmc);
- operationalComponents.clear();
- switch (ctx.getOperation()) {
- case INSERT:
- case DELETE:
- case FLUSH:
- case UPSERT:
- operationalComponents.add(memoryComponents.get(cmc));
- break;
- case SEARCH:
- if (memoryComponentsAllocated) {
- addOperationalMutableComponents(operationalComponents);
- }
- if (filterManager != null) {
- for (ILSMComponent c : immutableComponents) {
- if (c.getLSMComponentFilter().satisfy(
- ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMinFilterTuple(),
- ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMaxFilterTuple(),
- ((LSMRTreeOpContext) ctx).getFilterCmp())) {
- operationalComponents.add(c);
- }
- }
- } else {
- operationalComponents.addAll(immutableComponents);
- }
- break;
- case MERGE:
- operationalComponents.addAll(ctx.getComponentsToBeMerged());
- break;
- case FULL_MERGE:
- operationalComponents.addAll(immutableComponents);
- break;
- case REPLICATE:
- operationalComponents.addAll(ctx.getComponentsToBeReplicated());
- break;
- default:
- throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
- }
- }
-
- @Override
public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred)
throws HyracksDataException {
LSMRTreeOpContext ctx = (LSMRTreeOpContext) ictx;
cursor.open(ctx.getSearchInitialState(), pred);
}
- protected LSMComponentFileReferences getMergeTargetFileName(List<ILSMComponent> mergingDiskComponents)
- throws HyracksDataException {
- RTree firstTree = ((LSMRTreeDiskComponent) mergingDiskComponents.get(0)).getRTree();
- RTree lastTree =
- ((LSMRTreeDiskComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1)).getRTree();
- FileReference firstFile = firstTree.getFileReference();
- FileReference lastFile = lastTree.getFileReference();
- LSMComponentFileReferences fileRefs =
- fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
- return fileRefs;
- }
-
protected LSMRTreeDiskComponent createDiskComponent(ILSMDiskComponentFactory factory, FileReference insertFileRef,
FileReference deleteFileRef, FileReference bloomFilterFileRef, boolean createComponent)
throws HyracksDataException {
@@ -293,7 +177,7 @@
component.getBloomFilter().activate();
}
if (component.getLSMComponentFilter() != null && !createComponent) {
- filterManager.readFilter(component.getLSMComponentFilter(), component.getRTree());
+ getFilterManager().readFilter(component.getLSMComponentFilter(), component.getRTree());
}
return component;
}
@@ -373,18 +257,15 @@
}
}
}
- if (ctx.getFilterTuple() != null) {
- ctx.getFilterTuple().reset(tuple);
- memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter().update(ctx.getFilterTuple(),
- ctx.getFilterCmp());
- }
+ updateFilter(ctx, tuple);
}
+ @Override
protected LSMRTreeOpContext createOpContext(IModificationOperationCallback modCallback,
ISearchOperationCallback searchCallback) {
return new LSMRTreeOpContext(memoryComponents, rtreeLeafFrameFactory, rtreeInteriorFrameFactory,
- btreeLeafFrameFactory, modCallback, searchCallback, rtreeFields, filterFields, lsmHarness,
- comparatorFields, linearizerArray);
+ btreeLeafFrameFactory, modCallback, searchCallback, getTreeFields(), getFilterFields(), getLsmHarness(),
+ comparatorFields, linearizerArray, getFilterCmpFactories());
}
@Override
@@ -393,19 +274,20 @@
}
@Override
- public void validate() throws HyracksDataException {
+ protected void validateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
throw new UnsupportedOperationException("Validation not implemented for LSM R-Trees.");
}
@Override
- public long getMemoryAllocationSize() {
- long size = 0;
- for (ILSMComponent c : memoryComponents) {
- LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
- IBufferCache virtualBufferCache = mutableComponent.getRTree().getBufferCache();
- size += virtualBufferCache.getNumPages() * virtualBufferCache.getPageSize();
- }
- return size;
+ protected void validateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ throw new UnsupportedOperationException("Validation not implemented for LSM R-Trees.");
+ }
+
+ @Override
+ protected long getMemoryComponentSize(ILSMMemoryComponent c) {
+ LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+ IBufferCache virtualBufferCache = mutableComponent.getRTree().getBufferCache();
+ return virtualBufferCache.getNumPages() * (long) virtualBufferCache.getPageSize();
}
@Override
@@ -419,61 +301,47 @@
}
@Override
- public synchronized void allocateMemoryComponents() throws HyracksDataException {
- if (!isActivated) {
- throw new HyracksDataException("Failed to allocate memory components since the index is not active.");
- }
- if (memoryComponentsAllocated) {
- return;
- }
- for (ILSMComponent c : memoryComponents) {
- LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
- ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).open();
- mutableComponent.getRTree().create();
- mutableComponent.getBTree().create();
- mutableComponent.getRTree().activate();
- mutableComponent.getBTree().activate();
- }
- memoryComponentsAllocated = true;
+ protected void allocateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+ ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).open();
+ mutableComponent.getRTree().create();
+ mutableComponent.getBTree().create();
+ mutableComponent.getRTree().activate();
+ mutableComponent.getBTree().activate();
}
- private void addOperationalMutableComponents(List<ILSMComponent> operationalComponents) {
- int cmc = currentMutableComponentId.get();
- int numMutableComponents = memoryComponents.size();
- for (int i = 0; i < numMutableComponents - 1; i++) {
- ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
- LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
- if (mutableComponent.isReadable()) {
- // Make sure newest components are added first
- operationalComponents.add(0, mutableComponent);
- }
- }
- // The current mutable component is always added
- operationalComponents.add(0, memoryComponents.get(cmc));
+ @Override
+ protected void deactivateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+ mutableComponent.getRTree().deactivate();
+ mutableComponent.getBTree().deactivate();
+ mutableComponent.getRTree().destroy();
+ mutableComponent.getBTree().destroy();
+ ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).close();
}
- private synchronized void clearMemoryComponents() throws HyracksDataException {
- if (memoryComponentsAllocated) {
- for (ILSMComponent c : memoryComponents) {
- LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
- mutableComponent.getRTree().clear();
- mutableComponent.getBTree().clear();
- mutableComponent.reset();
- }
- }
+ @Override
+ protected void clearMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+ mutableComponent.getRTree().clear();
+ mutableComponent.getBTree().clear();
+ mutableComponent.reset();
}
- private synchronized void deallocateMemoryComponents() throws HyracksDataException {
- if (memoryComponentsAllocated) {
- for (ILSMComponent c : memoryComponents) {
- LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
- mutableComponent.getRTree().deactivate();
- mutableComponent.getBTree().deactivate();
- mutableComponent.getRTree().destroy();
- mutableComponent.getBTree().destroy();
- ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).close();
- }
- memoryComponentsAllocated = false;
- }
+ @Override
+ protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+ mutableComponent.getRTree().destroy();
+ mutableComponent.getBTree().destroy();
+ }
+
+ @Override
+ protected LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent,
+ ILSMDiskComponent lastComponent) throws HyracksDataException {
+ RTree firstTree = ((LSMRTreeDiskComponent) firstComponent).getRTree();
+ RTree lastTree = ((LSMRTreeDiskComponent) lastComponent).getRTree();
+ FileReference firstFile = firstTree.getFileReference();
+ FileReference lastFile = lastTree.getFileReference();
+ return fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
index d6b94ec..987aec7 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
@@ -24,7 +24,6 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
-import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
@@ -102,6 +101,11 @@
this.fieldCount = fieldCount;
}
+ @Override
+ public ExternalIndexHarness getLsmHarness() {
+ return (ExternalIndexHarness) super.getLsmHarness();
+ }
+
// The subsume merged components is overridden to account for:
// 1. the number of readers of components
// 2. maintaining two versions of the index
@@ -189,7 +193,7 @@
@Override
public synchronized void activate() throws HyracksDataException {
- if (isActivated) {
+ if (isActive) {
throw new HyracksDataException("Failed to activate the index since it is already activated.");
}
@@ -206,7 +210,7 @@
diskComponents.add(component);
secondDiskComponents.add(component);
}
- ((ExternalIndexHarness) lsmHarness).indexFirstTimeActivated();
+ getLsmHarness().indexFirstTimeActivated();
} else {
// This index has been opened before or is brand new with no components
// components. It should also maintain the version pointer
@@ -232,7 +236,7 @@
}
}
}
- isActivated = true;
+ isActive = true;
}
@Override
@@ -258,20 +262,29 @@
@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation;
- ITreeIndexCursor cursor = mergeOp.getCursor();
+ IIndexCursor cursor = mergeOp.getCursor();
ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
ILSMIndexOperationContext opCtx = ((LSMRTreeSortedCursor) cursor).getOpCtx();
opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
search(opCtx, cursor, rtreeSearchPred);
- LSMRTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getRTreeMergeTarget(),
- mergeOp.getBTreeMergeTarget(), mergeOp.getBloomFilterMergeTarget(), true);
+ LSMRTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(),
+ mergeOp.getBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
// In case we must keep the deleted-keys BTrees, then they must be
// merged *before* merging the r-trees so that
// lsmHarness.endSearch() is called once when the r-trees have been
// merged.
- if (mergeOp.isKeepDeletedTuples()) {
+ boolean keepDeleteTuples = false;
+ if (version == 0) {
+ keepDeleteTuples = mergeOp.getMergingComponents()
+ .get(mergeOp.getMergingComponents().size() - 1) != diskComponents.get(diskComponents.size() - 1);
+ } else {
+ keepDeleteTuples = mergeOp.getMergingComponents()
+ .get(mergeOp.getMergingComponents().size() - 1) != secondDiskComponents
+ .get(secondDiskComponents.size() - 1);
+ }
+ if (keepDeleteTuples) {
// Keep the deleted tuples since the oldest disk component is not
// included in the merge operation
@@ -323,7 +336,7 @@
@Override
public void deactivate(boolean flushOnExit) throws HyracksDataException {
- if (!isActivated) {
+ if (!isActive) {
throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
}
@@ -353,17 +366,17 @@
bloomFilter.deactivate();
}
}
- isActivated = false;
+ isActive = false;
}
// The clear method is not used anywhere in AsterixDB! we override it anyway
// to exit components first and clear the two lists
@Override
public void clear() throws HyracksDataException {
- if (!isActivated) {
+ if (!isActive) {
throw new HyracksDataException("Failed to clear the index since it is not activated.");
}
- ((ExternalIndexHarness) lsmHarness).indexClear();
+ getLsmHarness().indexClear();
for (ILSMComponent c : diskComponents) {
LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
@@ -394,7 +407,7 @@
@Override
public void destroy() throws HyracksDataException {
- if (isActivated) {
+ if (isActive) {
throw new HyracksDataException("Failed to destroy the index since it is activated.");
}
for (ILSMComponent c : diskComponents) {
@@ -479,16 +492,16 @@
// For initial load
@Override
- public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- return new LSMTwoPCRTreeBulkLoader(fillLevel, verifyInput, 0, checkIfEmptyIndex, false);
+ public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
+ return new LSMTwoPCRTreeBulkLoader(fillLevel, verifyInput, 0, false);
}
// For transaction bulk load <- could consolidate with the above method ->
@Override
- public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- return new LSMTwoPCRTreeBulkLoader(fillLevel, verifyInput, numElementsHint, checkIfEmptyIndex, true);
+ public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
+ return new LSMTwoPCRTreeBulkLoader(fillLevel, verifyInput, numElementsHint, true);
}
// The bulk loader used for both initial loading and transaction
@@ -504,15 +517,12 @@
private final boolean isTransaction;
public LSMTwoPCRTreeBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex, boolean isTransaction) throws HyracksDataException {
+ boolean isTransaction) throws HyracksDataException {
this.isTransaction = isTransaction;
// Create the appropriate target
if (isTransaction) {
component = createTransactionTarget();
} else {
- if (checkIfEmptyIndex && !isEmptyIndex()) {
- throw HyracksDataException.create(ErrorCode.LOAD_NON_EMPTY_INDEX);
- }
component = createBulkLoadTarget();
}
@@ -589,7 +599,7 @@
btree.deactivate();
bloomFilter.deactivate();
} else {
- lsmHarness.addBulkLoadedComponent(component);
+ getLsmHarness().addBulkLoadedComponent(component);
}
}
}
@@ -646,37 +656,28 @@
rctx.setOperation(IndexOperation.MERGE);
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
ITreeIndexCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer, buddyBTreeFields);
- LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
- ILSMIndexAccessor accessor = new LSMRTreeAccessor(lsmHarness, rctx);
+ LSMComponentFileReferences relMergeFileRefs =
+ getMergeFileReferences((ILSMDiskComponent) mergingComponents.get(0),
+ (ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1));
+ ILSMIndexAccessor accessor = new LSMRTreeAccessor(getLsmHarness(), rctx, buddyBTreeFields);
// create the merge operation.
LSMRTreeMergeOperation mergeOp = new LSMRTreeMergeOperation(accessor, mergingComponents, cursor,
relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(),
relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir());
- // set the keepDeletedTuples flag
- boolean keepDeleteTuples = false;
- if (version == 0) {
- keepDeleteTuples = mergeOp.getMergingComponents()
- .get(mergeOp.getMergingComponents().size() - 1) != diskComponents.get(diskComponents.size() - 1);
- } else {
- keepDeleteTuples = mergeOp.getMergingComponents()
- .get(mergeOp.getMergingComponents().size() - 1) != secondDiskComponents
- .get(secondDiskComponents.size() - 1);
- }
- mergeOp.setKeepDeletedTuples(keepDeleteTuples);
-
ioScheduler.scheduleOperation(mergeOp);
}
@Override
public ILSMIndexAccessor createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion)
throws HyracksDataException {
- return new LSMRTreeAccessor(lsmHarness, createOpContext(searchCallback, targetIndexVersion));
+ return new LSMRTreeAccessor(getLsmHarness(), createOpContext(searchCallback, targetIndexVersion),
+ buddyBTreeFields);
}
// This method creates the appropriate opContext for the targeted version
public ExternalRTreeOpContext createOpContext(ISearchOperationCallback searchCallback, int targetVersion) {
return new ExternalRTreeOpContext(rtreeCmpFactories, btreeCmpFactories, searchCallback, targetVersion,
- lsmHarness, comparatorFields, linearizerArray, rtreeLeafFrameFactory, rtreeInteriorFrameFactory,
+ getLsmHarness(), comparatorFields, linearizerArray, rtreeLeafFrameFactory, rtreeInteriorFrameFactory,
btreeLeafFrameFactory);
}
@@ -685,7 +686,7 @@
@Override
public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
- return new LSMRTreeAccessor(lsmHarness, createOpContext(searchCallback, version));
+ return new LSMRTreeAccessor(getLsmHarness(), createOpContext(searchCallback, version), buddyBTreeFields);
}
@Override
@@ -717,7 +718,7 @@
componentFileRefrences.getDeleteIndexFileReference(),
componentFileRefrences.getBloomFilterFileReference(), false);
}
- ((ExternalIndexHarness) lsmHarness).addTransactionComponents(component);
+ getLsmHarness().addTransactionComponents(component);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
index 76cf339..73be537 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
@@ -18,31 +18,17 @@
*/
package org.apache.hyracks.storage.am.lsm.rtree.impls;
-import java.util.LinkedList;
-import java.util.List;
-
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
-import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
-import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
public class ExternalRTreeOpContext extends AbstractLSMIndexOperationContext {
- private IndexOperation op;
private MultiComparator bTreeCmp;
private MultiComparator rTreeCmp;
- private final List<ILSMComponent> componentHolder;
- private final List<ILSMDiskComponent> componentsToBeMerged;
- private final List<ILSMDiskComponent> componentsToBeReplicated;
- private final ISearchOperationCallback searchCallback;
private final int targetIndexVersion;
- private ISearchPredicate searchPredicate;
private LSMRTreeCursorInitialState initialState;
public ExternalRTreeOpContext(IBinaryComparatorFactory[] rtreeCmpFactories,
@@ -50,10 +36,7 @@
int targetIndexVersion, ILSMHarness lsmHarness, int[] comparatorFields,
IBinaryComparatorFactory[] linearizerArray, ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory) {
- this.componentHolder = new LinkedList<>();
- this.componentsToBeMerged = new LinkedList<>();
- this.componentsToBeReplicated = new LinkedList<>();
- this.searchCallback = searchCallback;
+ super(null, null, null, searchCallback, null);
this.targetIndexVersion = targetIndexVersion;
this.bTreeCmp = MultiComparator.create(btreeCmpFactories);
this.rTreeCmp = MultiComparator.create(rtreeCmpFactories);
@@ -63,29 +46,10 @@
}
@Override
- public void setOperation(IndexOperation newOp) {
- reset();
- this.op = newOp;
- }
-
- @Override
public void setCurrentMutableComponentId(int currentMutableComponentId) {
// Do nothing. this should never be called for disk only indexes
}
- @Override
- public void reset() {
- super.reset();
- componentHolder.clear();
- componentsToBeMerged.clear();
- componentsToBeReplicated.clear();
- }
-
- @Override
- public IndexOperation getOperation() {
- return op;
- }
-
public MultiComparator getBTreeMultiComparator() {
return bTreeCmp;
}
@@ -94,46 +58,10 @@
return rTreeCmp;
}
- @Override
- public List<ILSMComponent> getComponentHolder() {
- return componentHolder;
- }
-
- @Override
- public ISearchOperationCallback getSearchOperationCallback() {
- return searchCallback;
- }
-
- // This should never be needed for disk only indexes
- @Override
- public IModificationOperationCallback getModificationCallback() {
- return null;
- }
-
- @Override
- public List<ILSMDiskComponent> getComponentsToBeMerged() {
- return componentsToBeMerged;
- }
-
public int getTargetIndexVersion() {
return targetIndexVersion;
}
- @Override
- public void setSearchPredicate(ISearchPredicate searchPredicate) {
- this.searchPredicate = searchPredicate;
- }
-
- @Override
- public ISearchPredicate getSearchPredicate() {
- return searchPredicate;
- }
-
- @Override
- public List<ILSMDiskComponent> getComponentsToBeReplicated() {
- return componentsToBeReplicated;
- }
-
public LSMRTreeCursorInitialState getInitialState() {
return initialState;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 8882639..640b311 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -45,25 +45,23 @@
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.tuples.DualTupleReference;
-import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
-import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
-import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.am.rtree.impls.RTree;
import org.apache.hyracks.storage.am.rtree.impls.RTreeSearchCursor;
@@ -73,11 +71,10 @@
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
-import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMRTree extends AbstractLSMRTree {
- protected int[] buddyBTreeFields;
+ protected final int[] buddyBTreeFields;
public LSMRTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
@@ -124,92 +121,42 @@
this.buddyBTreeFields = buddyBTreeFields;
}
- /**
- * Opens LSMRTree, cleaning up invalid files from base dir, and registering
- * all valid files as on-disk RTrees and BTrees.
- *
- * @throws HyracksDataException
- */
@Override
- public synchronized void activate() throws HyracksDataException {
- super.activate();
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- List<LSMComponentFileReferences> validFileReferences;
- validFileReferences = fileManager.cleanupAndGetValidFiles();
- immutableComponents.clear();
- for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
- LSMRTreeDiskComponent component;
- component = createDiskComponent(componentFactory, lsmComonentFileReference.getInsertIndexFileReference(),
- lsmComonentFileReference.getDeleteIndexFileReference(),
- lsmComonentFileReference.getBloomFilterFileReference(), false);
- immutableComponents.add(component);
- }
- isActivated = true;
- }
-
- @Override
- public synchronized void deactivate(boolean flushOnExit) throws HyracksDataException {
- super.deactivate(flushOnExit);
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
- RTree rtree = component.getRTree();
- BTree btree = component.getBTree();
- BloomFilter bloomFilter = component.getBloomFilter();
- rtree.deactivateCloseHandle();
- btree.deactivateCloseHandle();
- bloomFilter.deactivate();
- }
- isActivated = false;
- }
-
- @Override
- public synchronized void deactivate() throws HyracksDataException {
- deactivate(true);
- }
-
- @Override
- public synchronized void destroy() throws HyracksDataException {
- super.destroy();
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
- component.getBTree().destroy();
- component.getBloomFilter().destroy();
- component.getRTree().destroy();
- }
- fileManager.deleteDirs();
- }
-
- @Override
- public synchronized void clear() throws HyracksDataException {
- super.clear();
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
- component.getBTree().deactivate();
- component.getBloomFilter().deactivate();
- component.getRTree().deactivate();
- component.getBTree().destroy();
- component.getBloomFilter().destroy();
- component.getRTree().destroy();
- }
- immutableComponents.clear();
- }
-
- @Override
- public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ protected ILSMDiskComponent loadComponent(LSMComponentFileReferences lsmComonentFileReferences)
throws HyracksDataException {
- ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
- LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- ILSMIndexOperationContext rctx =
- createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- rctx.setOperation(IndexOperation.FLUSH);
- rctx.getComponentHolder().addAll(ctx.getComponentHolder());
- LSMRTreeAccessor accessor = new LSMRTreeAccessor(lsmHarness, rctx);
- ioScheduler.scheduleOperation(new LSMRTreeFlushOperation(accessor, flushingComponent,
- componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(),
- componentFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir()));
+ return createDiskComponent(componentFactory, lsmComonentFileReferences.getInsertIndexFileReference(),
+ lsmComonentFileReferences.getDeleteIndexFileReference(),
+ lsmComonentFileReferences.getBloomFilterFileReference(), false);
+ }
+
+ @Override
+ protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+ RTree rtree = component.getRTree();
+ BTree btree = component.getBTree();
+ BloomFilter bloomFilter = component.getBloomFilter();
+ rtree.deactivateCloseHandle();
+ btree.deactivateCloseHandle();
+ bloomFilter.deactivate();
+ }
+
+ @Override
+ protected void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+ component.getBTree().destroy();
+ component.getBloomFilter().destroy();
+ component.getRTree().destroy();
+ }
+
+ @Override
+ protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+ component.getBTree().deactivate();
+ component.getBloomFilter().deactivate();
+ component.getRTree().deactivate();
+ component.getBTree().destroy();
+ component.getBloomFilter().destroy();
+ component.getRTree().destroy();
}
@Override
@@ -226,8 +173,8 @@
RTreeSearchCursor rtreeScanCursor = (RTreeSearchCursor) memRTreeAccessor.createSearchCursor(false);
SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
- LSMRTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getRTreeFlushTarget(),
- flushOp.getBTreeFlushTarget(), flushOp.getBloomFilterFlushTarget(), true);
+ LSMRTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(),
+ flushOp.getBTreeTarget(), flushOp.getBloomFilterTarget(), true);
RTree diskRTree = component.getRTree();
IIndexBulkLoader rTreeBulkloader;
ITreeIndexCursor cursor;
@@ -314,8 +261,8 @@
List<ITupleReference> filterTuples = new ArrayList<>();
filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
- filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilter(component.getLSMComponentFilter(), component.getRTree());
+ getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
+ getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getRTree());
}
// Note. If we change the filter to write to metadata object, we don't need the if block above
flushingComponent.getMetadata().copy(component.getMetadata());
@@ -324,31 +271,16 @@
}
@Override
- public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
- throws HyracksDataException {
- ILSMIndexOperationContext rctx =
- createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- rctx.setOperation(IndexOperation.MERGE);
- List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
- ITreeIndexCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer, buddyBTreeFields);
- LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
- ILSMIndexAccessor accessor = new LSMRTreeAccessor(lsmHarness, rctx);
- ioScheduler.scheduleOperation(new LSMRTreeMergeOperation(accessor, mergingComponents, cursor,
- relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(),
- relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir()));
- }
-
- @Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation;
- ITreeIndexCursor cursor = mergeOp.getCursor();
+ IIndexCursor cursor = mergeOp.getCursor();
ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
ILSMIndexOperationContext opCtx = ((LSMRTreeSortedCursor) cursor).getOpCtx();
opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
search(opCtx, cursor, rtreeSearchPred);
- LSMRTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getRTreeMergeTarget(),
- mergeOp.getBTreeMergeTarget(), mergeOp.getBloomFilterMergeTarget(), true);
+ LSMRTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(),
+ mergeOp.getBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
// In case we must keep the deleted-keys BTrees, then they must be merged *before* merging the r-trees so that
// lsmHarness.endSearch() is called once when the r-trees have been merged.
@@ -392,8 +324,8 @@
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
}
- filterManager.updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getRTree());
+ getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
+ getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getRTree());
}
btreeBulkLoader.end();
@@ -415,47 +347,8 @@
@Override
public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
- return new LSMRTreeAccessor(lsmHarness, createOpContext(modificationCallback, searchCallback));
- }
-
- public class LSMRTreeAccessor extends LSMTreeIndexAccessor {
- private final DualTupleReference dualTuple;
-
- public LSMRTreeAccessor(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) {
- super(lsmHarness, ctx);
- dualTuple = new DualTupleReference(buddyBTreeFields);
- }
-
- @Override
- public ITreeIndexCursor createSearchCursor(boolean exclusive) {
- return new LSMRTreeSearchCursor(ctx, buddyBTreeFields);
- }
-
- @Override
- public void delete(ITupleReference tuple) throws HyracksDataException {
- ctx.setOperation(IndexOperation.DELETE);
- dualTuple.reset(tuple);
- lsmHarness.modify(ctx, false, dualTuple);
- }
-
- @Override
- public boolean tryDelete(ITupleReference tuple) throws HyracksDataException {
- ctx.setOperation(IndexOperation.DELETE);
- dualTuple.reset(tuple);
- return lsmHarness.modify(ctx, true, dualTuple);
- }
-
- @Override
- public void forceDelete(ITupleReference tuple) throws HyracksDataException {
- ctx.setOperation(IndexOperation.DELETE);
- dualTuple.reset(tuple);
- lsmHarness.forceModify(ctx, dualTuple);
- }
-
- public MultiComparator getMultiComparator() {
- LSMRTreeOpContext concreteCtx = (LSMRTreeOpContext) ctx;
- return concreteCtx.getCurrentRTreeOpContext().getCmp();
- }
+ return new LSMRTreeAccessor(getLsmHarness(), createOpContext(modificationCallback, searchCallback),
+ buddyBTreeFields);
}
protected ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException {
@@ -465,9 +358,9 @@
}
@Override
- public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- return new LSMRTreeBulkLoader(fillLevel, verifyInput, numElementsHint, checkIfEmptyIndex);
+ public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
+ return new LSMRTreeBulkLoader(this, fillLevel, verifyInput, numElementsHint);
}
// This function is modified for R-Trees without antimatter tuples to allow buddy B-Tree to have only primary keys
@@ -503,114 +396,7 @@
}
}
}
- if (ctx.getFilterTuple() != null) {
- ctx.getFilterTuple().reset(tuple);
- memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter().update(ctx.getFilterTuple(),
- ctx.getFilterCmp());
- }
- }
-
- public class LSMRTreeBulkLoader implements IIndexBulkLoader {
- private final ILSMDiskComponent component;
- private final IIndexBulkLoader bulkLoader;
- private final IIndexBulkLoader buddyBTreeBulkloader;
- private boolean cleanedUpArtifacts = false;
- private boolean isEmptyComponent = true;
- public final PermutingTupleReference indexTuple;
- public final PermutingTupleReference filterTuple;
- public final MultiComparator filterCmp;
-
- public LSMRTreeBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- if (checkIfEmptyIndex && !isEmptyIndex()) {
- throw HyracksDataException.create(ErrorCode.LOAD_NON_EMPTY_INDEX);
- }
- // Note that by using a flush target file name, we state that the
- // new bulk loaded tree is "newer" than any other merged tree.
- component = createBulkLoadTarget();
- bulkLoader = ((LSMRTreeDiskComponent) component).getRTree().createBulkLoader(fillFactor, verifyInput,
- numElementsHint, false);
- buddyBTreeBulkloader = ((LSMRTreeDiskComponent) component).getBTree().createBulkLoader(fillFactor,
- verifyInput, numElementsHint, false);
- if (filterFields != null) {
- indexTuple = new PermutingTupleReference(rtreeFields);
- filterCmp = MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories());
- filterTuple = new PermutingTupleReference(filterFields);
- } else {
- indexTuple = null;
- filterCmp = null;
- filterTuple = null;
- }
- }
-
- @Override
- public void add(ITupleReference tuple) throws HyracksDataException {
- try {
- ITupleReference t;
- if (indexTuple != null) {
- indexTuple.reset(tuple);
- t = indexTuple;
- } else {
- t = tuple;
- }
-
- bulkLoader.add(t);
-
- if (filterTuple != null) {
- filterTuple.reset(tuple);
- component.getLSMComponentFilter().update(filterTuple, filterCmp);
- }
- } catch (Exception e) {
- cleanupArtifacts();
- throw e;
- }
- if (isEmptyComponent) {
- isEmptyComponent = false;
- }
- }
-
- @Override
- public void end() throws HyracksDataException {
- if (!cleanedUpArtifacts) {
-
- if (component.getLSMComponentFilter() != null) {
- filterManager.writeFilter(component.getLSMComponentFilter(),
- ((LSMRTreeDiskComponent) component).getRTree());
- }
-
- bulkLoader.end();
- buddyBTreeBulkloader.end();
-
- if (isEmptyComponent) {
- cleanupArtifacts();
- } else {
- ioOpCallback.afterOperation(LSMOperationType.FLUSH, null, component);
- lsmHarness.addBulkLoadedComponent(component);
- }
- }
- }
-
- @Override
- public void abort() throws HyracksDataException {
- if (bulkLoader != null) {
- bulkLoader.abort();
- }
- if (buddyBTreeBulkloader != null) {
- buddyBTreeBulkloader.abort();
- }
- }
-
- protected void cleanupArtifacts() throws HyracksDataException {
- if (!cleanedUpArtifacts) {
- cleanedUpArtifacts = true;
- ((LSMRTreeDiskComponent) component).getRTree().deactivate();
- ((LSMRTreeDiskComponent) component).getRTree().destroy();
- ((LSMRTreeDiskComponent) component).getBTree().deactivate();
- ((LSMRTreeDiskComponent) component).getBTree().destroy();
- ((LSMRTreeDiskComponent) component).getBloomFilter().deactivate();
- ((LSMRTreeDiskComponent) component).getBloomFilter().destroy();
- }
- }
+ updateFilter(ctx, tuple);
}
@Override
@@ -631,4 +417,24 @@
return files;
}
+ @Override
+ protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx,
+ ILSMMemoryComponent flushingComponent, LSMComponentFileReferences componentFileRefs,
+ ILSMIOOperationCallback callback) throws HyracksDataException {
+ LSMRTreeAccessor accessor = new LSMRTreeAccessor(getLsmHarness(), opCtx, buddyBTreeFields);
+ return new LSMRTreeFlushOperation(accessor, flushingComponent, componentFileRefs.getInsertIndexFileReference(),
+ componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(),
+ callback, fileManager.getBaseDir());
+ }
+
+ @Override
+ protected ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx,
+ List<ILSMComponent> mergingComponents, LSMComponentFileReferences mergeFileRefs,
+ ILSMIOOperationCallback callback) throws HyracksDataException {
+ ITreeIndexCursor cursor = new LSMRTreeSortedCursor(opCtx, linearizer, buddyBTreeFields);
+ ILSMIndexAccessor accessor = new LSMRTreeAccessor(getLsmHarness(), opCtx, buddyBTreeFields);
+ return new LSMRTreeMergeOperation(accessor, mergingComponents, cursor,
+ mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getDeleteIndexFileReference(),
+ mergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir());
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
index b348020..edb7481 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
@@ -44,7 +44,6 @@
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
public abstract class LSMRTreeAbstractCursor implements ITreeIndexCursor {
@@ -154,12 +153,6 @@
}
@Override
- public ICachedPage getPage() {
- // do nothing
- return null;
- }
-
- @Override
public void close() throws HyracksDataException {
if (!open) {
return;
@@ -198,12 +191,8 @@
}
@Override
- public boolean exclusiveLatchNodes() {
+ public boolean isExclusiveLatchNodes() {
return false;
}
- @Override
- public void markCurrentTupleAsUpdated() throws HyracksDataException {
- throw new HyracksDataException("Updating tuples is not supported with this cursor.");
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAccessor.java
new file mode 100644
index 0000000..08748e2
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAccessor.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.rtree.impls;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.common.tuples.DualTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
+
+public class LSMRTreeAccessor extends LSMTreeIndexAccessor {
+ private final DualTupleReference dualTuple;
+
+ public LSMRTreeAccessor(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx, int[] buddyBTreeFields) {
+ super(lsmHarness, ctx, aCtx -> new LSMRTreeSearchCursor(aCtx, buddyBTreeFields));
+ dualTuple = new DualTupleReference(buddyBTreeFields);
+ }
+
+ @Override
+ public void delete(ITupleReference tuple) throws HyracksDataException {
+ getCtx().setOperation(IndexOperation.DELETE);
+ dualTuple.reset(tuple);
+ lsmHarness.modify(getCtx(), false, dualTuple);
+ }
+
+ @Override
+ public boolean tryDelete(ITupleReference tuple) throws HyracksDataException {
+ getCtx().setOperation(IndexOperation.DELETE);
+ dualTuple.reset(tuple);
+ return lsmHarness.modify(getCtx(), true, dualTuple);
+ }
+
+ @Override
+ public void forceDelete(ITupleReference tuple) throws HyracksDataException {
+ getCtx().setOperation(IndexOperation.DELETE);
+ dualTuple.reset(tuple);
+ lsmHarness.forceModify(getCtx(), dualTuple);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java
new file mode 100644
index 0000000..edc3e7d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.rtree.impls;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.apache.hyracks.storage.common.IIndexBulkLoader;
+import org.apache.hyracks.storage.common.MultiComparator;
+
+public class LSMRTreeBulkLoader implements IIndexBulkLoader {
+ private final ILSMDiskComponent component;
+ private final IIndexBulkLoader bulkLoader;
+ private final IIndexBulkLoader buddyBTreeBulkloader;
+ private boolean cleanedUpArtifacts = false;
+ private boolean isEmptyComponent = true;
+ public final PermutingTupleReference indexTuple;
+ public final PermutingTupleReference filterTuple;
+ public final MultiComparator filterCmp;
+ private final LSMRTree lsmIndex;
+
+ public LSMRTreeBulkLoader(LSMRTree lsmIndex, float fillFactor, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
+ this.lsmIndex = lsmIndex;
+ // Note that by using a flush target file name, we state that the
+ // new bulk loaded tree is "newer" than any other merged tree.
+ component = lsmIndex.createBulkLoadTarget();
+ bulkLoader = ((LSMRTreeDiskComponent) component).getRTree().createBulkLoader(fillFactor, verifyInput,
+ numElementsHint, false);
+ buddyBTreeBulkloader = ((LSMRTreeDiskComponent) component).getBTree().createBulkLoader(fillFactor, verifyInput,
+ numElementsHint, false);
+ if (lsmIndex.getFilterFields() != null) {
+ indexTuple = new PermutingTupleReference(lsmIndex.getTreeFields());
+ filterCmp = MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories());
+ filterTuple = new PermutingTupleReference(lsmIndex.getFilterFields());
+ } else {
+ indexTuple = null;
+ filterCmp = null;
+ filterTuple = null;
+ }
+ }
+
+ @Override
+ public void add(ITupleReference tuple) throws HyracksDataException {
+ try {
+ ITupleReference t;
+ if (indexTuple != null) {
+ indexTuple.reset(tuple);
+ t = indexTuple;
+ } else {
+ t = tuple;
+ }
+
+ bulkLoader.add(t);
+
+ if (filterTuple != null) {
+ filterTuple.reset(tuple);
+ component.getLSMComponentFilter().update(filterTuple, filterCmp);
+ }
+ } catch (Exception e) {
+ cleanupArtifacts();
+ throw e;
+ }
+ if (isEmptyComponent) {
+ isEmptyComponent = false;
+ }
+ }
+
+ @Override
+ public void end() throws HyracksDataException {
+ if (!cleanedUpArtifacts) {
+
+ if (component.getLSMComponentFilter() != null) {
+ lsmIndex.getFilterManager().writeFilter(component.getLSMComponentFilter(),
+ ((LSMRTreeDiskComponent) component).getRTree());
+ }
+
+ bulkLoader.end();
+ buddyBTreeBulkloader.end();
+
+ if (isEmptyComponent) {
+ cleanupArtifacts();
+ } else {
+ lsmIndex.getIOOperationCallback().afterOperation(LSMOperationType.FLUSH, null, component);
+ lsmIndex.getLsmHarness().addBulkLoadedComponent(component);
+ }
+ }
+ }
+
+ @Override
+ public void abort() throws HyracksDataException {
+ if (bulkLoader != null) {
+ bulkLoader.abort();
+ }
+ if (buddyBTreeBulkloader != null) {
+ buddyBTreeBulkloader.abort();
+ }
+ }
+
+ protected void cleanupArtifacts() throws HyracksDataException {
+ if (!cleanedUpArtifacts) {
+ cleanedUpArtifacts = true;
+ ((LSMRTreeDiskComponent) component).getRTree().deactivate();
+ ((LSMRTreeDiskComponent) component).getRTree().destroy();
+ ((LSMRTreeDiskComponent) component).getBTree().deactivate();
+ ((LSMRTreeDiskComponent) component).getBTree().destroy();
+ ((LSMRTreeDiskComponent) component).getBloomFilter().deactivate();
+ ((LSMRTreeDiskComponent) component).getBloomFilter().destroy();
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
index 618b3a7..f3e45ac 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
@@ -18,95 +18,30 @@
*/
package org.apache.hyracks.storage.am.lsm.rtree.impls;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IODeviceHandle;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.FlushOperation;
-public class LSMRTreeFlushOperation implements ILSMIOOperation, Comparable<LSMRTreeFlushOperation> {
+public class LSMRTreeFlushOperation extends FlushOperation {
- private final ILSMIndexAccessor accessor;
- private final ILSMComponent flushingComponent;
- private final FileReference rtreeFlushTarget;
private final FileReference btreeFlushTarget;
private final FileReference bloomFilterFlushTarget;
- private final ILSMIOOperationCallback callback;
- private final String indexIdentifier;
- public LSMRTreeFlushOperation(ILSMIndexAccessor accessor, ILSMComponent flushingComponent,
- FileReference rtreeFlushTarget, FileReference btreeFlushTarget, FileReference bloomFilterFlushTarget,
+ public LSMRTreeFlushOperation(ILSMIndexAccessor accessor, ILSMMemoryComponent flushingComponent,
+ FileReference flushTarget, FileReference btreeFlushTarget, FileReference bloomFilterFlushTarget,
ILSMIOOperationCallback callback, String indexIdentifier) {
- this.accessor = accessor;
- this.flushingComponent = flushingComponent;
- this.rtreeFlushTarget = rtreeFlushTarget;
+ super(accessor, flushingComponent, flushTarget, callback, indexIdentifier);
this.btreeFlushTarget = btreeFlushTarget;
this.bloomFilterFlushTarget = bloomFilterFlushTarget;
- this.callback = callback;
- this.indexIdentifier = indexIdentifier;
}
- @Override
- public Set<IODeviceHandle> getReadDevices() {
- return Collections.emptySet();
- }
-
- @Override
- public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<>();
- devs.add(rtreeFlushTarget.getDeviceHandle());
- if (btreeFlushTarget != null) {
- devs.add(btreeFlushTarget.getDeviceHandle());
- devs.add(bloomFilterFlushTarget.getDeviceHandle());
- }
- return devs;
- }
-
- @Override
- public Boolean call() throws HyracksDataException {
- accessor.flush(this);
- return true;
- }
-
- @Override
- public ILSMIOOperationCallback getCallback() {
- return callback;
- }
-
- public FileReference getRTreeFlushTarget() {
- return rtreeFlushTarget;
- }
-
- public FileReference getBTreeFlushTarget() {
+ public FileReference getBTreeTarget() {
return btreeFlushTarget;
}
- public FileReference getBloomFilterFlushTarget() {
+ public FileReference getBloomFilterTarget() {
return bloomFilterFlushTarget;
}
-
- public ILSMComponent getFlushingComponent() {
- return flushingComponent;
- }
-
- @Override
- public String getIndexUniqueIdentifier() {
- return indexIdentifier;
- }
-
- @Override
- public LSMIOOpertionType getIOOpertionType() {
- return LSMIOOpertionType.FLUSH;
- }
-
- @Override
- public int compareTo(LSMRTreeFlushOperation o) {
- return rtreeFlushTarget.getFile().getName().compareTo(o.getRTreeFlushTarget().getFile().getName());
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java
index 847533f..9b3aa0c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java
@@ -18,114 +18,32 @@
*/
package org.apache.hyracks.storage.am.lsm.rtree.impls;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation;
-public class LSMRTreeMergeOperation implements ILSMIOOperation {
- private final ILSMIndexAccessor accessor;
- private final List<ILSMComponent> mergingComponents;
- private final ITreeIndexCursor cursor;
- private final FileReference rtreeMergeTarget;
+public class LSMRTreeMergeOperation extends MergeOperation {
private final FileReference btreeMergeTarget;
private final FileReference bloomFilterMergeTarget;
- private final ILSMIOOperationCallback callback;
- private final String indexIdentifier;
- private boolean keepDeletedTuples;
public LSMRTreeMergeOperation(ILSMIndexAccessor accessor, List<ILSMComponent> mergingComponents,
- ITreeIndexCursor cursor, FileReference rtreeMergeTarget, FileReference btreeMergeTarget,
+ ITreeIndexCursor cursor, FileReference target, FileReference btreeMergeTarget,
FileReference bloomFilterMergeTarget, ILSMIOOperationCallback callback, String indexIdentifier) {
- this.accessor = accessor;
- this.mergingComponents = mergingComponents;
- this.cursor = cursor;
- this.rtreeMergeTarget = rtreeMergeTarget;
+ super(accessor, target, callback, indexIdentifier, mergingComponents, cursor);
this.btreeMergeTarget = btreeMergeTarget;
this.bloomFilterMergeTarget = bloomFilterMergeTarget;
- this.callback = callback;
- this.indexIdentifier = indexIdentifier;
}
- @Override
- public Set<IODeviceHandle> getReadDevices() {
- Set<IODeviceHandle> devs = new HashSet<>();
- for (ILSMComponent o : mergingComponents) {
- LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) o;
- devs.add(component.getRTree().getFileReference().getDeviceHandle());
- if (component.getBTree() != null) {
- devs.add(component.getBTree().getFileReference().getDeviceHandle());
- devs.add(component.getBloomFilter().getFileReference().getDeviceHandle());
- }
- }
- return devs;
- }
-
- @Override
- public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<>();
- devs.add(rtreeMergeTarget.getDeviceHandle());
- if (btreeMergeTarget != null) {
- devs.add(btreeMergeTarget.getDeviceHandle());
- devs.add(bloomFilterMergeTarget.getDeviceHandle());
- }
- return devs;
- }
-
- @Override
- public Boolean call() throws HyracksDataException {
- accessor.merge(this);
- return true;
- }
-
- @Override
- public ILSMIOOperationCallback getCallback() {
- return callback;
- }
-
- public FileReference getRTreeMergeTarget() {
- return rtreeMergeTarget;
- }
-
- public FileReference getBTreeMergeTarget() {
+ public FileReference getBTreeTarget() {
return btreeMergeTarget;
}
- public FileReference getBloomFilterMergeTarget() {
+ public FileReference getBloomFilterTarget() {
return bloomFilterMergeTarget;
}
-
- public ITreeIndexCursor getCursor() {
- return cursor;
- }
-
- public List<ILSMComponent> getMergingComponents() {
- return mergingComponents;
- }
-
- @Override
- public String getIndexUniqueIdentifier() {
- return indexIdentifier;
- }
-
- @Override
- public LSMIOOpertionType getIOOpertionType() {
- return LSMIOOpertionType.MERGE;
- }
-
- public boolean isKeepDeletedTuples() {
- return keepDeletedTuples;
- }
-
- public void setKeepDeletedTuples(boolean keepDeletedTuples) {
- this.keepDeletedTuples = keepDeletedTuples;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
index 83ca78c..a66fa62 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.lsm.rtree.impls;
-import java.util.LinkedList;
import java.util.List;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -28,9 +27,6 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
@@ -47,36 +43,22 @@
private RTree.RTreeAccessor currentMutableRTreeAccessor;
private BTree.BTreeAccessor[] mutableBTreeAccessors;
private BTree.BTreeAccessor currentMutableBTreeAccessor;
-
private RTreeOpContext[] rtreeOpContexts;
private BTreeOpContext[] btreeOpContexts;
private RTreeOpContext currentRTreeOpContext;
private BTreeOpContext currentBTreeOpContext;
-
- private IndexOperation op;
- private final List<ILSMComponent> componentHolder;
- private final List<ILSMDiskComponent> componentsToBeMerged;
- private final List<ILSMDiskComponent> componentsToBeReplicated;
- private IModificationOperationCallback modificationCallback;
- private ISearchOperationCallback searchCallback;
- private final PermutingTupleReference indexTuple;
- private final MultiComparator filterCmp;
- private final PermutingTupleReference filterTuple;
- private ISearchPredicate searchPredicate;
private LSMRTreeCursorInitialState searchInitialState;
public LSMRTreeOpContext(List<ILSMMemoryComponent> mutableComponents, ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback,
int[] rtreeFields, int[] filterFields, ILSMHarness lsmHarness, int[] comparatorFields,
- IBinaryComparatorFactory[] linearizerArray) {
+ IBinaryComparatorFactory[] linearizerArray, IBinaryComparatorFactory[] filterComparatorFactories) {
+ super(rtreeFields, filterFields, filterComparatorFactories, searchCallback, modificationCallback);
mutableRTreeAccessors = new RTree.RTreeAccessor[mutableComponents.size()];
mutableBTreeAccessors = new BTree.BTreeAccessor[mutableComponents.size()];
rtreeOpContexts = new RTreeOpContext[mutableComponents.size()];
btreeOpContexts = new BTreeOpContext[mutableComponents.size()];
-
- LSMRTreeMemoryComponent c = (LSMRTreeMemoryComponent) mutableComponents.get(0);
-
for (int i = 0; i < mutableComponents.size(); i++) {
LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) mutableComponents.get(i);
mutableRTreeAccessors[i] = (RTree.RTreeAccessor) mutableComponent.getRTree()
@@ -87,37 +69,14 @@
rtreeOpContexts[i] = mutableRTreeAccessors[i].getOpContext();
btreeOpContexts[i] = mutableBTreeAccessors[i].getOpContext();
}
-
- assert mutableComponents.size() > 0;
currentRTreeOpContext = rtreeOpContexts[0];
currentBTreeOpContext = btreeOpContexts[0];
- this.componentHolder = new LinkedList<>();
- this.componentsToBeMerged = new LinkedList<>();
- this.componentsToBeReplicated = new LinkedList<>();
- this.modificationCallback = modificationCallback;
- this.searchCallback = searchCallback;
-
- if (filterFields != null) {
- indexTuple = new PermutingTupleReference(rtreeFields);
- filterCmp = MultiComparator.create(c.getLSMComponentFilter().getFilterCmpFactories());
- filterTuple = new PermutingTupleReference(filterFields);
- } else {
- indexTuple = null;
- filterCmp = null;
- filterTuple = null;
- }
searchInitialState = new LSMRTreeCursorInitialState(rtreeLeafFrameFactory, rtreeInteriorFrameFactory,
btreeLeafFrameFactory, getBTreeMultiComparator(), lsmHarness, comparatorFields, linearizerArray,
searchCallback, componentHolder);
}
@Override
- public void setOperation(IndexOperation newOp) {
- reset();
- this.op = newOp;
- }
-
- @Override
public void setCurrentMutableComponentId(int currentMutableComponentId) {
currentMutableRTreeAccessor = mutableRTreeAccessors[currentMutableComponentId];
currentMutableBTreeAccessor = mutableBTreeAccessors[currentMutableComponentId];
@@ -130,43 +89,11 @@
}
}
- @Override
- public void reset() {
- super.reset();
- componentHolder.clear();
- componentsToBeMerged.clear();
- }
-
- @Override
- public IndexOperation getOperation() {
- return op;
- }
-
public MultiComparator getBTreeMultiComparator() {
return currentBTreeOpContext.getCmp();
}
@Override
- public List<ILSMComponent> getComponentHolder() {
- return componentHolder;
- }
-
- @Override
- public ISearchOperationCallback getSearchOperationCallback() {
- return searchCallback;
- }
-
- @Override
- public IModificationOperationCallback getModificationCallback() {
- return modificationCallback;
- }
-
- @Override
- public List<ILSMDiskComponent> getComponentsToBeMerged() {
- return componentsToBeMerged;
- }
-
- @Override
public void setSearchPredicate(ISearchPredicate searchPredicate) {
this.searchPredicate = searchPredicate;
}
@@ -176,23 +103,10 @@
return searchPredicate;
}
- @Override
- public List<ILSMDiskComponent> getComponentsToBeReplicated() {
- return componentsToBeReplicated;
- }
-
- public MultiComparator getFilterCmp() {
- return filterCmp;
- }
-
public LSMRTreeCursorInitialState getSearchInitialState() {
return searchInitialState;
}
- public PermutingTupleReference getIndexTuple() {
- return indexTuple;
- }
-
public RTree.RTreeAccessor getCurrentMutableRTreeAccessor() {
return currentMutableRTreeAccessor;
}
@@ -201,11 +115,7 @@
return currentMutableBTreeAccessor;
}
- public PermutingTupleReference getFilterTuple() {
- return filterTuple;
- }
-
public RTreeOpContext getCurrentRTreeOpContext() {
return currentRTreeOpContext;
}
-}
+}
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index 018e6d3a..bb85696 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -26,7 +26,6 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
-import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
@@ -36,33 +35,36 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
-import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor.ICursorFactory;
+import org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.am.rtree.impls.RTree;
import org.apache.hyracks.storage.am.rtree.impls.RTreeSearchCursor;
import org.apache.hyracks.storage.am.rtree.impls.SearchPredicate;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
+import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
@@ -70,6 +72,7 @@
import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
+ private static final ICursorFactory cursorFactory = opCtx -> new LSMRTreeWithAntiMatterTuplesSearchCursor(opCtx);
// On-disk components.
// For creating RTree's used in bulk load. Different from diskRTreeFactory
// because it should have a different tuple writer in it's leaf frames.
@@ -98,71 +101,26 @@
}
@Override
- public synchronized void activate() throws HyracksDataException {
- super.activate();
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- immutableComponents.clear();
- List<LSMComponentFileReferences> validFileReferences;
- validFileReferences = fileManager.cleanupAndGetValidFiles();
- for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
- LSMRTreeDiskComponent component;
- component = createDiskComponent(componentFactory, lsmComonentFileReference.getInsertIndexFileReference(),
- null, null, false);
- immutableComponents.add(component);
- }
- isActivated = true;
+ protected ILSMDiskComponent loadComponent(LSMComponentFileReferences refs) throws HyracksDataException {
+ return createDiskComponent(componentFactory, refs.getInsertIndexFileReference(), null, null, false);
}
@Override
- public synchronized void deactivate(boolean flushOnExit) throws HyracksDataException {
- super.deactivate(flushOnExit);
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- RTree rtree = ((LSMRTreeDiskComponent) c).getRTree();
- rtree.deactivateCloseHandle();
- }
- isActivated = false;
+ protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ RTree rtree = ((LSMRTreeDiskComponent) c).getRTree();
+ rtree.deactivateCloseHandle();
}
@Override
- public synchronized void deactivate() throws HyracksDataException {
- deactivate(true);
+ protected void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ ((LSMRTreeDiskComponent) c).getRTree().destroy();
}
@Override
- public synchronized void destroy() throws HyracksDataException {
- super.destroy();
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- RTree rtree = ((LSMRTreeDiskComponent) c).getRTree();
- rtree.destroy();
- }
- fileManager.deleteDirs();
- }
-
- @Override
- public synchronized void clear() throws HyracksDataException {
- super.clear();
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- RTree rtree = ((LSMRTreeDiskComponent) c).getRTree();
- rtree.deactivate();
- rtree.destroy();
- }
- immutableComponents.clear();
- }
-
- @Override
- public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
- throws HyracksDataException {
- ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
- LSMRTreeOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference();
- opCtx.setOperation(IndexOperation.FLUSH);
- opCtx.getComponentHolder().add(flushingComponent);
- ILSMIndexAccessor accessor = new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness, opCtx);
- ioScheduler.scheduleOperation(new LSMRTreeFlushOperation(accessor, flushingComponent,
- relFlushFileRefs.getInsertIndexFileReference(), null, null, callback, fileManager.getBaseDir()));
+ protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ RTree rtree = ((LSMRTreeDiskComponent) c).getRTree();
+ rtree.deactivate();
+ rtree.destroy();
}
@Override
@@ -177,8 +135,7 @@
RTreeSearchCursor rtreeScanCursor = (RTreeSearchCursor) memRTreeAccessor.createSearchCursor(false);
SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
- LSMRTreeDiskComponent component =
- createDiskComponent(componentFactory, flushOp.getRTreeFlushTarget(), null, null, true);
+ LSMRTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), null, null, true);
RTree diskRTree = component.getRTree();
// scan the memory BTree
@@ -248,8 +205,8 @@
List<ITupleReference> filterTuples = new ArrayList<>();
filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
- filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilter(component.getLSMComponentFilter(), component.getRTree());
+ getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
+ getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getRTree());
}
flushingComponent.getMetadata().copy(component.getMetadata());
rTreeBulkloader.end();
@@ -258,35 +215,16 @@
}
@Override
- public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
- throws HyracksDataException {
- LSMRTreeOpContext rctx = createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- rctx.setOperation(IndexOperation.MERGE);
- List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
- boolean returnDeletedTuples = false;
- if (ctx.getComponentHolder().get(ctx.getComponentHolder().size() - 1) != diskComponents
- .get(diskComponents.size() - 1)) {
- returnDeletedTuples = true;
- }
- ITreeIndexCursor cursor = new LSMRTreeWithAntiMatterTuplesSearchCursor(rctx, returnDeletedTuples);
- LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
- ILSMIndexAccessor accessor = new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness, rctx);
- ioScheduler.scheduleOperation(new LSMRTreeMergeOperation(accessor, mergingComponents, cursor,
- relMergeFileRefs.getInsertIndexFileReference(), null, null, callback, fileManager.getBaseDir()));
- }
-
- @Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
- LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation;
- ITreeIndexCursor cursor = mergeOp.getCursor();
+ MergeOperation mergeOp = (MergeOperation) operation;
+ IIndexCursor cursor = mergeOp.getCursor();
ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
search(opCtx, cursor, rtreeSearchPred);
// Bulk load the tuples from all on-disk RTrees into the new RTree.
- LSMRTreeDiskComponent component =
- createDiskComponent(componentFactory, mergeOp.getRTreeMergeTarget(), null, null, true);
+ LSMRTreeDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(), null, null, true);
RTree mergedRTree = component.getRTree();
IIndexBulkLoader bulkloader = mergedRTree.createBulkLoader(1.0f, false, 0L, false);
try {
@@ -304,8 +242,8 @@
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
}
- filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilter(component.getLSMComponentFilter(), component.getRTree());
+ getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
+ getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getRTree());
}
bulkloader.end();
@@ -315,30 +253,14 @@
@Override
public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
- return new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness,
- createOpContext(modificationCallback, searchCallback));
- }
-
- public class LSMRTreeWithAntiMatterTuplesAccessor extends LSMTreeIndexAccessor {
- public LSMRTreeWithAntiMatterTuplesAccessor(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) {
- super(lsmHarness, ctx);
- }
-
- @Override
- public ITreeIndexCursor createSearchCursor(boolean exclusive) {
- return new LSMRTreeWithAntiMatterTuplesSearchCursor(ctx);
- }
-
- public MultiComparator getMultiComparator() {
- LSMRTreeOpContext concreteCtx = (LSMRTreeOpContext) ctx;
- return concreteCtx.getCurrentRTreeOpContext().getCmp();
- }
+ LSMRTreeOpContext opCtx = createOpContext(modificationCallback, searchCallback);
+ return new LSMTreeIndexAccessor(getLsmHarness(), opCtx, cursorFactory);
}
@Override
- public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- return new LSMRTreeWithAntiMatterTuplesBulkLoader(fillLevel, verifyInput, numElementsHint, checkIfEmptyIndex);
+ public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
+ return new LSMRTreeWithAntiMatterTuplesBulkLoader(fillLevel, verifyInput, numElementsHint);
}
public class LSMRTreeWithAntiMatterTuplesBulkLoader implements IIndexBulkLoader {
@@ -350,11 +272,8 @@
public final PermutingTupleReference filterTuple;
public final MultiComparator filterCmp;
- public LSMRTreeWithAntiMatterTuplesBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- if (checkIfEmptyIndex && !isEmptyIndex()) {
- throw HyracksDataException.create(ErrorCode.LOAD_NON_EMPTY_INDEX);
- }
+ public LSMRTreeWithAntiMatterTuplesBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
// Note that by using a flush target file name, we state that the
// new bulk loaded tree is "newer" than any other merged tree.
@@ -362,10 +281,10 @@
bulkLoader = ((LSMRTreeDiskComponent) component).getRTree().createBulkLoader(fillFactor, verifyInput,
numElementsHint, false);
- if (filterFields != null) {
- indexTuple = new PermutingTupleReference(rtreeFields);
+ if (getFilterFields() != null) {
+ indexTuple = new PermutingTupleReference(getTreeFields());
filterCmp = MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories());
- filterTuple = new PermutingTupleReference(filterFields);
+ filterTuple = new PermutingTupleReference(getFilterFields());
} else {
indexTuple = null;
filterCmp = null;
@@ -405,7 +324,7 @@
if (!cleanedUpArtifacts) {
if (component.getLSMComponentFilter() != null) {
- filterManager.writeFilter(component.getLSMComponentFilter(),
+ getFilterManager().writeFilter(component.getLSMComponentFilter(),
((LSMRTreeDiskComponent) component).getRTree());
}
bulkLoader.end();
@@ -414,7 +333,7 @@
cleanupArtifacts();
} else {
ioOpCallback.afterOperation(LSMOperationType.FLUSH, null, component);
- lsmHarness.addBulkLoadedComponent(component);
+ getLsmHarness().addBulkLoadedComponent(component);
}
}
}
@@ -455,4 +374,27 @@
files.add(rtree.getFileReference().getFile().getAbsolutePath());
return files;
}
+
+ @Override
+ protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx,
+ ILSMMemoryComponent flushingComponent, LSMComponentFileReferences componentFileRefs,
+ ILSMIOOperationCallback callback) throws HyracksDataException {
+ ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getLsmHarness(), opCtx, cursorFactory);
+ return new LSMRTreeFlushOperation(accessor, flushingComponent, componentFileRefs.getInsertIndexFileReference(),
+ null, null, callback, fileManager.getBaseDir());
+ }
+
+ @Override
+ protected ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx,
+ List<ILSMComponent> mergingComponents, LSMComponentFileReferences mergeFileRefs,
+ ILSMIOOperationCallback callback) throws HyracksDataException {
+ boolean returnDeletedTuples = false;
+ if (mergingComponents.get(mergingComponents.size() - 1) != diskComponents.get(diskComponents.size() - 1)) {
+ returnDeletedTuples = true;
+ }
+ ITreeIndexCursor cursor = new LSMRTreeWithAntiMatterTuplesSearchCursor(opCtx, returnDeletedTuples);
+ ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getLsmHarness(), opCtx, cursorFactory);
+ return new MergeOperation(accessor, mergeFileRefs.getInsertIndexFileReference(), callback,
+ fileManager.getBaseDir(), mergingComponents, cursor);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java
index cc34a89..af9fef6 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java
@@ -27,7 +27,6 @@
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
public class LSMRTreeWithAntiMatterTuplesFlushCursor implements ITreeIndexCursor {
private final TreeTupleSorter rTreeTupleSorter;
@@ -155,11 +154,6 @@
}
@Override
- public ICachedPage getPage() {
- return null;
- }
-
- @Override
public void setBufferCache(IBufferCache bufferCache) {
}
@@ -170,12 +164,7 @@
}
@Override
- public boolean exclusiveLatchNodes() {
+ public boolean isExclusiveLatchNodes() {
return false;
}
-
- @Override
- public void markCurrentTupleAsUpdated() throws HyracksDataException {
- throw new HyracksDataException("Updating tuples is not supported with this cursor.");
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/TreeTupleSorter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/TreeTupleSorter.java
index 724ddcf..cad1f74 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/TreeTupleSorter.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/TreeTupleSorter.java
@@ -75,8 +75,8 @@
// We don't latch pages since this code is only used by flush () before
// bulk-loading the r-tree to disk and flush is not concurrent.
//
- ICachedPage node1 = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, tPointers[currentTupleIndex * 2]),
- false);
+ ICachedPage node1 =
+ bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, tPointers[currentTupleIndex * 2]), false);
try {
leafFrame1.setPage(node1);
frameTuple1.resetByTupleOffset(leafFrame1.getBuffer().array(), tPointers[currentTupleIndex * 2 + 1]);
@@ -222,11 +222,6 @@
}
@Override
- public ICachedPage getPage() {
- return null;
- }
-
- @Override
public void setBufferCache(IBufferCache bufferCache) {
// do nothing
}
@@ -237,12 +232,7 @@
}
@Override
- public boolean exclusiveLatchNodes() {
+ public boolean isExclusiveLatchNodes() {
return false;
}
-
- @Override
- public void markCurrentTupleAsUpdated() throws HyracksDataException {
- throw new HyracksDataException("Updating tuples is not supported with this cursor.");
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
index bb3d2f5..4302c61 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
@@ -97,11 +97,6 @@
return pageId;
}
- @Override
- public ICachedPage getPage() {
- return page;
- }
-
protected boolean fetchNextLeafPage() throws HyracksDataException {
boolean succeeded = false;
if (readLatched) {
@@ -114,8 +109,12 @@
int pageId = pathList.getLastPageId();
long parentLsn = pathList.getLastPageLsn();
pathList.moveLast();
- if(pageId <0) throw new IllegalStateException();
- if(fileId<0) throw new IllegalStateException();
+ if (pageId < 0) {
+ throw new IllegalStateException();
+ }
+ if (fileId < 0) {
+ throw new IllegalStateException();
+ }
ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
node.acquireReadLatch();
readLatched = true;
@@ -265,12 +264,7 @@
}
@Override
- public boolean exclusiveLatchNodes() {
+ public boolean isExclusiveLatchNodes() {
return false;
}
-
- @Override
- public void markCurrentTupleAsUpdated() throws HyracksDataException {
- throw new HyracksDataException("Updating tuples is not supported with this cursor.");
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndex.java
index b23a345..255e0d0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndex.java
@@ -101,18 +101,20 @@
ISearchOperationCallback searchCallback) throws HyracksDataException;
/**
+ * Strictly a test method
+ *
* Ensures that all pages (and tuples) of the index are logically consistent.
* An assertion error is thrown if validation fails.
*
* @throws HyracksDataException
* if there is an error performing validation
*/
- public void validate() throws HyracksDataException;
+ void validate() throws HyracksDataException;
/**
* @return the {@link IBufferCache} underlying this index.
*/
- public IBufferCache getBufferCache();
+ IBufferCache getBufferCache();
/**
* @return the size, in bytes, of pre-allocated memory space that this index was allotted.
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeTestWorker.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeTestWorker.java
index 4848711..024b72d 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeTestWorker.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeTestWorker.java
@@ -30,7 +30,8 @@
import org.apache.hyracks.storage.am.common.TestOperationSelector.TestOperation;
import org.apache.hyracks.storage.am.common.datagen.DataGenThread;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree;
-import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree.LSMBTreeAccessor;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeOpContext;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallbackFactory;
import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.common.IIndexCursor;
@@ -52,9 +53,10 @@
@Override
public void performOp(ITupleReference tuple, TestOperation op) throws HyracksDataException {
- LSMBTreeAccessor accessor = (LSMBTreeAccessor) indexAccessor;
+ LSMTreeIndexAccessor accessor = (LSMTreeIndexAccessor) indexAccessor;
IIndexCursor searchCursor = accessor.createSearchCursor(false);
- MultiComparator cmp = accessor.getMultiComparator();
+ LSMBTreeOpContext concreteCtx = (LSMBTreeOpContext) accessor.getCtx();
+ MultiComparator cmp = concreteCtx.getCmp();
RangePredicate rangePred = new RangePredicate(tuple, tuple, true, true, cmp, cmp);
switch (op) {
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyTreeFactory.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyTreeFactory.java
deleted file mode 100644
index dafd6b9..0000000
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyTreeFactory.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.lsm.common;
-
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.storage.am.common.api.ITreeIndex;
-import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
-
-public class DummyTreeFactory extends TreeIndexFactory<ITreeIndex> {
-
- public DummyTreeFactory() {
- super(null, null, null, null, null, null, null, 0);
- }
-
- @Override
- public ITreeIndex createIndexInstance(FileReference file) {
- return null;
- }
-
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
similarity index 83%
rename from hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyLSMIndexFileManager.java
rename to hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
index e03f765..9d8f208 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyLSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.hyracks.storage.am.lsm.common;
+package org.apache.hyracks.storage.am.lsm.common.component;
import java.io.File;
import java.io.FilenameFilter;
@@ -31,11 +31,10 @@
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
-public class DummyLSMIndexFileManager extends AbstractLSMIndexFileManager {
+public class TestLsmIndexFileManager extends AbstractLSMIndexFileManager {
- public DummyLSMIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
- TreeIndexFactory<? extends ITreeIndex> treeFactory) {
- super(ioManager, fileMapProvider, file, treeFactory);
+ public TestLsmIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file) {
+ super(ioManager, fileMapProvider, file, null);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMComponentFilterReferenceTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMComponentFilterReferenceTest.java
similarity index 98%
rename from hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMComponentFilterReferenceTest.java
rename to hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMComponentFilterReferenceTest.java
index 0257bad..af37d80 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMComponentFilterReferenceTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMComponentFilterReferenceTest.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.storage.am.lsm.common;
+package org.apache.hyracks.storage.am.lsm.common.test;
import java.nio.ByteBuffer;
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMIndexFileManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMIndexFileManagerTest.java
similarity index 96%
rename from hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMIndexFileManagerTest.java
rename to hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMIndexFileManagerTest.java
index 70e3ed6..67a6f69 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMIndexFileManagerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMIndexFileManagerTest.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.hyracks.storage.am.lsm.common;
+package org.apache.hyracks.storage.am.lsm.common.test;
import static org.junit.Assert.assertEquals;
@@ -39,6 +39,7 @@
import org.apache.hyracks.control.nc.io.DefaultDeviceResolver;
import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
+import org.apache.hyracks.storage.am.lsm.common.component.TestLsmIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
@@ -77,8 +78,7 @@
}
public void sortOrderTest(boolean testFlushFileName) throws InterruptedException, HyracksDataException {
- ILSMIndexFileManager fileManager =
- new DummyLSMIndexFileManager(ioManager, fileMapProvider, file, new DummyTreeFactory());
+ ILSMIndexFileManager fileManager = new TestLsmIndexFileManager(ioManager, fileMapProvider, file);
LinkedList<String> fileNames = new LinkedList<>();
int numFileNames = 100;
@@ -123,8 +123,7 @@
File f = new File(dirPath);
f.mkdirs();
FileReference file = ioManager.resolveAbsolutePath(f.getAbsolutePath());
- ILSMIndexFileManager fileManager =
- new DummyLSMIndexFileManager(ioManager, fileMapProvider, file, new DummyTreeFactory());
+ ILSMIndexFileManager fileManager = new TestLsmIndexFileManager(ioManager, fileMapProvider, file);
fileManager.createDirs();
List<FileReference> flushFiles = new ArrayList<>();
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualBufferCacheTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualBufferCacheTest.java
similarity index 98%
rename from hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualBufferCacheTest.java
rename to hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualBufferCacheTest.java
index b0d0ecb..59afa19 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualBufferCacheTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualBufferCacheTest.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.storage.am.lsm.common;
+package org.apache.hyracks.storage.am.lsm.common.test;
import static org.junit.Assert.assertTrue;
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualFreePageManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualFreePageManagerTest.java
similarity index 98%
rename from hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualFreePageManagerTest.java
rename to hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualFreePageManagerTest.java
index 764a82f..f4ec55d 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualFreePageManagerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualFreePageManagerTest.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.hyracks.storage.am.lsm.common;
+package org.apache.hyracks.storage.am.lsm.common.test;
import static org.junit.Assert.assertEquals;
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeTestWorker.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeTestWorker.java
index 22ccf88..95b4f44 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeTestWorker.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeTestWorker.java
@@ -30,7 +30,8 @@
import org.apache.hyracks.storage.am.common.datagen.DataGenThread;
import org.apache.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTree;
-import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTree.LSMRTreeAccessor;
+import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeAccessor;
+import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeOpContext;
import org.apache.hyracks.storage.am.rtree.impls.SearchPredicate;
import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.common.MultiComparator;
@@ -54,7 +55,8 @@
public void performOp(ITupleReference tuple, TestOperation op) throws HyracksDataException {
LSMRTreeAccessor accessor = (LSMRTreeAccessor) indexAccessor;
ITreeIndexCursor searchCursor = accessor.createSearchCursor(false);
- MultiComparator cmp = accessor.getMultiComparator();
+ LSMRTreeOpContext concreteCtx = (LSMRTreeOpContext) accessor.getCtx();
+ MultiComparator cmp = concreteCtx.getCurrentRTreeOpContext().getCmp();
SearchPredicate rangePred = new SearchPredicate(tuple, cmp);
switch (op) {
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesTestWorker.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesTestWorker.java
index 4f043bd..3917203 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesTestWorker.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesTestWorker.java
@@ -25,9 +25,10 @@
import org.apache.hyracks.storage.am.common.TestOperationSelector.TestOperation;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.datagen.DataGenThread;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.rtree.impls.AbstractLSMRTree;
-import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeWithAntiMatterTuples.LSMRTreeWithAntiMatterTuplesAccessor;
+import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeOpContext;
import org.apache.hyracks.storage.am.rtree.impls.SearchPredicate;
import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.common.MultiComparator;
@@ -41,9 +42,10 @@
@Override
public void performOp(ITupleReference tuple, TestOperation op) throws HyracksDataException {
- LSMRTreeWithAntiMatterTuplesAccessor accessor = (LSMRTreeWithAntiMatterTuplesAccessor) indexAccessor;
+ LSMTreeIndexAccessor accessor = (LSMTreeIndexAccessor) indexAccessor;
ITreeIndexCursor searchCursor = accessor.createSearchCursor(false);
- MultiComparator cmp = accessor.getMultiComparator();
+ LSMRTreeOpContext concreteCtx = (LSMRTreeOpContext) accessor.getCtx();
+ MultiComparator cmp = concreteCtx.getCurrentRTreeOpContext().getCmp();
SearchPredicate rangePred = new SearchPredicate(tuple, cmp);
switch (op) {