Merged hyracks_lsm_tree r2599:r2780.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_lsm_experiments@2781 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/FileReference.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/FileReference.java
index e6f3481..ffe41d0 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/FileReference.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/FileReference.java
@@ -37,7 +37,7 @@
return file;
}
- public IODeviceHandle getDevideHandle() {
+ public IODeviceHandle getDeviceHandle() {
return dev;
}
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/common/TreeOperatorTestHelper.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/common/TreeOperatorTestHelper.java
index 362badb..935724b 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/common/TreeOperatorTestHelper.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/common/TreeOperatorTestHelper.java
@@ -23,6 +23,8 @@
protected final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
protected final String sep = System.getProperty("file.separator");
+ protected static int DEFAULT_MEM_PAGE_SIZE = 32768;
+ protected static int DEFAULT_MEM_NUM_PAGES = 1000;
public String getPrimaryIndexName() {
return System.getProperty("java.io.tmpdir") + sep + "primary" + simpleDateFormat.format(new Date());
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/PartitionedWordInvertedIndexTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/PartitionedWordInvertedIndexTest.java
index 1bd6878..62d4362 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/PartitionedWordInvertedIndexTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/PartitionedWordInvertedIndexTest.java
@@ -27,9 +27,9 @@
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.ShortSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ConstantMergePolicyProvider;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.FlushControllerProvider;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ImmediateSchedulerProvider;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.RefCountingOperationTrackerFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.SynchronousSchedulerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ThreadCountingOperationTrackerFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.dataflow.PartitionedLSMInvertedIndexDataflowHelperFactory;
public class PartitionedWordInvertedIndexTest extends AbstractfWordInvertedIndexTest {
@@ -52,9 +52,9 @@
PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) };
invertedIndexDataflowHelperFactory = new PartitionedLSMInvertedIndexDataflowHelperFactory(
- new FlushControllerProvider(), new ConstantMergePolicyProvider(ImmediateSchedulerProvider.INSTANCE,
- MERGE_THRESHOLD), RefCountingOperationTrackerFactory.INSTANCE,
- ImmediateSchedulerProvider.INSTANCE);
+ new ConstantMergePolicyProvider(MERGE_THRESHOLD), ThreadCountingOperationTrackerFactory.INSTANCE,
+ SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE, DEFAULT_MEM_PAGE_SIZE,
+ DEFAULT_MEM_NUM_PAGES);
}
@Override
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java
index 69c7c90..c35c6c9 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java
@@ -25,9 +25,9 @@
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ConstantMergePolicyProvider;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.FlushControllerProvider;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ImmediateSchedulerProvider;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.RefCountingOperationTrackerFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.SynchronousSchedulerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ThreadCountingOperationTrackerFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexDataflowHelperFactory;
public class WordInvertedIndexTest extends AbstractfWordInvertedIndexTest {
@@ -46,9 +46,9 @@
PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY),
PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) };
- invertedIndexDataflowHelperFactory = new LSMInvertedIndexDataflowHelperFactory(new FlushControllerProvider(),
- new ConstantMergePolicyProvider(ImmediateSchedulerProvider.INSTANCE, MERGE_THRESHOLD),
- RefCountingOperationTrackerFactory.INSTANCE, ImmediateSchedulerProvider.INSTANCE);
+ invertedIndexDataflowHelperFactory = new LSMInvertedIndexDataflowHelperFactory(new ConstantMergePolicyProvider(
+ MERGE_THRESHOLD), ThreadCountingOperationTrackerFactory.INSTANCE,
+ SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE, DEFAULT_MEM_PAGE_SIZE, DEFAULT_MEM_NUM_PAGES);
}
@Override
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java
index 6dd0307..fce6e69 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java
@@ -18,10 +18,11 @@
import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.AbstractLSMIndexDataflowHelper;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ConstantMergePolicyProvider;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.FlushControllerProvider;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.RefCountingOperationTrackerFactory;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ImmediateSchedulerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.SynchronousSchedulerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ThreadCountingOperationTrackerFactory;
import edu.uci.ics.hyracks.tests.am.common.LSMTreeOperatorTestHelper;
public class LSMBTreeOperatorTestHelper extends LSMTreeOperatorTestHelper {
@@ -33,9 +34,9 @@
}
public IIndexDataflowHelperFactory createDataFlowHelperFactory() {
- return new LSMBTreeDataflowHelperFactory(new FlushControllerProvider(), new ConstantMergePolicyProvider(
- ImmediateSchedulerProvider.INSTANCE, MERGE_THRESHOLD), RefCountingOperationTrackerFactory.INSTANCE,
- ImmediateSchedulerProvider.INSTANCE);
+ return new LSMBTreeDataflowHelperFactory(new ConstantMergePolicyProvider(MERGE_THRESHOLD),
+ ThreadCountingOperationTrackerFactory.INSTANCE, SynchronousSchedulerProvider.INSTANCE,
+ NoOpIOOperationCallback.INSTANCE, DEFAULT_MEM_PAGE_SIZE, DEFAULT_MEM_NUM_PAGES);
}
}
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
index 0ee5c89..84b34b7 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
@@ -21,9 +21,9 @@
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ConstantMergePolicyProvider;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.FlushControllerProvider;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ImmediateSchedulerProvider;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.RefCountingOperationTrackerFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.SynchronousSchedulerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ThreadCountingOperationTrackerFactory;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.dataflow.LSMRTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
import edu.uci.ics.hyracks.tests.am.common.LSMTreeOperatorTestHelper;
@@ -40,9 +40,8 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory) {
return new LSMRTreeDataflowHelperFactory(valueProviderFactories, rtreePolicyType, btreeComparatorFactories,
- new FlushControllerProvider(), new ConstantMergePolicyProvider(ImmediateSchedulerProvider.INSTANCE,
- MERGE_THRESHOLD), RefCountingOperationTrackerFactory.INSTANCE,
- ImmediateSchedulerProvider.INSTANCE, linearizerCmpFactory);
+ new ConstantMergePolicyProvider(MERGE_THRESHOLD), ThreadCountingOperationTrackerFactory.INSTANCE,
+ SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE, linearizerCmpFactory,
+ DEFAULT_MEM_PAGE_SIZE, DEFAULT_MEM_NUM_PAGES);
}
-
}
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java
index 86115a4..bb31dcd 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java
@@ -21,9 +21,9 @@
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ConstantMergePolicyProvider;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.FlushControllerProvider;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ImmediateSchedulerProvider;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.RefCountingOperationTrackerFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.SynchronousSchedulerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ThreadCountingOperationTrackerFactory;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.dataflow.LSMRTreeWithAntiMatterTuplesDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
import edu.uci.ics.hyracks.tests.am.common.LSMTreeOperatorTestHelper;
@@ -40,9 +40,9 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory) {
return new LSMRTreeWithAntiMatterTuplesDataflowHelperFactory(valueProviderFactories, rtreePolicyType,
- btreeComparatorFactories, new FlushControllerProvider(), new ConstantMergePolicyProvider(
- ImmediateSchedulerProvider.INSTANCE, MERGE_THRESHOLD),
- RefCountingOperationTrackerFactory.INSTANCE, ImmediateSchedulerProvider.INSTANCE, linearizerCmpFactory);
+ btreeComparatorFactories, new ConstantMergePolicyProvider(MERGE_THRESHOLD),
+ ThreadCountingOperationTrackerFactory.INSTANCE, SynchronousSchedulerProvider.INSTANCE,
+ NoOpIOOperationCallback.INSTANCE, linearizerCmpFactory);
}
}
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java
index 023bdd9..0e0e899 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java
@@ -50,6 +50,9 @@
private static IHyracksClientConnection hcc;
private final List<File> outputFiles;
+
+ protected static int DEFAULT_MEM_PAGE_SIZE = 32768;
+ protected static int DEFAULT_MEM_NUM_PAGES = 1000;
@Rule
public TemporaryFolder outputFolder = new TemporaryFolder();
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
index fa2d216..e46efff 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
@@ -78,7 +78,8 @@
resourceID = resourceIdFactory.createId();
ILocalResourceFactory localResourceFactory = opDesc.getLocalResourceFactoryProvider()
.getLocalResourceFactory();
- localResourceRepository.insert(localResourceFactory.createLocalResource(resourceID, file.getFile().getPath()));
+ localResourceRepository.insert(localResourceFactory.createLocalResource(resourceID, file.getFile()
+ .getPath(), partition));
} catch (IOException e) {
throw new HyracksDataException(e);
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java
index a0189c2..1848b8f 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java
@@ -27,6 +27,9 @@
}
private boolean evictCandidateIndex() throws HyracksDataException {
+ // Why min()? As a heuristic for eviction, we will take an open index (an index consuming memory)
+ // that is not being used (refcount == 0) and has been least recently used. The sort order defined
+ // for IndexInfo maintains this. See IndexInfo.compareTo().
IndexInfo info = Collections.min(indexInfos.values());
if (info.referenceCount != 0 || !info.isOpen) {
return false;
@@ -126,6 +129,15 @@
@Override
public int compareTo(IndexInfo i) {
// sort by (isOpen, referenceCount, lastAccess) ascending, where true < false
+ //
+ // Example sort order:
+ // -------------------
+ // (F, 0, 70) <-- largest
+ // (F, 0, 60)
+ // (T, 10, 80)
+ // (T, 10, 70)
+ // (T, 9, 90)
+ // (T, 0, 100) <-- smallest
if (isOpen && !i.isOpen) {
return -1;
} else if (!isOpen && i.isOpen) {
@@ -149,7 +161,8 @@
}
public String toString() {
- return "{lastAccess: " + lastAccess + ", refCount: " + referenceCount + "}";
+ return "{index: " + index + ", isOpen: " + isOpen + ", refCount: " + referenceCount + ", lastAccess: "
+ + lastAccess + "}";
}
}
}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/IndexOperation.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/IndexOperation.java
index 8990f3f..1885348 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/IndexOperation.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/IndexOperation.java
@@ -24,5 +24,6 @@
DISKORDERSCAN,
PHYSICALDELETE,
NOOP,
- MERGE
+ MERGE,
+ FLUSH
}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
index 76a1c0c..0bed4ca 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
@@ -25,7 +25,7 @@
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -38,17 +38,18 @@
public class LSMBTreeDataflowHelper extends AbstractLSMIndexDataflowHelper {
public LSMBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) {
- this(opDesc, ctx, partition, DEFAULT_MEM_PAGE_SIZE, DEFAULT_MEM_NUM_PAGES, flushController, mergePolicy,
- opTrackerFactory, ioScheduler);
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
+ this(opDesc, ctx, partition, DEFAULT_MEM_PAGE_SIZE, DEFAULT_MEM_NUM_PAGES, mergePolicy, opTrackerFactory,
+ ioScheduler, ioOpCallbackProvider);
}
public LSMBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- int memPageSize, int memNumPages, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) {
- super(opDesc, ctx, partition, memPageSize, memNumPages, flushController, mergePolicy, opTrackerFactory,
- ioScheduler);
+ int memPageSize, int memNumPages, ILSMMergePolicy mergePolicy,
+ ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
+ super(opDesc, ctx, partition, memPageSize, memNumPages, mergePolicy, opTrackerFactory, ioScheduler,
+ ioOpCallbackProvider);
}
@Override
@@ -60,7 +61,7 @@
IInMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(memNumPages, metaDataFrameFactory);
return LSMBTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ctx.getIOManager(), file, opDesc
.getStorageManager().getBufferCache(ctx), opDesc.getStorageManager().getFileMapProvider(ctx),
- treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc.getTreeIndexComparatorFactories(), flushController,
- mergePolicy, opTrackerFactory, ioScheduler, partition);
+ treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc.getTreeIndexComparatorFactories(), mergePolicy,
+ opTrackerFactory, ioScheduler, ioOpCallbackProvider, partition);
}
}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
index 8a177c0..f451748 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
@@ -18,7 +18,7 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushControllerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicyProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -28,17 +28,17 @@
private static final long serialVersionUID = 1L;
- public LSMBTreeDataflowHelperFactory(ILSMFlushControllerProvider flushControllerProvider,
- ILSMMergePolicyProvider mergePolicyProvider, ILSMOperationTrackerFactory opTrackerFactory,
- ILSMIOOperationSchedulerProvider ioSchedulerProvider) {
- super(flushControllerProvider, mergePolicyProvider, opTrackerFactory, ioSchedulerProvider);
+ public LSMBTreeDataflowHelperFactory(ILSMMergePolicyProvider mergePolicyProvider,
+ ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
+ ILSMIOOperationCallbackProvider ioOpCallbackProvider, int memPageSize, int memNumPages) {
+ super(mergePolicyProvider, opTrackerFactory, ioSchedulerProvider, ioOpCallbackProvider, memPageSize,
+ memNumPages);
}
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition) {
- return new LSMBTreeDataflowHelper(opDesc, ctx, partition, flushControllerProvider.getFlushController(ctx),
- mergePolicyProvider.getMergePolicy(ctx), opTrackerFactory,
- ioSchedulerProvider.getIOScheduler(ctx));
+ return new LSMBTreeDataflowHelper(opDesc, ctx, partition, memPageSize, memNumPages, mergePolicyProvider.getMergePolicy(ctx),
+ opTrackerFactory, ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackProvider);
}
}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index ed53290..4cd4974 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -16,7 +16,6 @@
package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
import java.io.File;
-import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
@@ -31,6 +30,7 @@
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IInMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOperationContext;
@@ -49,21 +49,21 @@
import edu.uci.ics.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleReference;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentState;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMFlushOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
@@ -73,14 +73,13 @@
public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
// In-memory components.
- private final LSMBTreeComponent mutableComponent;
+ private final LSMBTreeMutableComponent mutableComponent;
- // On-disk components.
// For creating BTree's used in flush and merge.
- private final LSMBTreeComponentFactory componentFactory;
+ private final LSMBTreeImmutableComponentFactory componentFactory;
// For creating BTree's used in bulk load. Different from diskBTreeFactory
// because it should have a different tuple writer in it's leaf frames.
- private final LSMBTreeComponentFactory bulkLoadComponentFactory;
+ private final LSMBTreeImmutableComponentFactory bulkLoadComponentFactory;
// Common for in-memory and on-disk components.
private final ITreeIndexFrameFactory insertLeafFrameFactory;
@@ -92,18 +91,19 @@
ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager,
TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory,
IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) {
- super(memFreePageManager, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider, flushController,
- mergePolicy, opTrackerFactory, ioScheduler);
- mutableComponent = new LSMBTreeComponent(new BTree(memBufferCache,
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
+ super(memFreePageManager, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider, mergePolicy,
+ opTrackerFactory, ioScheduler, ioOpCallbackProvider);
+ mutableComponent = new LSMBTreeMutableComponent(new BTree(memBufferCache,
((InMemoryBufferCache) memBufferCache).getFileMapProvider(), memFreePageManager, interiorFrameFactory,
- insertLeafFrameFactory, cmpFactories, fieldCount, new FileReference(new File("membtree"))));
+ insertLeafFrameFactory, cmpFactories, fieldCount, new FileReference(new File("membtree"))),
+ memFreePageManager);
this.insertLeafFrameFactory = insertLeafFrameFactory;
this.deleteLeafFrameFactory = deleteLeafFrameFactory;
this.cmpFactories = cmpFactories;
- componentFactory = new LSMBTreeComponentFactory(diskBTreeFactory);
- bulkLoadComponentFactory = new LSMBTreeComponentFactory(bulkLoadBTreeFactory);
+ componentFactory = new LSMBTreeImmutableComponentFactory(diskBTreeFactory);
+ bulkLoadComponentFactory = new LSMBTreeImmutableComponentFactory(bulkLoadBTreeFactory);
}
@Override
@@ -114,7 +114,7 @@
fileManager.deleteDirs();
fileManager.createDirs();
- immutableComponents.clear();
+ componentsRef.get().clear();
}
@Override
@@ -126,6 +126,7 @@
((InMemoryBufferCache) mutableComponent.getBTree().getBufferCache()).open();
mutableComponent.getBTree().create();
mutableComponent.getBTree().activate();
+ List<ILSMComponent> immutableComponents = componentsRef.get();
immutableComponents.clear();
List<LSMComponentFileReferences> validFileReferences;
try {
@@ -134,7 +135,7 @@
throw new HyracksDataException(e);
}
for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
- LSMBTreeComponent btree;
+ LSMBTreeImmutableComponent btree;
try {
btree = createDiskComponent(componentFactory, lsmComonentFileReference.getInsertIndexFileReference(),
false);
@@ -147,22 +148,26 @@
}
@Override
- public synchronized void deactivate() throws HyracksDataException {
+ public synchronized void deactivate(boolean flushOnExit) throws HyracksDataException {
if (!isActivated) {
return;
}
- BlockingIOOperationCallback cb = new BlockingIOOperationCallback();
- ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- lsmHarness.getIOScheduler().scheduleOperation(accessor.createFlushOperation(cb));
- try {
- cb.waitForIO();
- } catch (InterruptedException e) {
- throw new HyracksDataException(e);
+ if (flushOnExit) {
+ BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(
+ ioOpCallbackProvider.getIOOperationCallback(this));
+ ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ accessor.scheduleFlush(cb);
+ try {
+ cb.waitForIO();
+ } catch (InterruptedException e) {
+ throw new HyracksDataException(e);
+ }
}
+ List<ILSMComponent> immutableComponents = componentsRef.get();
for (ILSMComponent c : immutableComponents) {
- BTree btree = (BTree) ((LSMBTreeComponent) c).getBTree();
+ BTree btree = (BTree) ((LSMBTreeImmutableComponent) c).getBTree();
btree.deactivate();
}
mutableComponent.getBTree().deactivate();
@@ -172,13 +177,19 @@
}
@Override
+ public synchronized void deactivate() throws HyracksDataException {
+ deactivate(true);
+ }
+
+ @Override
public void destroy() throws HyracksDataException {
if (isActivated) {
throw new HyracksDataException("Failed to destroy the index since it is activated.");
}
+ List<ILSMComponent> immutableComponents = componentsRef.get();
for (ILSMComponent c : immutableComponents) {
- BTree btree = (BTree) ((LSMBTreeComponent) c).getBTree();
+ BTree btree = (BTree) ((LSMBTreeImmutableComponent) c).getBTree();
btree.destroy();
}
mutableComponent.getBTree().destroy();
@@ -191,9 +202,10 @@
throw new HyracksDataException("Failed to clear the index since it is not activated.");
}
+ List<ILSMComponent> immutableComponents = componentsRef.get();
mutableComponent.getBTree().clear();
for (ILSMComponent c : immutableComponents) {
- BTree btree = (BTree) ((LSMBTreeComponent) c).getBTree();
+ BTree btree = (BTree) ((LSMBTreeImmutableComponent) c).getBTree();
btree.deactivate();
btree.destroy();
}
@@ -201,33 +213,33 @@
}
@Override
- public List<ILSMComponent> getOperationalComponents(IIndexOperationContext ctx) {
- List<ILSMComponent> operationalComponents = new ArrayList<ILSMComponent>();
+ public void getOperationalComponents(ILSMIndexOperationContext ctx) {
+ List<ILSMComponent> immutableComponents = componentsRef.get();
+ List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
+ operationalComponents.clear();
switch (ctx.getOperation()) {
+ case UPDATE:
+ case UPSERT:
+ case PHYSICALDELETE:
+ case FLUSH:
+ case DELETE:
+ operationalComponents.add(mutableComponent);
+ break;
case SEARCH:
case INSERT:
- // TODO: We should add the mutable component at some point.
+ operationalComponents.add(mutableComponent);
operationalComponents.addAll(immutableComponents);
break;
case MERGE:
- // TODO: determining the participating components in a merge should probably the task of the merge policy.
- if (immutableComponents.size() > 1) {
- for (ILSMComponent c : immutableComponents) {
- if (c.negativeCompareAndSet(LSMComponentState.MERGING, LSMComponentState.MERGING)) {
- operationalComponents.add(c);
- }
- }
- }
+ operationalComponents.addAll(immutableComponents);
break;
default:
throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
}
- return operationalComponents;
}
@Override
- public void insertUpdateOrDelete(ITupleReference tuple, IIndexOperationContext ictx) throws HyracksDataException,
- IndexException {
+ public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
LSMBTreeOpContext ctx = (LSMBTreeOpContext) ictx;
switch (ctx.getOperation()) {
case PHYSICALDELETE:
@@ -269,7 +281,7 @@
// TODO: Can we just remove the above code that search the mutable component and do it together with the search call below? i.e. instead of passing false to the lsmHarness.search(), we pass true to include the mutable component?
// the key was not in the inmemory component, so check the disk components
- lsmHarness.search(searchCursor, predicate, ctx, false);
+ search(ctx, searchCursor, predicate);
try {
if (searchCursor.hasNext()) {
throw new BTreeDuplicateKeyException("Failed to insert key since key already exists.");
@@ -283,16 +295,71 @@
}
@Override
+ public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred)
+ throws HyracksDataException, IndexException {
+ LSMBTreeOpContext ctx = (LSMBTreeOpContext) ictx;
+ LSMBTreeRangeSearchCursor lsmTreeCursor = (LSMBTreeRangeSearchCursor) cursor;
+ List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
+ int numBTrees = operationalComponents.size();
+ assert numBTrees > 0;
+
+ boolean includeMutableComponent = operationalComponents.get(0) == mutableComponent;
+ LSMBTreeCursorInitialState initialState = new LSMBTreeCursorInitialState(numBTrees, insertLeafFrameFactory,
+ ctx.cmp, includeMutableComponent, lsmHarness, ctx.memBTreeAccessor, pred, ctx.searchCallback,
+ operationalComponents);
+ lsmTreeCursor.open(initialState, pred);
+
+ int cursorIx;
+ ListIterator<ILSMComponent> diskBTreesIter = operationalComponents.listIterator();
+ if (includeMutableComponent) {
+ // Open cursor of in-memory BTree at index 0.
+ ctx.memBTreeAccessor.search(lsmTreeCursor.getCursor(0), pred);
+ // Skip 0 because it is the in-memory BTree.
+ cursorIx = 1;
+ diskBTreesIter.next();
+ } else {
+ cursorIx = 0;
+ }
+
+ // Open cursors of on-disk BTrees.
+ int numDiskComponents = includeMutableComponent ? numBTrees - 1 : numBTrees;
+ ITreeIndexAccessor[] diskBTreeAccessors = new ITreeIndexAccessor[numDiskComponents];
+ int diskBTreeIx = 0;
+ while (diskBTreesIter.hasNext()) {
+ BTree diskBTree = (BTree) ((LSMBTreeImmutableComponent) diskBTreesIter.next()).getBTree();
+ diskBTreeAccessors[diskBTreeIx] = diskBTree.createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ diskBTreeAccessors[diskBTreeIx].search(lsmTreeCursor.getCursor(cursorIx), pred);
+ cursorIx++;
+ diskBTreeIx++;
+ }
+ lsmTreeCursor.initPriorityQueue();
+ }
+
+ @Override
+ public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ throws HyracksDataException {
+ LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
+ LSMBTreeOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ assert ctx.getComponentHolder().size() == 1;
+ ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
+ opCtx.setOperation(IndexOperation.FLUSH);
+ opCtx.getComponentHolder().add(flushingComponent);
+ ILSMIndexAccessorInternal flushAccessor = new LSMBTreeAccessor(lsmHarness, opCtx);
+ ioScheduler.scheduleOperation(new LSMFlushOperation(flushAccessor, flushingComponent, componentFileRefs
+ .getInsertIndexFileReference(), callback));
+ }
+
+ @Override
public ILSMComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
LSMFlushOperation flushOp = (LSMFlushOperation) operation;
- // Bulk load a new on-disk BTree from the in-memory BTree.
+ LSMBTreeMutableComponent flushingComponent = (LSMBTreeMutableComponent) flushOp.getFlushingComponent();
+ IIndexAccessor accessor = flushingComponent.getBTree().createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ IIndexCursor scanCursor = accessor.createSearchCursor();
RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null);
- ITreeIndexAccessor memBTreeAccessor = mutableComponent.getBTree().createAccessor(
- NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- IIndexCursor scanCursor = memBTreeAccessor.createSearchCursor();
- memBTreeAccessor.search(scanCursor, nullPred);
- LSMBTreeComponent component = createDiskComponent(componentFactory, flushOp.getFlushTarget(), true);
- // Bulk load the tuples from the in-memory BTree into the new disk BTree.
+ accessor.search(scanCursor, nullPred);
+ LSMBTreeImmutableComponent component = createDiskComponent(componentFactory, flushOp.getFlushTarget(), true);
IIndexBulkLoader bulkLoader = component.getBTree().createBulkLoader(1.0f, false);
try {
while (scanCursor.hasNext()) {
@@ -306,71 +373,26 @@
return component;
}
- @Override
- public void resetMutableComponent() throws HyracksDataException {
- memFreePageManager.reset();
- mutableComponent.getBTree().clear();
- }
+ public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ throws HyracksDataException, IndexException {
+ LSMBTreeOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
+ opCtx.getComponentHolder().addAll(mergingComponents);
+ ITreeIndexCursor cursor = new LSMBTreeRangeSearchCursor(opCtx);
+ RangePredicate rangePred = new RangePredicate(null, null, true, true, null, null);
+ search(opCtx, cursor, rangePred);
- private ILSMComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
- LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- return createDiskComponent(bulkLoadComponentFactory, componentFileRefs.getInsertIndexFileReference(), true);
- }
-
- private LSMBTreeComponent createDiskComponent(LSMBTreeComponentFactory factory, FileReference fileRef,
- boolean createComponent) throws HyracksDataException, IndexException {
- // Create new BTree instance.
- LSMBTreeComponent component = (LSMBTreeComponent) factory
- .createLSMComponentInstance(new LSMComponentFileReferences(fileRef, null));
- if (createComponent) {
- component.getBTree().create();
- }
- // BTree will be closed during cleanup of merge().
- component.getBTree().activate();
- return component;
- }
-
- @Override
- public void search(IIndexCursor cursor, List<ILSMComponent> immutableComponents, ISearchPredicate pred,
- IIndexOperationContext ictx, boolean includeMutableComponent) throws HyracksDataException, IndexException {
- LSMBTreeOpContext ctx = (LSMBTreeOpContext) ictx;
- LSMBTreeRangeSearchCursor lsmTreeCursor = (LSMBTreeRangeSearchCursor) cursor;
- int numDiskComponents = immutableComponents.size();
- int numBTrees = (includeMutableComponent) ? numDiskComponents + 1 : numDiskComponents;
-
- List<ILSMComponent> operationalComponents = new ArrayList<ILSMComponent>();
- if (includeMutableComponent) {
- operationalComponents.add(getMutableComponent());
- }
- operationalComponents.addAll(immutableComponents);
- LSMBTreeCursorInitialState initialState = new LSMBTreeCursorInitialState(numBTrees, insertLeafFrameFactory,
- ctx.cmp, includeMutableComponent, lsmHarness, ctx.memBTreeAccessor, pred, ctx.searchCallback,
- operationalComponents);
- lsmTreeCursor.open(initialState, pred);
-
- int cursorIx;
- if (includeMutableComponent) {
- // Open cursor of in-memory BTree at index 0.
- ctx.memBTreeAccessor.search(lsmTreeCursor.getCursor(0), pred);
- // Skip 0 because it is the in-memory BTree.
- cursorIx = 1;
- } else {
- cursorIx = 0;
- }
-
- // Open cursors of on-disk BTrees.
- ITreeIndexAccessor[] diskBTreeAccessors = new ITreeIndexAccessor[numDiskComponents];
- int diskBTreeIx = 0;
- ListIterator<ILSMComponent> diskBTreesIter = immutableComponents.listIterator();
- while (diskBTreesIter.hasNext()) {
- BTree diskBTree = (BTree) ((LSMBTreeComponent) diskBTreesIter.next()).getBTree();
- diskBTreeAccessors[diskBTreeIx] = diskBTree.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
- diskBTreeAccessors[diskBTreeIx].search(lsmTreeCursor.getCursor(cursorIx), pred);
- cursorIx++;
- diskBTreeIx++;
- }
- lsmTreeCursor.initPriorityQueue();
+ opCtx.setOperation(IndexOperation.MERGE);
+ BTree firstBTree = (BTree) ((LSMBTreeImmutableComponent) mergingComponents.get(0)).getBTree();
+ BTree lastBTree = (BTree) ((LSMBTreeImmutableComponent) mergingComponents.get(mergingComponents.size() - 1))
+ .getBTree();
+ FileReference firstFile = diskFileMapProvider.lookupFileName(firstBTree.getFileId());
+ FileReference lastFile = diskFileMapProvider.lookupFileName(lastBTree.getFileId());
+ LSMComponentFileReferences relMergeFileRefs = fileManager.getRelMergeFileReference(firstFile.getFile()
+ .getName(), lastFile.getFile().getName());
+ ILSMIndexAccessorInternal accessor = new LSMBTreeAccessor(lsmHarness, opCtx);
+ ioScheduler.scheduleOperation(new LSMBTreeMergeOperation(accessor, mergingComponents, cursor, relMergeFileRefs
+ .getInsertIndexFileReference(), callback));
}
@Override
@@ -378,17 +400,8 @@
throws HyracksDataException, IndexException {
LSMBTreeMergeOperation mergeOp = (LSMBTreeMergeOperation) operation;
ITreeIndexCursor cursor = mergeOp.getCursor();
-
mergedComponents.addAll(mergeOp.getMergingComponents());
-
- // Nothing to merge.
- if (mergedComponents.size() <= 1) {
- cursor.close();
- return null;
- }
-
- // Bulk load the tuples from all on-disk BTrees into the new BTree.
- LSMBTreeComponent mergedBTree = createDiskComponent(componentFactory, mergeOp.getMergeTarget(), true);
+ LSMBTreeImmutableComponent mergedBTree = createDiskComponent(componentFactory, mergeOp.getMergeTarget(), true);
IIndexBulkLoader bulkLoader = mergedBTree.getBTree().createBulkLoader(1.0f, false);
try {
while (cursor.hasNext()) {
@@ -403,11 +416,36 @@
return mergedBTree;
}
+ private LSMBTreeImmutableComponent createDiskComponent(LSMBTreeImmutableComponentFactory factory,
+ FileReference fileRef, boolean createComponent) throws HyracksDataException, IndexException {
+ // Create new BTree instance.
+ LSMBTreeImmutableComponent component = (LSMBTreeImmutableComponent) factory
+ .createLSMComponentInstance(new LSMComponentFileReferences(fileRef, null));
+ if (createComponent) {
+ component.getBTree().create();
+ }
+ // BTree will be closed during cleanup of merge().
+ component.getBTree().activate();
+ return component;
+ }
+
@Override
public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput) throws TreeIndexException {
return new LSMBTreeBulkLoader(fillLevel, verifyInput);
}
+ private ILSMComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
+ LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
+ return createDiskComponent(bulkLoadComponentFactory, componentFileRefs.getInsertIndexFileReference(), true);
+ }
+
+ @Override
+ public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException {
+ BTree btree = ((LSMBTreeImmutableComponent) lsmComponent).getBTree();
+ forceFlushDirtyPages(btree);
+ markAsValidInternal(btree);
+ }
+
public class LSMBTreeBulkLoader implements IIndexBulkLoader {
private final ILSMComponent component;
private final BTreeBulkLoader bulkLoader;
@@ -420,8 +458,8 @@
} catch (IndexException e) {
throw new TreeIndexException(e);
}
- bulkLoader = (BTreeBulkLoader) ((LSMBTreeComponent) component).getBTree().createBulkLoader(fillFactor,
- verifyInput);
+ bulkLoader = (BTreeBulkLoader) ((LSMBTreeImmutableComponent) component).getBTree().createBulkLoader(
+ fillFactor, verifyInput);
}
@Override
@@ -441,8 +479,8 @@
}
protected void handleException() throws HyracksDataException {
- ((LSMBTreeComponent) component).getBTree().deactivate();
- ((LSMBTreeComponent) component).getBTree().destroy();
+ ((LSMBTreeImmutableComponent) component).getBTree().deactivate();
+ ((LSMBTreeImmutableComponent) component).getBTree().destroy();
}
@Override
@@ -453,40 +491,6 @@
}
- @Override
- public ITreeIndexFrameFactory getLeafFrameFactory() {
- return mutableComponent.getBTree().getLeafFrameFactory();
- }
-
- @Override
- public ITreeIndexFrameFactory getInteriorFrameFactory() {
- return mutableComponent.getBTree().getInteriorFrameFactory();
- }
-
- @Override
- public IFreePageManager getFreePageManager() {
- return mutableComponent.getBTree().getFreePageManager();
- }
-
- @Override
- public int getFieldCount() {
- return mutableComponent.getBTree().getFieldCount();
- }
-
- @Override
- public int getRootPageId() {
- return mutableComponent.getBTree().getRootPageId();
- }
-
- @Override
- public int getFileId() {
- return mutableComponent.getBTree().getFileId();
- }
-
- public IBinaryComparatorFactory[] getComparatorFactories() {
- return cmpFactories;
- }
-
public LSMBTreeOpContext createOpContext(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
return new LSMBTreeOpContext(mutableComponent.getBTree(), insertLeafFrameFactory, deleteLeafFrameFactory,
@@ -494,7 +498,7 @@
}
@Override
- public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
+ public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
return new LSMBTreeAccessor(lsmHarness, createOpContext(modificationCallback, searchCallback));
}
@@ -513,50 +517,6 @@
LSMBTreeOpContext concreteCtx = (LSMBTreeOpContext) ctx;
return concreteCtx.cmp;
}
-
- @Override
- public ILSMIOOperation createFlushOperation(ILSMIOOperationCallback callback) {
- LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- return new LSMFlushOperation(lsmHarness.getIndex(), componentFileRefs.getInsertIndexFileReference(),
- callback);
- }
- }
-
- public ILSMIOOperation createMergeOperation(ILSMIOOperationCallback callback) throws HyracksDataException {
- LSMBTreeOpContext ctx = createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- ctx.setOperation(IndexOperation.MERGE);
- ITreeIndexCursor cursor = new LSMBTreeRangeSearchCursor(ctx);
- RangePredicate rangePred = new RangePredicate(null, null, true, true, null, null);
- // Ordered scan, ignoring the in-memory BTree.
- // We get back a snapshot of the on-disk BTrees that are going to be
- // merged now, so we can clean them up after the merge has completed.
- List<ILSMComponent> mergingDiskComponents;
- try {
- mergingDiskComponents = lsmHarness.search(cursor, (RangePredicate) rangePred, ctx, false);
- if (mergingDiskComponents.size() <= 1) {
- cursor.close();
- return null;
- }
- } catch (IndexException e) {
- throw new HyracksDataException(e);
- }
-
- BTree firstBTree = (BTree) ((LSMBTreeComponent) mergingDiskComponents.get(0)).getBTree();
- BTree lastBTree = (BTree) ((LSMBTreeComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1))
- .getBTree();
- FileReference firstFile = diskFileMapProvider.lookupFileName(firstBTree.getFileId());
- FileReference lastFile = diskFileMapProvider.lookupFileName(lastBTree.getFileId());
- LSMComponentFileReferences relMergeFileRefs = fileManager.getRelMergeFileReference(firstFile.getFile()
- .getName(), lastFile.getFile().getName());
- return new LSMBTreeMergeOperation(lsmHarness.getIndex(), mergingDiskComponents, cursor,
- relMergeFileRefs.getInsertIndexFileReference(), callback);
- }
-
- @Override
- public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException {
- BTree btree = ((LSMBTreeComponent) lsmComponent).getBTree();
- forceFlushDirtyPages(btree);
- markAsValidInternal(btree);
}
@Override
@@ -564,19 +524,33 @@
return diskBufferCache;
}
- public boolean isEmptyIndex() throws HyracksDataException {
- return immutableComponents.isEmpty()
- && mutableComponent.getBTree().isEmptyTree(
- mutableComponent.getBTree().getInteriorFrameFactory().createFrame());
+ public IBinaryComparatorFactory[] getComparatorFactories() {
+ return cmpFactories;
}
@Override
- public void validate() throws HyracksDataException {
- mutableComponent.getBTree().validate();
- for (ILSMComponent c : immutableComponents) {
- BTree btree = (BTree) ((LSMBTreeComponent) c).getBTree();
- btree.validate();
- }
+ public ITreeIndexFrameFactory getInteriorFrameFactory() {
+ return mutableComponent.getBTree().getInteriorFrameFactory();
+ }
+
+ @Override
+ public int getFieldCount() {
+ return mutableComponent.getBTree().getFieldCount();
+ }
+
+ @Override
+ public int getFileId() {
+ return mutableComponent.getBTree().getFileId();
+ }
+
+ @Override
+ public IFreePageManager getFreePageManager() {
+ return mutableComponent.getBTree().getFreePageManager();
+ }
+
+ @Override
+ public ITreeIndexFrameFactory getLeafFrameFactory() {
+ return mutableComponent.getBTree().getLeafFrameFactory();
}
@Override
@@ -586,7 +560,24 @@
}
@Override
- public ILSMComponent getMutableComponent() {
- return mutableComponent;
+ public int getRootPageId() {
+ return mutableComponent.getBTree().getRootPageId();
+ }
+
+ public boolean isEmptyIndex() throws HyracksDataException {
+ List<ILSMComponent> immutableComponents = componentsRef.get();
+ return immutableComponents.isEmpty()
+ && mutableComponent.getBTree().isEmptyTree(
+ mutableComponent.getBTree().getInteriorFrameFactory().createFrame());
+ }
+
+ @Override
+ public void validate() throws HyracksDataException {
+ mutableComponent.getBTree().validate();
+ List<ILSMComponent> immutableComponents = componentsRef.get();
+ for (ILSMComponent c : immutableComponents) {
+ BTree btree = (BTree) ((LSMBTreeImmutableComponent) c).getBTree();
+ btree.validate();
+ }
}
}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeImmutableComponent.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeImmutableComponent.java
new file mode 100644
index 0000000..2251a49
--- /dev/null
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeImmutableComponent.java
@@ -0,0 +1,24 @@
+package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractImmutableLSMComponent;
+
+public class LSMBTreeImmutableComponent extends AbstractImmutableLSMComponent {
+
+ private final BTree btree;
+
+ public LSMBTreeImmutableComponent(BTree btree) {
+ this.btree = btree;
+ }
+
+ @Override
+ public void destroy() throws HyracksDataException {
+ btree.deactivate();
+ btree.destroy();
+ }
+
+ public BTree getBTree() {
+ return btree;
+ }
+}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeComponentFactory.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeImmutableComponentFactory.java
similarity index 84%
rename from hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeComponentFactory.java
rename to hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeImmutableComponentFactory.java
index 23e0d83..696fc2c 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeComponentFactory.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeImmutableComponentFactory.java
@@ -23,16 +23,16 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-public class LSMBTreeComponentFactory implements ILSMComponentFactory {
+public class LSMBTreeImmutableComponentFactory implements ILSMComponentFactory {
private final TreeIndexFactory<BTree> btreeFactory;
- public LSMBTreeComponentFactory(TreeIndexFactory<BTree> btreeFactory) {
+ public LSMBTreeImmutableComponentFactory(TreeIndexFactory<BTree> btreeFactory) {
this.btreeFactory = btreeFactory;
}
@Override
public ILSMComponent createLSMComponentInstance(LSMComponentFileReferences cfr) throws IndexException {
- return new LSMBTreeComponent(btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()));
+ return new LSMBTreeImmutableComponent(btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()));
}
@Override
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java
index ec8afaf..a3a7097 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java
@@ -15,33 +15,32 @@
package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
-import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
-import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
public class LSMBTreeMergeOperation implements ILSMIOOperation {
- private final ILSMIndex index;
+ private final ILSMIndexAccessorInternal accessor;
private final List<ILSMComponent> mergingComponents;
private final ITreeIndexCursor cursor;
private final FileReference mergeTarget;
private final ILSMIOOperationCallback callback;
- public LSMBTreeMergeOperation(ILSMIndex index, List<ILSMComponent> mergingComponents, ITreeIndexCursor cursor,
- FileReference mergeTarget, ILSMIOOperationCallback callback) {
- this.index = index;
+ public LSMBTreeMergeOperation(ILSMIndexAccessorInternal accessor, List<ILSMComponent> mergingComponents,
+ ITreeIndexCursor cursor, FileReference mergeTarget, ILSMIOOperationCallback callback) {
+ this.accessor = accessor;
this.mergingComponents = mergingComponents;
this.cursor = cursor;
this.mergeTarget = mergeTarget;
@@ -49,24 +48,22 @@
}
@Override
- public List<IODeviceHandle> getReadDevices() {
- List<IODeviceHandle> devs = new ArrayList<IODeviceHandle>();
+ public Set<IODeviceHandle> getReadDevices() {
+ Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
for (ILSMComponent o : mergingComponents) {
- LSMBTreeComponent component = (LSMBTreeComponent) o;
- devs.add(component.getBTree().getFileReference().getDevideHandle());
+ LSMBTreeImmutableComponent component = (LSMBTreeImmutableComponent) o;
+ devs.add(component.getBTree().getFileReference().getDeviceHandle());
}
return devs;
}
@Override
- public List<IODeviceHandle> getWriteDevices() {
- return Collections.singletonList(mergeTarget.getDevideHandle());
+ public Set<IODeviceHandle> getWriteDevices() {
+ return Collections.singleton(mergeTarget.getDeviceHandle());
}
@Override
public void perform() throws HyracksDataException, IndexException {
- ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
accessor.merge(this);
}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeComponent.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeMutableComponent.java
similarity index 66%
rename from hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeComponent.java
rename to hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeMutableComponent.java
index 5264f4f..30e79b4 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeComponent.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeMutableComponent.java
@@ -17,31 +17,31 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractLSMComponent;
+import edu.uci.ics.hyracks.storage.am.common.api.IInMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractMutableLSMComponent;
-public class LSMBTreeComponent extends AbstractLSMComponent {
+public class LSMBTreeMutableComponent extends AbstractMutableLSMComponent {
private final BTree btree;
+ private final IInMemoryFreePageManager mfpm;
- public LSMBTreeComponent(BTree btree) {
+ public LSMBTreeMutableComponent(BTree btree, IInMemoryFreePageManager mfpm) {
this.btree = btree;
- }
-
- @Override
- public void destroy() throws HyracksDataException {
- btree.deactivate();
- btree.destroy();
- }
-
- @Override
- public void reset() throws HyracksDataException {
- ((InMemoryFreePageManager) btree.getFreePageManager()).reset();
- btree.clear();
+ this.mfpm = mfpm;
}
public BTree getBTree() {
return btree;
}
+ @Override
+ protected boolean isFull() {
+ return mfpm.isFull();
+ }
+
+ @Override
+ protected void reset() throws HyracksDataException {
+ btree.clear();
+ }
+
}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
index 9474354..a9c39f6 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
@@ -15,6 +15,9 @@
package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
+import java.util.LinkedList;
+import java.util.List;
+
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
@@ -25,6 +28,7 @@
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
public final class LSMBTreeOpContext implements ILSMIndexOperationContext {
@@ -40,6 +44,7 @@
public final MultiComparator cmp;
public final IModificationOperationCallback modificationCallback;
public final ISearchOperationCallback searchCallback;
+ private final List<ILSMComponent> componentHolder;
public LSMBTreeOpContext(BTree memBTree, ITreeIndexFrameFactory insertLeafFrameFactory,
ITreeIndexFrameFactory deleteLeafFrameFactory, IModificationOperationCallback modificationCallback,
@@ -61,18 +66,19 @@
if (deleteLeafFrame != null && this.cmp != null) {
deleteLeafFrame.setMultiComparator(cmp);
}
+ this.componentHolder = new LinkedList<ILSMComponent>();
this.modificationCallback = modificationCallback;
this.searchCallback = searchCallback;
}
@Override
public void setOperation(IndexOperation newOp) {
+ reset();
this.op = newOp;
switch (newOp) {
case SEARCH:
setMemBTreeAccessor();
break;
-
case DISKORDERSCAN:
case UPDATE:
// Attention: It is important to leave the leafFrame and
@@ -82,12 +88,10 @@
// previously requested operation.
setMemBTreeAccessor();
return;
-
case UPSERT:
case INSERT:
setInsertMode();
break;
-
case PHYSICALDELETE:
case DELETE:
setDeleteMode();
@@ -117,6 +121,7 @@
@Override
public void reset() {
+ componentHolder.clear();
}
public IndexOperation getOperation() {
@@ -124,6 +129,11 @@
}
@Override
+ public List<ILSMComponent> getComponentHolder() {
+ return componentHolder;
+ }
+
+ @Override
public ISearchOperationCallback getSearchOperationCallback() {
return searchCallback;
}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
index 3fcf5a5..3706230 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
@@ -32,7 +32,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory;
import edu.uci.ics.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -48,17 +48,19 @@
public static LSMBTree createLSMTree(IInMemoryBufferCache memBufferCache,
IInMemoryFreePageManager memFreePageManager, IIOManager ioManager, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] cmpFactories, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) {
+ IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy,
+ ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
return createLSMTree(memBufferCache, memFreePageManager, ioManager, file, diskBufferCache, diskFileMapProvider,
- typeTraits, cmpFactories, flushController, mergePolicy, opTrackerFactory, ioScheduler, 0);
+ typeTraits, cmpFactories, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackProvider, 0);
}
public static LSMBTree createLSMTree(IInMemoryBufferCache memBufferCache,
IInMemoryFreePageManager memFreePageManager, IIOManager ioManager, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] cmpFactories, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler, int startIODeviceIndex) {
+ IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy,
+ ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackProvider ioOpCallbackProvider, int startIODeviceIndex) {
LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
cmpFactories.length, false);
LSMBTreeTupleWriterFactory deleteTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
@@ -83,8 +85,8 @@
LSMBTree lsmTree = new LSMBTree(memBufferCache, memFreePageManager, interiorFrameFactory,
insertLeafFrameFactory, deleteLeafFrameFactory, fileNameManager, diskBTreeFactory,
- bulkLoadBTreeFactory, diskFileMapProvider, typeTraits.length, cmpFactories, flushController,
- mergePolicy, opTrackerFactory, ioScheduler);
+ bulkLoadBTreeFactory, diskFileMapProvider, typeTraits.length, cmpFactories, mergePolicy,
+ opTrackerFactory, ioScheduler, ioOpCallbackProvider);
return lsmTree;
}
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponent.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponent.java
index 6d3f050..fa00f85 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponent.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponent.java
@@ -1,30 +1,10 @@
package edu.uci.ics.hyracks.storage.am.lsm.common.api;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentState;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMOperationType;
public interface ILSMComponent {
- public void activate();
+ public boolean threadEnter(LSMOperationType opType) throws InterruptedException;
- public void deactivate();
-
- public void threadEnter();
-
- public void threadExit();
-
- public int getThreadReferenceCount();
-
- public void setState(LSMComponentState state);
-
- public boolean negativeCompareAndSet(LSMComponentState compare, LSMComponentState update);
-
- public LSMComponentState getState();
-
- // TODO: create two interfaces one for immutable and another for mutable components.
-
- // Only for immutable components.
- public void destroy() throws HyracksDataException;
-
- // Only for mutable components.
- public void reset() throws HyracksDataException;
+ public void threadExit(LSMOperationType opType, boolean failedOperation) throws HyracksDataException;
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFlushController.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFlushController.java
deleted file mode 100644
index fb58880..0000000
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFlushController.java
+++ /dev/null
@@ -1,7 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.lsm.common.api;
-
-public interface ILSMFlushController {
- public void setFlushStatus(ILSMIndex index, boolean needsFlush);
-
- public boolean getFlushStatus(ILSMIndex index);
-}
\ No newline at end of file
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFlushControllerProvider.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFlushControllerProvider.java
deleted file mode 100644
index 107b8a2..0000000
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFlushControllerProvider.java
+++ /dev/null
@@ -1,9 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.lsm.common.api;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-
-public interface ILSMFlushControllerProvider extends Serializable {
- public ILSMFlushController getFlushController(IHyracksTaskContext ctx);
-}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMHarness.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMHarness.java
index 65b6c9d..04d7fa3 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMHarness.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMHarness.java
@@ -15,8 +15,6 @@
package edu.uci.ics.hyracks.storage.am.lsm.common.api;
-import java.util.List;
-
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
@@ -24,32 +22,33 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
public interface ILSMHarness {
- public boolean insertUpdateOrDelete(ITupleReference tuple, ILSMIndexOperationContext ictx, boolean tryOperation)
- throws HyracksDataException, IndexException;
- public boolean noOp(ILSMIndexOperationContext ictx, boolean tryOperation) throws HyracksDataException;
-
- public List<ILSMComponent> search(IIndexCursor cursor, ISearchPredicate pred, ILSMIndexOperationContext ctx,
- boolean includeMutableComponent) throws HyracksDataException, IndexException;
-
- // Eventually includeMutableComponent and ctx should be removed.
- public void closeSearchCursor(List<ILSMComponent> operationalComponents, boolean includeMutableComponent,
- ILSMIndexOperationContext ctx) throws HyracksDataException;
-
- public ILSMIOOperation createMergeOperation(ILSMIOOperationCallback callback) throws HyracksDataException,
+ public void forceModify(ILSMIndexOperationContext ctx, ITupleReference tuple) throws HyracksDataException,
IndexException;
- public void merge(ILSMIOOperation operation) throws HyracksDataException, IndexException;
+ public boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple)
+ throws HyracksDataException, IndexException;
- public void flush(ILSMIOOperation operation) throws HyracksDataException, IndexException;
+ public void noOp(ILSMIndexOperationContext ctx) throws HyracksDataException;
+
+ public void search(ILSMIndexOperationContext ctx, IIndexCursor cursor, ISearchPredicate pred)
+ throws HyracksDataException, IndexException;
+
+ public void endSearch(ILSMIndexOperationContext ctx) throws HyracksDataException;
+
+ public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ throws HyracksDataException, IndexException;
+
+ public void merge(ILSMIndexOperationContext ctx, ILSMIOOperation operation) throws HyracksDataException,
+ IndexException;
+
+ public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ throws HyracksDataException;
+
+ public void flush(ILSMIndexOperationContext ctx, ILSMIOOperation operation) throws HyracksDataException,
+ IndexException;
public void addBulkLoadedComponent(ILSMComponent index) throws HyracksDataException, IndexException;
- public ILSMIndex getIndex();
-
- public ILSMFlushController getFlushController();
-
public ILSMOperationTracker getOperationTracker();
-
- public ILSMIOOperationScheduler getIOScheduler();
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
index 459b0d9..15fdc6e 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
@@ -1,15 +1,15 @@
package edu.uci.ics.hyracks.storage.am.lsm.common.api;
-import java.util.List;
+import java.util.Set;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
public interface ILSMIOOperation {
- public List<IODeviceHandle> getReadDevices();
+ public Set<IODeviceHandle> getReadDevices();
- public List<IODeviceHandle> getWriteDevices();
+ public Set<IODeviceHandle> getWriteDevices();
public void perform() throws HyracksDataException, IndexException;
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallbackProvider.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallbackProvider.java
new file mode 100644
index 0000000..b5c9741
--- /dev/null
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallbackProvider.java
@@ -0,0 +1,5 @@
+package edu.uci.ics.hyracks.storage.am.lsm.common.api;
+
+public interface ILSMIOOperationCallbackProvider {
+ public ILSMIOOperationCallback getIOOperationCallback(ILSMIndex index);
+}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndex.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndex.java
index 5d4ad0b..0a5ce40 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndex.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndex.java
@@ -15,7 +15,10 @@
package edu.uci.ics.hyracks.storage.am.lsm.common.api;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.common.api.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMHarness;
/**
@@ -27,7 +30,12 @@
* concurrent searches/updates/merges may be ongoing.
*/
public interface ILSMIndex extends IIndex {
- public ILSMFlushController getFlushController();
+ public void deactivate(boolean flushOnExit) throws HyracksDataException;
+
+ public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
+ ISearchOperationCallback searchCallback);
+
+ public boolean getFlushStatus(ILSMIndex index);
public ILSMOperationTracker getOperationTracker();
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
index 1ad1924..a85c24b 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
@@ -19,7 +19,6 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
-import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
/**
* Client handle for performing operations
@@ -29,26 +28,9 @@
* concurrent operations).
*/
public interface ILSMIndexAccessor extends IIndexAccessor {
- public ILSMIOOperation createFlushOperation(ILSMIOOperationCallback callback);
+ public void scheduleFlush(ILSMIOOperationCallback callback) throws HyracksDataException;
- public ILSMIOOperation createMergeOperation(ILSMIOOperationCallback callback) throws HyracksDataException,
- IndexException;
-
- /**
- * Force a flush of the in-memory component.
- *
- * @throws HyracksDataException
- * @throws TreeIndexException
- */
- public void flush(ILSMIOOperation operation) throws HyracksDataException, IndexException;
-
- /**
- * Merge all on-disk components.
- *
- * @throws HyracksDataException
- * @throws TreeIndexException
- */
- public void merge(ILSMIOOperation operation) throws HyracksDataException, IndexException;
+ public void scheduleMerge(ILSMIOOperationCallback callback) throws HyracksDataException, IndexException;
/**
* Deletes the tuple from the memory component only.
@@ -121,23 +103,18 @@
* If there is no matching tuple in the index.
*/
public boolean tryUpsert(ITupleReference tuple) throws HyracksDataException, IndexException;
-
+
+ public void forcePhysicalDelete(ITupleReference tuple) throws HyracksDataException, IndexException;
+
+ public void forceInsert(ITupleReference tuple) throws HyracksDataException, IndexException;
+
+ public void forceDelete(ITupleReference tuple) throws HyracksDataException, IndexException;
+
/**
* This method can be used to increase the number of 'active' operations of an index artificially,
* without actually modifying the index.
- * If the operation would have to wait for a flush then we return false.
- * Otherwise, beforeOperation() and afterOperation() of the ILSMOperationTracker are called,
- * and true is returned.
- *
- * @throws HyracksDataException
- */
- public boolean tryNoOp() throws HyracksDataException;
-
- /**
- * This method can be used to increase the number of 'active' operations of an index artificially,
- * without actually modifying the index.
- * This method may block waiting for a flush to finish, and will eventually call
- * beforeOperation() and afterOperation() of the ILSMOperationTracker.
+ * This method does not block and is guaranteed to trigger the {@link ILSMOperationTracker}'s beforeOperation
+ * and afterOperation calls.
*
* @throws HyracksDataException
*/
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessorInternal.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessorInternal.java
new file mode 100644
index 0000000..8c28d47
--- /dev/null
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessorInternal.java
@@ -0,0 +1,24 @@
+package edu.uci.ics.hyracks.storage.am.lsm.common.api;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+
+public interface ILSMIndexAccessorInternal extends ILSMIndexAccessor {
+
+ /**
+ * Force a flush of the in-memory component.
+ *
+ * @throws HyracksDataException
+ * @throws TreeIndexException
+ */
+ public void flush(ILSMIOOperation operation) throws HyracksDataException, IndexException;
+
+ /**
+ * Merge all on-disk components.
+ *
+ * @throws HyracksDataException
+ * @throws TreeIndexException
+ */
+ public void merge(ILSMIOOperation operation) throws HyracksDataException, IndexException;
+}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
index 53a5c00..e98165b 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
@@ -22,39 +22,49 @@
import edu.uci.ics.hyracks.storage.am.common.api.IInMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOperationContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
public interface ILSMIndexInternal extends ILSMIndex {
+ public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
+ ISearchOperationCallback searchCallback);
- public void insertUpdateOrDelete(ITupleReference tuple, IIndexOperationContext ictx) throws HyracksDataException,
- IndexException;
+ public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException;
- public void search(IIndexCursor cursor, List<ILSMComponent> diskComponents, ISearchPredicate pred,
- IIndexOperationContext ictx, boolean includeMemComponent) throws HyracksDataException, IndexException;
+ public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred)
+ throws HyracksDataException, IndexException;
+
+ public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ throws HyracksDataException;
public ILSMComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException;
- public ILSMComponent merge(List<ILSMComponent> mergedComponents, ILSMIOOperation operation)
+ public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException, IndexException;
- public ILSMIOOperation createMergeOperation(ILSMIOOperationCallback callback) throws HyracksDataException,
- IndexException;
+ public ILSMComponent merge(List<ILSMComponent> mergedComponents, ILSMIOOperation operation)
+ throws HyracksDataException, IndexException;
public void addComponent(ILSMComponent index);
public void subsumeMergedComponents(ILSMComponent newComponent, List<ILSMComponent> mergedComponents);
- public List<ILSMComponent> getOperationalComponents(IIndexOperationContext ctx);
+ /**
+ * Populates the context's component holder with a snapshot of the components involved in the operation.
+ *
+ * @param ctx
+ * - the operation's context
+ */
+ public void getOperationalComponents(ILSMIndexOperationContext ctx);
public IInMemoryFreePageManager getInMemoryFreePageManager();
- public void resetMutableComponent() throws HyracksDataException;
-
- public ILSMComponent getMutableComponent();
-
public List<ILSMComponent> getImmutableComponents();
public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException;
+ public void setFlushStatus(ILSMIndex index, boolean needsFlush);
+
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java
index 9230fe4..864d0e7 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java
@@ -1,10 +1,15 @@
package edu.uci.ics.hyracks.storage.am.lsm.common.api;
+import java.util.List;
+
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
public interface ILSMIndexOperationContext extends IIndexOperationContext {
+ public List<ILSMComponent> getComponentHolder();
+
public ISearchOperationCallback getSearchOperationCallback();
+
public IModificationOperationCallback getModificationCallback();
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMOperationTracker.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMOperationTracker.java
index f07eaf9..c3f1f3e 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMOperationTracker.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMOperationTracker.java
@@ -4,6 +4,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMOperationType;
/**
* This interface exposes methods for tracking and setting the status of operations for the purpose
@@ -21,8 +22,8 @@
* then this method does not block and returns false.
* Otherwise, this method returns true, and the operation is considered 'active' in the index.
*/
- public boolean beforeOperation(ISearchOperationCallback searchCallback,
- IModificationOperationCallback modificationCallback, boolean tryOperation) throws HyracksDataException;
+ public void beforeOperation(LSMOperationType opType, ISearchOperationCallback searchCallback,
+ IModificationOperationCallback modificationCallback) throws HyracksDataException;
/**
* An {@link ILSMIndex} will call this method after an operation has left the index,
@@ -30,7 +31,7 @@
* After this method has been called, the operation is still considered 'active',
* until the issuer of the operation declares it completed by calling completeOperation().
*/
- public void afterOperation(ISearchOperationCallback searchCallback,
+ public void afterOperation(LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException;
/**
@@ -38,6 +39,6 @@
* The use of this method indicates that the operation is no longer 'active'
* for the purpose of coordinating flushes/merges.
*/
- public void completeOperation(ISearchOperationCallback searchCallback,
+ public void completeOperation(LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException;
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java
index d1d70e8..66ca349 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java
@@ -18,7 +18,7 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -31,27 +31,28 @@
protected final int memPageSize;
protected final int memNumPages;
- protected final ILSMFlushController flushController;
- protected final ILSMMergePolicy mergePolicy;
+ protected final ILSMMergePolicy mergePolicy;
protected final ILSMIOOperationScheduler ioScheduler;
protected final ILSMOperationTrackerFactory opTrackerFactory;
+ protected final ILSMIOOperationCallbackProvider ioOpCallbackProvider;
public AbstractLSMIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
- ILSMIOOperationScheduler ioScheduler) {
- this(opDesc, ctx, partition, DEFAULT_MEM_PAGE_SIZE, DEFAULT_MEM_NUM_PAGES, flushController, mergePolicy,
- opTrackerFactory, ioScheduler);
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
+ this(opDesc, ctx, partition, DEFAULT_MEM_PAGE_SIZE, DEFAULT_MEM_NUM_PAGES, mergePolicy, opTrackerFactory,
+ ioScheduler, ioOpCallbackProvider);
}
public AbstractLSMIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- int memPageSize, int memNumPages, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) {
+ int memPageSize, int memNumPages, ILSMMergePolicy mergePolicy,
+ ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
super(opDesc, ctx, partition);
this.memPageSize = memPageSize;
this.memNumPages = memNumPages;
- this.flushController = flushController;
this.mergePolicy = mergePolicy;
this.opTrackerFactory = opTrackerFactory;
this.ioScheduler = ioScheduler;
+ this.ioOpCallbackProvider = ioOpCallbackProvider;
}
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelperFactory.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelperFactory.java
index de48bff..a2f2a11 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelperFactory.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelperFactory.java
@@ -16,7 +16,7 @@
package edu.uci.ics.hyracks.storage.am.lsm.common.dataflow;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushControllerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicyProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -24,17 +24,21 @@
public abstract class AbstractLSMIndexDataflowHelperFactory implements IIndexDataflowHelperFactory {
protected static final long serialVersionUID = 1L;
- protected final ILSMFlushControllerProvider flushControllerProvider;
protected final ILSMMergePolicyProvider mergePolicyProvider;
protected final ILSMOperationTrackerFactory opTrackerFactory;
protected final ILSMIOOperationSchedulerProvider ioSchedulerProvider;
+ protected final ILSMIOOperationCallbackProvider ioOpCallbackProvider;
+ protected final int memPageSize;
+ protected final int memNumPages;
- public AbstractLSMIndexDataflowHelperFactory(ILSMFlushControllerProvider flushControllerProvider,
- ILSMMergePolicyProvider mergePolicyProvider, ILSMOperationTrackerFactory opTrackerFactory,
- ILSMIOOperationSchedulerProvider ioSchedulerProvider) {
- this.flushControllerProvider = flushControllerProvider;
+ public AbstractLSMIndexDataflowHelperFactory(ILSMMergePolicyProvider mergePolicyProvider,
+ ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
+ ILSMIOOperationCallbackProvider ioOpCallbackProvider, int memPageSize, int memNumPages) {
this.mergePolicyProvider = mergePolicyProvider;
this.opTrackerFactory = opTrackerFactory;
this.ioSchedulerProvider = ioSchedulerProvider;
+ this.ioOpCallbackProvider = ioOpCallbackProvider;
+ this.memPageSize = memPageSize;
+ this.memNumPages = memNumPages;
}
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/LSMIndexInsertUpdateDeleteOperatorNodePushable.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/LSMIndexInsertUpdateDeleteOperatorNodePushable.java
index 51f979f..baa9648 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/LSMIndexInsertUpdateDeleteOperatorNodePushable.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/LSMIndexInsertUpdateDeleteOperatorNodePushable.java
@@ -46,11 +46,7 @@
if (tupleFilter != null) {
frameTuple.reset(accessor, i);
if (!tupleFilter.accept(frameTuple)) {
- if (!lsmAccessor.tryNoOp()) {
- flushPartialFrame(lastFlushedTupleIndex, i);
- lastFlushedTupleIndex = (i == 0) ? 0 : i - 1;
- lsmAccessor.noOp();
- }
+ lsmAccessor.noOp();
continue;
}
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractImmutableLSMComponent.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractImmutableLSMComponent.java
new file mode 100644
index 0000000..b6fc2f7
--- /dev/null
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractImmutableLSMComponent.java
@@ -0,0 +1,69 @@
+package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+
+public abstract class AbstractImmutableLSMComponent implements ILSMComponent {
+
+ private ComponentState state;
+ private int readerCount;
+
+ private enum ComponentState {
+ READABLE,
+ READABLE_MERGING,
+ KILLED
+ }
+
+ public AbstractImmutableLSMComponent() {
+ state = ComponentState.READABLE;
+ readerCount = 0;
+ }
+
+ @Override
+ public synchronized boolean threadEnter(LSMOperationType opType) {
+ if (state == ComponentState.KILLED) {
+ return false;
+ }
+
+ switch (opType) {
+ case MODIFICATION:
+ case SEARCH:
+ readerCount++;
+ break;
+ case MERGE:
+ if (state == ComponentState.READABLE_MERGING) {
+ return false;
+ }
+ state = ComponentState.READABLE_MERGING;
+ readerCount++;
+ break;
+ default:
+ throw new UnsupportedOperationException("Unsupported operation " + opType);
+ }
+ return true;
+ }
+
+ @Override
+ public synchronized void threadExit(LSMOperationType opType, boolean failedOperation) throws HyracksDataException {
+ switch (opType) {
+ case MERGE:
+ if (failedOperation) {
+ state = ComponentState.READABLE;
+ }
+ case MODIFICATION:
+ case SEARCH:
+ readerCount--;
+
+ if (readerCount == 0 && state == ComponentState.READABLE_MERGING) {
+ destroy();
+ state = ComponentState.KILLED;
+ }
+ break;
+ default:
+ throw new UnsupportedOperationException("Unsupported operation " + opType);
+ }
+ }
+
+ protected abstract void destroy() throws HyracksDataException;
+
+}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMComponent.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMComponent.java
deleted file mode 100644
index e1a3c36..0000000
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMComponent.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright 2009-2012 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
-
-import java.util.concurrent.atomic.AtomicInteger;
-
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
-
-public abstract class AbstractLSMComponent implements ILSMComponent {
-
- private final AtomicInteger threadRef = new AtomicInteger();
- private LSMComponentState state;
-
- private final Object componentSync = new Object();
-
- @Override
- public void activate() {
-
- }
-
- @Override
- public void deactivate() {
-
- }
-
- @Override
- public void threadEnter() {
- threadRef.incrementAndGet();
- }
-
- @Override
- public void threadExit() {
- threadRef.decrementAndGet();
- }
-
- @Override
- public int getThreadReferenceCount() {
- return threadRef.get();
- }
-
- @Override
- public void setState(LSMComponentState state) {
- synchronized (componentSync) {
- this.state = state;
- }
- }
-
- @Override
- public LSMComponentState getState() {
- synchronized (componentSync) {
- return state;
- }
- }
-
- @Override
- public boolean negativeCompareAndSet(LSMComponentState compare, LSMComponentState update) {
- synchronized (componentSync) {
- if (state != compare) {
- state = update;
- return true;
- }
- }
- return false;
- }
-}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index 13505d8..3cc7196 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -15,17 +15,20 @@
package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
+import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.common.api.IInMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexInternal;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -39,6 +42,9 @@
public abstract class AbstractLSMIndex implements ILSMIndexInternal {
protected final ILSMHarness lsmHarness;
+ protected final ILSMIOOperationScheduler ioScheduler;
+ protected final ILSMIOOperationCallbackProvider ioOpCallbackProvider;
+
// In-memory components.
protected final IInMemoryFreePageManager memFreePageManager;
@@ -46,22 +52,27 @@
protected final IBufferCache diskBufferCache;
protected final ILSMIndexFileManager fileManager;
protected final IFileMapProvider diskFileMapProvider;
- protected final LinkedList<ILSMComponent> immutableComponents;
+ protected final AtomicReference<List<ILSMComponent>> componentsRef;
protected boolean isActivated;
+ private boolean needsFlush = false;
+
public AbstractLSMIndex(IInMemoryFreePageManager memFreePageManager, IBufferCache diskBufferCache,
- ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) {
+ ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, ILSMMergePolicy mergePolicy,
+ ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
this.memFreePageManager = memFreePageManager;
this.diskBufferCache = diskBufferCache;
this.diskFileMapProvider = diskFileMapProvider;
- this.immutableComponents = new LinkedList<ILSMComponent>();
this.fileManager = fileManager;
+ this.ioScheduler = ioScheduler;
+ this.ioOpCallbackProvider = ioOpCallbackProvider;
ILSMOperationTracker opTracker = opTrackerFactory.createOperationTracker(this);
- lsmHarness = new LSMHarness(this, flushController, mergePolicy, opTracker, ioScheduler);
+ lsmHarness = new LSMHarness(this, mergePolicy, opTracker);
isActivated = false;
+ componentsRef = new AtomicReference<List<ILSMComponent>>();
+ componentsRef.set(new LinkedList<ILSMComponent>());
}
protected void forceFlushDirtyPages(ITreeIndex treeIndex) throws HyracksDataException {
@@ -118,15 +129,30 @@
}
@Override
- public void addComponent(ILSMComponent index) {
- immutableComponents.addFirst(index);
+ public void addComponent(ILSMComponent c) {
+ List<ILSMComponent> oldList = componentsRef.get();
+ List<ILSMComponent> newList = new ArrayList<ILSMComponent>();
+ newList.add(c);
+ for (ILSMComponent oc : oldList) {
+ newList.add(oc);
+ }
+ componentsRef.set(newList);
}
@Override
public void subsumeMergedComponents(ILSMComponent newComponent, List<ILSMComponent> mergedComponents) {
- int firstComponentIndex = immutableComponents.indexOf(mergedComponents.get(0));
- immutableComponents.removeAll(mergedComponents);
- immutableComponents.add(firstComponentIndex, newComponent);
+ List<ILSMComponent> oldList = componentsRef.get();
+ List<ILSMComponent> newList = new ArrayList<ILSMComponent>();
+ int swapIndex = oldList.indexOf(mergedComponents.get(0));
+ int swapSize = mergedComponents.size();
+ for (int i = 0; i < oldList.size(); i++) {
+ if (i < swapIndex || i >= swapIndex + swapSize) {
+ newList.add(oldList.get(i));
+ } else if (i == swapIndex) {
+ newList.add(newComponent);
+ }
+ }
+ componentsRef.set(newList);
}
@Override
@@ -136,12 +162,17 @@
@Override
public List<ILSMComponent> getImmutableComponents() {
- return immutableComponents;
+ return componentsRef.get();
}
@Override
- public ILSMFlushController getFlushController() {
- return lsmHarness.getFlushController();
+ public void setFlushStatus(ILSMIndex index, boolean needsFlush) {
+ this.needsFlush = needsFlush;
+ }
+
+ @Override
+ public boolean getFlushStatus(ILSMIndex index) {
+ return needsFlush;
}
@Override
@@ -151,7 +182,7 @@
@Override
public ILSMIOOperationScheduler getIOScheduler() {
- return lsmHarness.getIOScheduler();
+ return ioScheduler;
}
@Override
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMutableLSMComponent.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMutableLSMComponent.java
new file mode 100644
index 0000000..1a6636a
--- /dev/null
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMutableLSMComponent.java
@@ -0,0 +1,104 @@
+package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+
+public abstract class AbstractMutableLSMComponent implements ILSMComponent {
+
+ private int readerCount;
+ private int writerCount;
+ private ComponentState state;
+
+ private enum ComponentState {
+ READABLE_WRITABLE,
+ READABLE_UNWRITABLE,
+ READABLE_UNWRITABLE_FLUSHING,
+ UNREADABLE_UNWRITABLE
+ }
+
+ public AbstractMutableLSMComponent() {
+ readerCount = 0;
+ writerCount = 0;
+ state = ComponentState.READABLE_WRITABLE;
+ }
+
+ @Override
+ public synchronized boolean threadEnter(LSMOperationType opType) throws InterruptedException {
+ switch (opType) {
+ case FORCE_MODIFICATION:
+ if (state != ComponentState.READABLE_WRITABLE && state != ComponentState.READABLE_UNWRITABLE) {
+ return false;
+ }
+ writerCount++;
+ break;
+ case MODIFICATION:
+ if (state != ComponentState.READABLE_WRITABLE) {
+ return false;
+ }
+ writerCount++;
+ break;
+ case SEARCH:
+ if (state == ComponentState.UNREADABLE_UNWRITABLE) {
+ return false;
+ }
+ readerCount++;
+ break;
+ case FLUSH:
+ if (state == ComponentState.READABLE_UNWRITABLE_FLUSHING
+ || state == ComponentState.UNREADABLE_UNWRITABLE) {
+ return false;
+ }
+
+ state = ComponentState.READABLE_UNWRITABLE_FLUSHING;
+ while (writerCount > 0) {
+ wait();
+ }
+ readerCount++;
+ break;
+ default:
+ throw new UnsupportedOperationException("Unsupported operation " + opType);
+ }
+ return true;
+ }
+
+ @Override
+ public synchronized void threadExit(LSMOperationType opType, boolean failedOperation) throws HyracksDataException {
+ switch (opType) {
+ case FORCE_MODIFICATION:
+ case MODIFICATION:
+ writerCount--;
+ if (state == ComponentState.READABLE_WRITABLE && isFull()) {
+ state = ComponentState.READABLE_UNWRITABLE;
+ }
+ break;
+ case SEARCH:
+ readerCount--;
+ if (state == ComponentState.UNREADABLE_UNWRITABLE && readerCount == 0) {
+ reset();
+ state = ComponentState.READABLE_WRITABLE;
+ } else if (state == ComponentState.READABLE_WRITABLE && isFull()) {
+ state = ComponentState.READABLE_UNWRITABLE;
+ }
+ break;
+ case FLUSH:
+ if (failedOperation) {
+ state = isFull() ? ComponentState.READABLE_UNWRITABLE : ComponentState.READABLE_WRITABLE;
+ }
+ readerCount--;
+ if (readerCount == 0) {
+ reset();
+ state = ComponentState.READABLE_WRITABLE;
+ } else if (state == ComponentState.READABLE_UNWRITABLE_FLUSHING) {
+ state = ComponentState.UNREADABLE_UNWRITABLE;
+ }
+ break;
+ default:
+ throw new UnsupportedOperationException("Unsupported operation " + opType);
+ }
+ notifyAll();
+ }
+
+ protected abstract boolean isFull();
+
+ protected abstract void reset() throws HyracksDataException;
+}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BlockingIOOperationCallback.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BlockingIOOperationCallbackWrapper.java
similarity index 69%
rename from hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BlockingIOOperationCallback.java
rename to hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BlockingIOOperationCallbackWrapper.java
index 6b6bb22..7558721 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BlockingIOOperationCallback.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BlockingIOOperationCallbackWrapper.java
@@ -7,10 +7,16 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-public class BlockingIOOperationCallback implements ILSMIOOperationCallback {
+public class BlockingIOOperationCallbackWrapper implements ILSMIOOperationCallback {
private boolean notified = false;
+ private final ILSMIOOperationCallback wrappedCallback;
+
+ public BlockingIOOperationCallbackWrapper(ILSMIOOperationCallback callback) {
+ this.wrappedCallback = callback;
+ }
+
public synchronized void waitForIO() throws InterruptedException {
if (!notified) {
this.wait();
@@ -20,18 +26,19 @@
@Override
public void beforeOperation(ILSMIOOperation operation) throws HyracksDataException {
- // Do nothing.
+ wrappedCallback.beforeOperation(operation);
}
@Override
public void afterOperation(ILSMIOOperation operation, List<ILSMComponent> oldComponents, ILSMComponent newComponent)
throws HyracksDataException {
- // Do nothing.
+ wrappedCallback.afterOperation(operation, oldComponents, newComponent);
}
@Override
public synchronized void afterFinalize(ILSMIOOperation operation, ILSMComponent newComponent)
throws HyracksDataException {
+ wrappedCallback.afterFinalize(operation, newComponent);
this.notifyAll();
notified = true;
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ConstantMergePolicy.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ConstantMergePolicy.java
index 4705566..1c72abf 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ConstantMergePolicy.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ConstantMergePolicy.java
@@ -18,37 +18,25 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
public class ConstantMergePolicy implements ILSMMergePolicy {
- private final ILSMIOOperationScheduler ioScheduler;
-
private final int threshold;
- public ConstantMergePolicy(ILSMIOOperationScheduler ioScheduler, int threshold) {
- this.ioScheduler = ioScheduler;
+ public ConstantMergePolicy(int threshold) {
this.threshold = threshold;
}
@Override
- public void diskComponentAdded(final ILSMIndex index, int totalNumDiskComponents) throws HyracksDataException, IndexException {
+ public void diskComponentAdded(final ILSMIndex index, int totalNumDiskComponents) throws HyracksDataException,
+ IndexException {
if (totalNumDiskComponents >= threshold) {
ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
- ILSMIOOperation op;
- try {
- op = accessor.createMergeOperation(NoOpIOOperationCallback.INSTANCE);
- if (op != null) {
- ioScheduler.scheduleOperation(op);
- }
- } catch (LSMMergeInProgressException e) {
- // Do nothing
- }
+ accessor.scheduleMerge(NoOpIOOperationCallback.INSTANCE);
}
}
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ConstantMergePolicyProvider.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ConstantMergePolicyProvider.java
index 1234137..b404c9b 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ConstantMergePolicyProvider.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ConstantMergePolicyProvider.java
@@ -1,7 +1,6 @@
package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicyProvider;
@@ -9,18 +8,15 @@
private static final long serialVersionUID = 1L;
- private final ILSMIOOperationSchedulerProvider schedulerProvider;
-
private final int threshold;
- public ConstantMergePolicyProvider(ILSMIOOperationSchedulerProvider schedulerProvider, int threshold) {
- this.schedulerProvider = schedulerProvider;
+ public ConstantMergePolicyProvider(int threshold) {
this.threshold = threshold;
}
@Override
public ILSMMergePolicy getMergePolicy(IHyracksTaskContext ctx) {
- return new ConstantMergePolicy(schedulerProvider.getIOScheduler(ctx), threshold);
+ return new ConstantMergePolicy(threshold);
}
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/FlushController.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/FlushController.java
deleted file mode 100644
index 6fd18f9..0000000
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/FlushController.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
-
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
-
-public class FlushController implements ILSMFlushController {
-
- private boolean needsFlush = false;
-
- @Override
- public void setFlushStatus(ILSMIndex index, boolean needsFlush) {
- this.needsFlush = needsFlush;
- }
-
- @Override
- public boolean getFlushStatus(ILSMIndex index) {
- return needsFlush;
- }
-}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/FlushControllerProvider.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/FlushControllerProvider.java
deleted file mode 100644
index e988f65..0000000
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/FlushControllerProvider.java
+++ /dev/null
@@ -1,16 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushControllerProvider;
-
-public class FlushControllerProvider implements ILSMFlushControllerProvider {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public ILSMFlushController getFlushController(IHyracksTaskContext ctx) {
- return new FlushController();
- }
-
-}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMFlushOperation.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMFlushOperation.java
index 0c70bc1..6ce1f08 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMFlushOperation.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMFlushOperation.java
@@ -1,44 +1,44 @@
package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
import java.util.Collections;
-import java.util.List;
+import java.util.Set;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
-import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
public class LSMFlushOperation implements ILSMIOOperation {
- private final ILSMIndex index;
+ private final ILSMIndexAccessorInternal accessor;
+ private final ILSMComponent flushingComponent;
private final FileReference flushTarget;
private final ILSMIOOperationCallback callback;
- public LSMFlushOperation(ILSMIndex index, FileReference flushTarget, ILSMIOOperationCallback callback) {
- this.index = index;
+ public LSMFlushOperation(ILSMIndexAccessorInternal accessor, ILSMComponent flushingComponent,
+ FileReference flushTarget, ILSMIOOperationCallback callback) {
+ this.accessor = accessor;
+ this.flushingComponent = flushingComponent;
this.flushTarget = flushTarget;
this.callback = callback;
}
@Override
- public List<IODeviceHandle> getReadDevices() {
- return Collections.emptyList();
+ public Set<IODeviceHandle> getReadDevices() {
+ return Collections.emptySet();
}
@Override
- public List<IODeviceHandle> getWriteDevices() {
- return Collections.singletonList(flushTarget.getDevideHandle());
+ public Set<IODeviceHandle> getWriteDevices() {
+ return Collections.singleton(flushTarget.getDeviceHandle());
}
@Override
public void perform() throws HyracksDataException, IndexException {
- ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
accessor.flush(this);
}
@@ -51,4 +51,11 @@
return flushTarget;
}
+ public ILSMIndexAccessorInternal getAccessor() {
+ return accessor;
+ }
+
+ public ILSMComponent getFlushingComponent() {
+ return flushingComponent;
+ }
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 7eef5f4..da87ed9 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -17,248 +17,215 @@
import java.util.ArrayList;
import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexInternal;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
-/**
- * Common code for synchronizing LSM operations like
- * updates/searches/flushes/merges on any {@link ILSMIndex}. This class only deals with
- * synchronizing LSM operations, and delegates the concrete implementations of
- * actual operations to {@link ILSMIndex} (passed in the constructor).
- * Concurrency behavior:
- * All operations except merge (insert/update/delete/search) are blocked during a flush.
- * During a merge, all operations (except another merge) can proceed concurrently.
- * A merge and a flush can proceed concurrently.
- */
public class LSMHarness implements ILSMHarness {
- protected final Logger LOGGER = Logger.getLogger(LSMHarness.class.getName());
-
- private ILSMIndexInternal lsmIndex;
-
- // All accesses to the LSM-Tree's on-disk components are synchronized on diskComponentsSync.
- private Object diskComponentsSync = new Object();
-
- // Flush and Merge Policies
- private final ILSMFlushController flushController;
+ private final ILSMIndexInternal lsmIndex;
private final ILSMMergePolicy mergePolicy;
private final ILSMOperationTracker opTracker;
- private final ILSMIOOperationScheduler ioScheduler;
- public LSMHarness(ILSMIndexInternal lsmIndex, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler) {
+ public LSMHarness(ILSMIndexInternal lsmIndex, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker) {
this.lsmIndex = lsmIndex;
this.opTracker = opTracker;
- this.flushController = flushController;
this.mergePolicy = mergePolicy;
- this.ioScheduler = ioScheduler;
}
- private void threadExit(ILSMIndexOperationContext opCtx) throws HyracksDataException {
- if (!lsmIndex.getFlushController().getFlushStatus(lsmIndex) && lsmIndex.getInMemoryFreePageManager().isFull()) {
- lsmIndex.getFlushController().setFlushStatus(lsmIndex, true);
+ private void threadExit(ILSMIndexOperationContext opCtx, LSMOperationType opType) throws HyracksDataException {
+ if (!lsmIndex.getFlushStatus(lsmIndex) && lsmIndex.getInMemoryFreePageManager().isFull()) {
+ lsmIndex.setFlushStatus(lsmIndex, true);
}
- opTracker.afterOperation(opCtx.getSearchOperationCallback(), opCtx.getModificationCallback());
+ opTracker.afterOperation(opType, opCtx.getSearchOperationCallback(), opCtx.getModificationCallback());
}
- @Override
- public boolean insertUpdateOrDelete(ITupleReference tuple, ILSMIndexOperationContext ctx, boolean tryOperation)
- throws HyracksDataException, IndexException {
- if (!opTracker.beforeOperation(ctx.getSearchOperationCallback(), ctx.getModificationCallback(), tryOperation)) {
- return false;
- }
- try {
- lsmIndex.insertUpdateOrDelete(tuple, ctx);
- } finally {
- threadExit(ctx);
+ private boolean getAndEnterComponents(ILSMIndexOperationContext ctx, LSMOperationType opType, boolean tryOperation)
+ throws HyracksDataException {
+ int numEntered = 0;
+ boolean entranceSuccessful = false;
+ List<ILSMComponent> entered = new ArrayList<ILSMComponent>();
+
+ while (!entranceSuccessful) {
+ entered.clear();
+ lsmIndex.getOperationalComponents(ctx);
+ List<ILSMComponent> components = ctx.getComponentHolder();
+ try {
+ for (ILSMComponent c : components) {
+ if (!c.threadEnter(opType)) {
+ break;
+ }
+ numEntered++;
+ entered.add(c);
+ }
+ entranceSuccessful = numEntered == components.size();
+ } catch (InterruptedException e) {
+ entranceSuccessful = false;
+ throw new HyracksDataException(e);
+ } finally {
+ if (!entranceSuccessful) {
+ for (ILSMComponent c : components) {
+ if (numEntered <= 0) {
+ break;
+ }
+ c.threadExit(opType, true);
+ numEntered--;
+ }
+ }
+ }
+ if (tryOperation && !entranceSuccessful) {
+ return false;
+ }
}
+ opTracker.beforeOperation(opType, ctx.getSearchOperationCallback(), ctx.getModificationCallback());
return true;
}
- @Override
- public boolean noOp(ILSMIndexOperationContext ctx, boolean tryOperation) throws HyracksDataException {
- if (!opTracker.beforeOperation(ctx.getSearchOperationCallback(), ctx.getModificationCallback(), tryOperation)) {
- return false;
- }
- threadExit(ctx);
- return true;
- }
-
- @Override
- public void flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Flushing LSM-Index: " + lsmIndex);
- }
- ILSMComponent newComponent = null;
+ private void exitComponents(ILSMIndexOperationContext ctx, LSMOperationType opType, boolean failedOperation)
+ throws HyracksDataException {
try {
- operation.getCallback().beforeOperation(operation);
- newComponent = lsmIndex.flush(operation);
- operation.getCallback().afterOperation(operation, null, newComponent);
-
- // The implementation of this call must take any necessary steps to make
- // the new component permanent, and mark it as valid (usually this means
- // forcing all pages of the tree to disk, possibly with some extra
- // information to mark the tree as valid).
- lsmIndex.markAsValid(newComponent);
+ for (ILSMComponent c : ctx.getComponentHolder()) {
+ c.threadExit(opType, failedOperation);
+ }
} finally {
- operation.getCallback().afterFinalize(operation, newComponent);
+ threadExit(ctx, opType);
}
- lsmIndex.resetMutableComponent();
- synchronized (diskComponentsSync) {
- lsmIndex.addComponent(newComponent);
- mergePolicy.diskComponentAdded(lsmIndex, lsmIndex.getImmutableComponents().size());
- }
-
- // Unblock entering threads waiting for the flush
- flushController.setFlushStatus(lsmIndex, false);
}
@Override
- public List<ILSMComponent> search(IIndexCursor cursor, ISearchPredicate pred, ILSMIndexOperationContext ctx,
- boolean includeMutableComponent) throws HyracksDataException, IndexException {
- // If the search doesn't include the in-memory component, then we don't have
- // to synchronize with a flush.
- if (includeMutableComponent) {
- opTracker.beforeOperation(ctx.getSearchOperationCallback(), ctx.getModificationCallback(), false);
- }
-
- // Get a snapshot of the current on-disk Trees.
- // If includeMutableComponent is true, then no concurrent
- // flush can add another on-disk Tree (due to threadEnter());
- // If includeMutableComponent is false, then it is possible that a concurrent
- // flush adds another on-disk Tree.
- // Since this mode is only used for merging trees, it doesn't really
- // matter if the merge excludes the new on-disk Tree.
- List<ILSMComponent> operationalComponents;
- synchronized (diskComponentsSync) {
- operationalComponents = lsmIndex.getOperationalComponents(ctx);
- }
- for (ILSMComponent c : operationalComponents) {
- c.threadEnter();
- }
-
- lsmIndex.search(cursor, operationalComponents, pred, ctx, includeMutableComponent);
- return operationalComponents;
- }
-
- @Override
- public ILSMIOOperation createMergeOperation(ILSMIOOperationCallback callback) throws HyracksDataException,
+ public void forceModify(ILSMIndexOperationContext ctx, ITupleReference tuple) throws HyracksDataException,
IndexException {
- ILSMIOOperation mergeOp = lsmIndex.createMergeOperation(callback);
- return mergeOp;
+ LSMOperationType opType = LSMOperationType.FORCE_MODIFICATION;
+ modify(ctx, false, tuple, opType);
}
@Override
- public void merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Merging LSM-Index: " + lsmIndex);
+ public boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple)
+ throws HyracksDataException, IndexException {
+ LSMOperationType opType = LSMOperationType.MODIFICATION;
+ return modify(ctx, tryOperation, tuple, opType);
+ }
+
+ private boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple,
+ LSMOperationType opType) throws HyracksDataException, IndexException {
+ if (!getAndEnterComponents(ctx, opType, tryOperation)) {
+ return false;
+ }
+ try {
+ lsmIndex.modify(ctx, tuple);
+ } finally {
+ exitComponents(ctx, opType, false);
}
+ return true;
+ }
+
+ @Override
+ public void search(ILSMIndexOperationContext ctx, IIndexCursor cursor, ISearchPredicate pred)
+ throws HyracksDataException, IndexException {
+ LSMOperationType opType = LSMOperationType.SEARCH;
+ getAndEnterComponents(ctx, opType, false);
+ try {
+ lsmIndex.search(ctx, cursor, pred);
+ } catch (HyracksDataException e) {
+ exitComponents(ctx, opType, true);
+ throw e;
+ } catch (IndexException e) {
+ exitComponents(ctx, opType, true);
+ throw e;
+ }
+ }
+
+ @Override
+ public void endSearch(ILSMIndexOperationContext ctx) throws HyracksDataException {
+ if (ctx.getOperation() == IndexOperation.SEARCH) {
+ exitComponents(ctx, LSMOperationType.SEARCH, false);
+ }
+ }
+
+ @Override
+ public void noOp(ILSMIndexOperationContext ctx) throws HyracksDataException {
+ LSMOperationType opType = LSMOperationType.NOOP;
+ opTracker.beforeOperation(opType, ctx.getSearchOperationCallback(), ctx.getModificationCallback());
+ threadExit(ctx, opType);
+ }
+
+ @Override
+ public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ throws HyracksDataException {
+ if (!getAndEnterComponents(ctx, LSMOperationType.FLUSH, true)) {
+ return;
+ }
+ lsmIndex.setFlushStatus(lsmIndex, false);
+ lsmIndex.scheduleFlush(ctx, callback);
+ }
+
+ @Override
+ public void flush(ILSMIndexOperationContext ctx, ILSMIOOperation operation) throws HyracksDataException,
+ IndexException {
+ operation.getCallback().beforeOperation(operation);
+ ILSMComponent newComponent = lsmIndex.flush(operation);
+ operation.getCallback().afterOperation(operation, null, newComponent);
+ lsmIndex.markAsValid(newComponent);
+ operation.getCallback().afterFinalize(operation, newComponent);
+
+ lsmIndex.addComponent(newComponent);
+ int numComponents = lsmIndex.getImmutableComponents().size();
+
+ mergePolicy.diskComponentAdded(lsmIndex, numComponents);
+ exitComponents(ctx, LSMOperationType.FLUSH, false);
+ }
+
+ @Override
+ public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ throws HyracksDataException, IndexException {
+ LSMOperationType opType = LSMOperationType.MERGE;
+ if (!getAndEnterComponents(ctx, opType, false)) {
+ return;
+ }
+ if (ctx.getComponentHolder().size() > 1) {
+ lsmIndex.scheduleMerge(ctx, callback);
+ } else {
+ exitComponents(ctx, opType, true);
+ }
+ }
+
+ @Override
+ public void merge(ILSMIndexOperationContext ctx, ILSMIOOperation operation) throws HyracksDataException,
+ IndexException {
List<ILSMComponent> mergedComponents = new ArrayList<ILSMComponent>();
- ILSMComponent newComponent = null;
- try {
- operation.getCallback().beforeOperation(operation);
- newComponent = lsmIndex.merge(mergedComponents, operation);
- operation.getCallback().afterOperation(operation, mergedComponents, newComponent);
-
- // No merge happened.
- if (newComponent == null) {
- return;
- }
-
- // TODO: Move this to just before the merge cleanup and remove latching on disk components
- // The implementation of this call must take any necessary steps to make
- // the new component permanent, and mark it as valid (usually this means
- // forcing all pages of the tree to disk, possibly with some extra
- // information to mark the tree as valid).
- lsmIndex.markAsValid(newComponent);
- } finally {
- operation.getCallback().afterFinalize(operation, newComponent);
- }
-
- // Remove the old components from the list, and add the new merged component(s).
- try {
- synchronized (diskComponentsSync) {
- lsmIndex.subsumeMergedComponents(newComponent, mergedComponents);
- }
- } finally {
- // Cleanup merged components in case there are no more searchers accessing them.
- for (ILSMComponent c : mergedComponents) {
- c.setState(LSMComponentState.DONE_MERGING);
- if (c.getThreadReferenceCount() == 0) {
- c.destroy();
- }
- }
- }
+ operation.getCallback().beforeOperation(operation);
+ ILSMComponent newComponent = lsmIndex.merge(mergedComponents, operation);
+ ctx.getComponentHolder().addAll(mergedComponents);
+ operation.getCallback().afterOperation(operation, mergedComponents, newComponent);
+ lsmIndex.markAsValid(newComponent);
+ operation.getCallback().afterFinalize(operation, newComponent);
+ lsmIndex.subsumeMergedComponents(newComponent, mergedComponents);
+ exitComponents(ctx, LSMOperationType.MERGE, false);
}
@Override
- public void closeSearchCursor(List<ILSMComponent> operationalComponents, boolean includeMutableComponent,
- ILSMIndexOperationContext ctx) throws HyracksDataException {
- // TODO: we should not worry about the mutable component.
- if (includeMutableComponent) {
- threadExit(ctx);
- }
- try {
- for (ILSMComponent c : operationalComponents) {
- c.threadExit();
- }
- } finally {
- for (ILSMComponent c : operationalComponents) {
- if (c.getState() == LSMComponentState.DONE_MERGING && c.getThreadReferenceCount() == 0) {
- c.destroy();
- }
- }
- }
- }
-
- @Override
- public void addBulkLoadedComponent(ILSMComponent index) throws HyracksDataException, IndexException {
- // The implementation of this call must take any necessary steps to make
- // the new component permanent, and mark it as valid (usually this means
- // forcing all pages of the tree to disk, possibly with some extra
- // information to mark the tree as valid).
- lsmIndex.markAsValid(index);
- synchronized (diskComponentsSync) {
- lsmIndex.addComponent(index);
- mergePolicy.diskComponentAdded(lsmIndex, lsmIndex.getImmutableComponents().size());
- }
- }
-
- @Override
- public ILSMFlushController getFlushController() {
- return flushController;
+ public void addBulkLoadedComponent(ILSMComponent c) throws HyracksDataException, IndexException {
+ lsmIndex.markAsValid(c);
+ lsmIndex.addComponent(c);
+ int numComponents = lsmIndex.getImmutableComponents().size();
+ mergePolicy.diskComponentAdded(lsmIndex, numComponents);
}
@Override
public ILSMOperationTracker getOperationTracker() {
return opTracker;
}
-
- @Override
- public ILSMIOOperationScheduler getIOScheduler() {
- return ioScheduler;
- }
-
- @Override
- public ILSMIndex getIndex() {
- return lsmIndex;
- }
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
index 3b66c72..92d3679 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
@@ -83,7 +83,7 @@
rangeCursors = null;
if (searcherRefCount != null) {
- lsmHarness.closeSearchCursor(operationalComponents, includeMemComponent, opCtx);
+ lsmHarness.endSearch(opCtx);
}
}
@@ -115,7 +115,7 @@
}
rangeCursors = null;
} finally {
- lsmHarness.closeSearchCursor(operationalComponents, includeMemComponent, opCtx);
+ lsmHarness.endSearch(opCtx);
}
}
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMMergeInProgressException.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMMergeInProgressException.java
deleted file mode 100644
index ee239d3..0000000
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMMergeInProgressException.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright 2009-2012 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
-
-import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
-
-public class LSMMergeInProgressException extends TreeIndexException {
-
- private static final long serialVersionUID = 1L;
-
- public LSMMergeInProgressException(Exception e) {
- super(e);
- }
-
- public LSMMergeInProgressException(String message) {
- super(message);
- }
-
-}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMOperationType.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMOperationType.java
new file mode 100644
index 0000000..981cefe
--- /dev/null
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMOperationType.java
@@ -0,0 +1,10 @@
+package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
+
+public enum LSMOperationType {
+ SEARCH,
+ MODIFICATION,
+ FORCE_MODIFICATION,
+ FLUSH,
+ MERGE,
+ NOOP
+}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
index d752ab4..7cc29a5 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
@@ -24,10 +24,10 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
-public abstract class LSMTreeIndexAccessor implements ILSMIndexAccessor {
+public abstract class LSMTreeIndexAccessor implements ILSMIndexAccessorInternal {
protected ILSMHarness lsmHarness;
protected ILSMIndexOperationContext ctx;
@@ -39,88 +39,108 @@
@Override
public void insert(ITupleReference tuple) throws HyracksDataException, IndexException {
ctx.setOperation(IndexOperation.INSERT);
- lsmHarness.insertUpdateOrDelete(tuple, ctx, false);
+ lsmHarness.modify(ctx, false, tuple);
}
@Override
public void update(ITupleReference tuple) throws HyracksDataException, IndexException {
// Update is the same as insert.
ctx.setOperation(IndexOperation.UPDATE);
- lsmHarness.insertUpdateOrDelete(tuple, ctx, false);
+ lsmHarness.modify(ctx, false, tuple);
}
@Override
public void delete(ITupleReference tuple) throws HyracksDataException, IndexException {
ctx.setOperation(IndexOperation.DELETE);
- lsmHarness.insertUpdateOrDelete(tuple, ctx, false);
+ lsmHarness.modify(ctx, false, tuple);
}
@Override
public void upsert(ITupleReference tuple) throws HyracksDataException, IndexException {
ctx.setOperation(IndexOperation.UPSERT);
- lsmHarness.insertUpdateOrDelete(tuple, ctx, false);
+ lsmHarness.modify(ctx, false, tuple);
}
@Override
public boolean tryInsert(ITupleReference tuple) throws HyracksDataException, IndexException {
ctx.setOperation(IndexOperation.INSERT);
- return lsmHarness.insertUpdateOrDelete(tuple, ctx, true);
+ return lsmHarness.modify(ctx, true, tuple);
}
@Override
public boolean tryDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
ctx.setOperation(IndexOperation.DELETE);
- return lsmHarness.insertUpdateOrDelete(tuple, ctx, true);
+ return lsmHarness.modify(ctx, true, tuple);
}
@Override
public boolean tryUpdate(ITupleReference tuple) throws HyracksDataException, IndexException {
// Update is the same as insert.
ctx.setOperation(IndexOperation.UPDATE);
- return lsmHarness.insertUpdateOrDelete(tuple, ctx, true);
+ return lsmHarness.modify(ctx, true, tuple);
}
@Override
public boolean tryUpsert(ITupleReference tuple) throws HyracksDataException, IndexException {
ctx.setOperation(IndexOperation.UPSERT);
- return lsmHarness.insertUpdateOrDelete(tuple, ctx, true);
+ return lsmHarness.modify(ctx, true, tuple);
}
-
+
@Override
public void search(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException, IndexException {
ctx.setOperation(IndexOperation.SEARCH);
- lsmHarness.search(cursor, searchPred, ctx, true);
+ lsmHarness.search(ctx, cursor, searchPred);
}
@Override
public void flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
- lsmHarness.flush(operation);
+ lsmHarness.flush(ctx, operation);
}
@Override
public void merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
- lsmHarness.merge(operation);
+ ctx.setOperation(IndexOperation.MERGE);
+ lsmHarness.merge(ctx, operation);
}
@Override
public void physicalDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
ctx.setOperation(IndexOperation.PHYSICALDELETE);
- lsmHarness.insertUpdateOrDelete(tuple, ctx, false);
+ lsmHarness.modify(ctx, false, tuple);
}
@Override
- public ILSMIOOperation createMergeOperation(ILSMIOOperationCallback callback) throws HyracksDataException,
- IndexException {
- return lsmHarness.createMergeOperation(callback);
+ public void scheduleFlush(ILSMIOOperationCallback callback) throws HyracksDataException {
+ ctx.setOperation(IndexOperation.FLUSH);
+ lsmHarness.scheduleFlush(ctx, callback);
}
-
+
@Override
- public boolean tryNoOp() throws HyracksDataException {
- return lsmHarness.noOp(ctx, true);
+ public void scheduleMerge(ILSMIOOperationCallback callback) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.MERGE);
+ lsmHarness.scheduleMerge(ctx, callback);
}
@Override
public void noOp() throws HyracksDataException {
- lsmHarness.noOp(ctx, false);
+ lsmHarness.noOp(ctx);
+ }
+
+ @Override
+ public void forcePhysicalDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.PHYSICALDELETE);
+ lsmHarness.forceModify(ctx, tuple);
+ }
+
+ @Override
+ public void forceInsert(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.INSERT);
+ lsmHarness.forceModify(ctx, tuple);
+ }
+
+ @Override
+ public void forceDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.DELETE);
+ lsmHarness.forceModify(ctx, tuple);
}
}
\ No newline at end of file
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/NoOpIOOperationCallback.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/NoOpIOOperationCallback.java
index 74cf4f17..999568e 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/NoOpIOOperationCallback.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/NoOpIOOperationCallback.java
@@ -6,8 +6,10 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
-public enum NoOpIOOperationCallback implements ILSMIOOperationCallback {
+public enum NoOpIOOperationCallback implements ILSMIOOperationCallback, ILSMIOOperationCallbackProvider {
INSTANCE;
@Override
@@ -25,4 +27,9 @@
public void afterFinalize(ILSMIOOperation operation, ILSMComponent newComponent) throws HyracksDataException {
// Do nothing.
}
+
+ @Override
+ public ILSMIOOperationCallback getIOOperationCallback(ILSMIndex index) {
+ return INSTANCE;
+ }
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/NoOpOperationTrackerFactory.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/NoOpOperationTrackerFactory.java
index e4c584a..97ec50e 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/NoOpOperationTrackerFactory.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/NoOpOperationTrackerFactory.java
@@ -4,7 +4,6 @@
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -24,21 +23,18 @@
return new ILSMOperationTracker() {
@Override
- public void completeOperation(ISearchOperationCallback searchCallback,
+ public void completeOperation(LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException {
// Do nothing.
}
@Override
- public boolean beforeOperation(ISearchOperationCallback searchCallback,
- IModificationOperationCallback modificationCallback, boolean tryOperation)
- throws HyracksDataException {
- // Do nothing.
- return true;
+ public void beforeOperation(LSMOperationType opType, ISearchOperationCallback searchCallback,
+ IModificationOperationCallback modificationCallback) throws HyracksDataException {
}
@Override
- public void afterOperation(ISearchOperationCallback searchCallback,
+ public void afterOperation(LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException {
// Do nothing.
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ReferenceCountingOperationTracker.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ReferenceCountingOperationTracker.java
deleted file mode 100644
index 2f41464..0000000
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ReferenceCountingOperationTracker.java
+++ /dev/null
@@ -1,83 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
-import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
-import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
-
-public class ReferenceCountingOperationTracker implements ILSMOperationTracker {
-
- private int threadRefCount = 0;
- private final ILSMIndex index;
- private final FlushOperationCallback FLUSHCALLBACK_INSTANCE = new FlushOperationCallback();
-
- public ReferenceCountingOperationTracker(ILSMIndex index) {
- this.index = index;
- }
-
- @Override
- public synchronized boolean beforeOperation(ISearchOperationCallback searchCallback,
- IModificationOperationCallback modificationCallback, boolean tryOperation) throws HyracksDataException {
- // Wait for pending flushes to complete.
- // If flushFlag is set, then the flush is queued to occur by the last exiting thread.
- // This operation should wait for that flush to occur before proceeding.
- if (index.getFlushController().getFlushStatus(index)) {
- if (tryOperation) {
- return false;
- }
- try {
- this.wait();
- } catch (InterruptedException e) {
- throw new HyracksDataException(e);
- }
- }
- threadRefCount++;
- return true;
- }
-
- @Override
- public void afterOperation(ISearchOperationCallback searchCallback,
- IModificationOperationCallback modificationCallback) throws HyracksDataException {
- // The operation is considered inactive, immediately after leaving the index.
- completeOperation(searchCallback, modificationCallback);
- }
-
- @Override
- public synchronized void completeOperation(ISearchOperationCallback searchCallback,
- IModificationOperationCallback modificationCallback) throws HyracksDataException {
- threadRefCount--;
-
- // Flush will only be handled by last exiting thread.
- if (index.getFlushController().getFlushStatus(index) && threadRefCount == 0) {
- ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
- index.getIOScheduler().scheduleOperation(accessor.createFlushOperation(FLUSHCALLBACK_INSTANCE));
- }
- }
-
- private class FlushOperationCallback implements ILSMIOOperationCallback {
- @Override
- public void beforeOperation(ILSMIOOperation operation) throws HyracksDataException {
- // Do nothing.
- }
-
- @Override
- public void afterOperation(ILSMIOOperation operation, List<ILSMComponent> oldComponents,
- ILSMComponent newComponent) throws HyracksDataException {
- // Do nothing.
- }
-
- @Override
- public void afterFinalize(ILSMIOOperation operation, ILSMComponent newComponent) throws HyracksDataException {
- ReferenceCountingOperationTracker.this.notifyAll();
- }
- }
-}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ImmediateScheduler.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/SynchronousScheduler.java
similarity index 89%
rename from hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ImmediateScheduler.java
rename to hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/SynchronousScheduler.java
index 03a41dc..9bbd394 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ImmediateScheduler.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/SynchronousScheduler.java
@@ -5,7 +5,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
-public enum ImmediateScheduler implements ILSMIOOperationScheduler {
+public enum SynchronousScheduler implements ILSMIOOperationScheduler {
INSTANCE;
@Override
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ImmediateSchedulerProvider.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/SynchronousSchedulerProvider.java
similarity index 75%
rename from hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ImmediateSchedulerProvider.java
rename to hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/SynchronousSchedulerProvider.java
index 91c0a2d..72d9d1d 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ImmediateSchedulerProvider.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/SynchronousSchedulerProvider.java
@@ -4,12 +4,12 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
-public enum ImmediateSchedulerProvider implements ILSMIOOperationSchedulerProvider {
+public enum SynchronousSchedulerProvider implements ILSMIOOperationSchedulerProvider {
INSTANCE;
@Override
public ILSMIOOperationScheduler getIOScheduler(IHyracksTaskContext ctx) {
- return ImmediateScheduler.INSTANCE;
+ return SynchronousScheduler.INSTANCE;
}
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/RefCountingOperationTrackerFactory.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ThreadCountingOperationTrackerFactory.java
similarity index 60%
rename from hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/RefCountingOperationTrackerFactory.java
rename to hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ThreadCountingOperationTrackerFactory.java
index aa4e551..3b4b00f 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/RefCountingOperationTrackerFactory.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ThreadCountingOperationTrackerFactory.java
@@ -4,18 +4,18 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
-public class RefCountingOperationTrackerFactory implements ILSMOperationTrackerFactory {
+public class ThreadCountingOperationTrackerFactory implements ILSMOperationTrackerFactory {
private static final long serialVersionUID = 1L;
- public static RefCountingOperationTrackerFactory INSTANCE = new RefCountingOperationTrackerFactory();
+ public static ThreadCountingOperationTrackerFactory INSTANCE = new ThreadCountingOperationTrackerFactory();
@Override
public ILSMOperationTracker createOperationTracker(ILSMIndex index) {
- return new ReferenceCountingOperationTracker(index);
+ return new ThreadCountingTracker(index);
}
// Enforce singleton.
- private RefCountingOperationTrackerFactory() {
+ private ThreadCountingOperationTrackerFactory() {
}
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java
new file mode 100644
index 0000000..7fee06e
--- /dev/null
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java
@@ -0,0 +1,49 @@
+package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+
+public class ThreadCountingTracker implements ILSMOperationTracker {
+ private final AtomicInteger threadRefCount;
+ private final ILSMIndex index;
+
+ public ThreadCountingTracker(ILSMIndex index) {
+ this.index = index;
+ this.threadRefCount = new AtomicInteger();
+ }
+
+ @Override
+ public void beforeOperation(LSMOperationType opType, ISearchOperationCallback searchCallback,
+ IModificationOperationCallback modificationCallback) throws HyracksDataException {
+ if (opType == LSMOperationType.MODIFICATION) {
+ threadRefCount.incrementAndGet();
+ }
+ }
+
+ @Override
+ public void afterOperation(LSMOperationType opType, ISearchOperationCallback searchCallback,
+ IModificationOperationCallback modificationCallback) throws HyracksDataException {
+ // The operation is considered inactive, immediately after leaving the index.
+ completeOperation(opType, searchCallback, modificationCallback);
+ }
+
+ @Override
+ public void completeOperation(LSMOperationType opType, ISearchOperationCallback searchCallback,
+ IModificationOperationCallback modificationCallback) throws HyracksDataException {
+ // Flush will only be handled by last exiting thread.
+ if (opType == LSMOperationType.MODIFICATION) {
+ if (threadRefCount.decrementAndGet() == 0 && index.getFlushStatus(index)) {
+ ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ accessor.scheduleFlush(NoOpIOOperationCallback.INSTANCE);
+ }
+ }
+ }
+}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java
index f7165b8..3d8b391 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java
@@ -23,7 +23,7 @@
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -40,17 +40,18 @@
public final class LSMInvertedIndexDataflowHelper extends AbstractLSMIndexDataflowHelper {
public LSMInvertedIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) {
- this(opDesc, ctx, partition, DEFAULT_MEM_PAGE_SIZE, DEFAULT_MEM_NUM_PAGES, flushController, mergePolicy,
- opTrackerFactory, ioScheduler);
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
+ this(opDesc, ctx, partition, DEFAULT_MEM_PAGE_SIZE, DEFAULT_MEM_NUM_PAGES, mergePolicy, opTrackerFactory,
+ ioScheduler, ioOpCallbackProvider);
}
public LSMInvertedIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- int memPageSize, int memNumPages, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) {
- super(opDesc, ctx, partition, memPageSize, memNumPages, flushController, mergePolicy, opTrackerFactory,
- ioScheduler);
+ int memPageSize, int memNumPages, ILSMMergePolicy mergePolicy,
+ ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
+ super(opDesc, ctx, partition, memPageSize, memNumPages, mergePolicy, opTrackerFactory, ioScheduler,
+ ioOpCallbackProvider);
}
@Override
@@ -68,8 +69,8 @@
diskFileMapProvider, invIndexOpDesc.getInvListsTypeTraits(),
invIndexOpDesc.getInvListsComparatorFactories(), invIndexOpDesc.getTokenTypeTraits(),
invIndexOpDesc.getTokenComparatorFactories(), invIndexOpDesc.getTokenizerFactory(),
- diskBufferCache, ctx.getIOManager(), file.getFile().getPath(), flushController, mergePolicy,
- opTrackerFactory, ioScheduler, partition);
+ diskBufferCache, ctx.getIOManager(), file.getFile().getPath(), mergePolicy, opTrackerFactory,
+ ioScheduler, ioOpCallbackProvider, partition);
return invIndex;
} catch (IndexException e) {
throw new HyracksDataException(e);
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
index e3c8d0f..9796ebc 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
@@ -18,7 +18,7 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushControllerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicyProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -28,18 +28,18 @@
private static final long serialVersionUID = 1L;
- public LSMInvertedIndexDataflowHelperFactory(ILSMFlushControllerProvider flushControllerProvider,
- ILSMMergePolicyProvider mergePolicyProvider, ILSMOperationTrackerFactory opTrackerProvider,
- ILSMIOOperationSchedulerProvider ioSchedulerProvider) {
- super(flushControllerProvider, mergePolicyProvider, opTrackerProvider, ioSchedulerProvider);
+ public LSMInvertedIndexDataflowHelperFactory(ILSMMergePolicyProvider mergePolicyProvider,
+ ILSMOperationTrackerFactory opTrackerProvider, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
+ ILSMIOOperationCallbackProvider ioOpCallbackProvider, int memPageSize, int memNumPages) {
+ super(mergePolicyProvider, opTrackerProvider, ioSchedulerProvider, ioOpCallbackProvider, memPageSize,
+ memNumPages);
}
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition) {
- return new LSMInvertedIndexDataflowHelper(opDesc, ctx, partition,
- flushControllerProvider.getFlushController(ctx), mergePolicyProvider.getMergePolicy(ctx),
- opTrackerFactory, ioSchedulerProvider.getIOScheduler(ctx));
+ return new LSMInvertedIndexDataflowHelper(opDesc, ctx, partition, memPageSize, memNumPages, mergePolicyProvider.getMergePolicy(ctx),
+ opTrackerFactory, ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackProvider);
}
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java
index aa20c05..c5b4f07 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java
@@ -23,7 +23,7 @@
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -40,18 +40,18 @@
public final class PartitionedLSMInvertedIndexDataflowHelper extends AbstractLSMIndexDataflowHelper {
public PartitionedLSMInvertedIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) {
- this(opDesc, ctx, partition, DEFAULT_MEM_PAGE_SIZE, DEFAULT_MEM_NUM_PAGES, flushController, mergePolicy,
- opTrackerFactory, ioScheduler);
+ int partition, ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
+ this(opDesc, ctx, partition, DEFAULT_MEM_PAGE_SIZE, DEFAULT_MEM_NUM_PAGES, mergePolicy, opTrackerFactory,
+ ioScheduler, ioOpCallbackProvider);
}
public PartitionedLSMInvertedIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition, int memPageSize, int memNumPages, ILSMFlushController flushController,
- ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
- ILSMIOOperationScheduler ioScheduler) {
- super(opDesc, ctx, partition, memPageSize, memNumPages, flushController, mergePolicy, opTrackerFactory,
- ioScheduler);
+ int partition, int memPageSize, int memNumPages, ILSMMergePolicy mergePolicy,
+ ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
+ super(opDesc, ctx, partition, memPageSize, memNumPages, mergePolicy, opTrackerFactory, ioScheduler,
+ ioOpCallbackProvider);
}
@Override
@@ -69,8 +69,8 @@
memFreePageManager, diskFileMapProvider, invIndexOpDesc.getInvListsTypeTraits(),
invIndexOpDesc.getInvListsComparatorFactories(), invIndexOpDesc.getTokenTypeTraits(),
invIndexOpDesc.getTokenComparatorFactories(), invIndexOpDesc.getTokenizerFactory(),
- diskBufferCache, ctx.getIOManager(), file.getFile().getPath(), flushController, mergePolicy,
- opTrackerFactory, ioScheduler, partition);
+ diskBufferCache, ctx.getIOManager(), file.getFile().getPath(), mergePolicy, opTrackerFactory,
+ ioScheduler, ioOpCallbackProvider, partition);
return invIndex;
} catch (IndexException e) {
throw new HyracksDataException(e);
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
index 34367db..8a8aad2 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
@@ -18,7 +18,7 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushControllerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicyProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -28,18 +28,19 @@
private static final long serialVersionUID = 1L;
- public PartitionedLSMInvertedIndexDataflowHelperFactory(ILSMFlushControllerProvider flushControllerProvider,
- ILSMMergePolicyProvider mergePolicyProvider, ILSMOperationTrackerFactory opTrackerProvider,
- ILSMIOOperationSchedulerProvider ioSchedulerProvider) {
- super(flushControllerProvider, mergePolicyProvider, opTrackerProvider, ioSchedulerProvider);
+ public PartitionedLSMInvertedIndexDataflowHelperFactory(ILSMMergePolicyProvider mergePolicyProvider,
+ ILSMOperationTrackerFactory opTrackerProvider, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
+ ILSMIOOperationCallbackProvider ioOpCallbackProvider, int memPageSize, int memNumPages) {
+ super(mergePolicyProvider, opTrackerProvider, ioSchedulerProvider, ioOpCallbackProvider, memNumPages,
+ memNumPages);
}
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition) {
- return new PartitionedLSMInvertedIndexDataflowHelper(opDesc, ctx, partition,
- flushControllerProvider.getFlushController(ctx), mergePolicyProvider.getMergePolicy(ctx),
- opTrackerFactory, ioSchedulerProvider.getIOScheduler(ctx));
+ return new PartitionedLSMInvertedIndexDataflowHelper(opDesc, ctx, partition, memPageSize, memNumPages,
+ mergePolicyProvider.getMergePolicy(ctx), opTrackerFactory, ioSchedulerProvider.getIOScheduler(ctx),
+ ioOpCallbackProvider);
}
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 74deb47..da6bc28 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -48,20 +48,21 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentState;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInvertedIndex;
@@ -77,7 +78,7 @@
public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex {
// In-memory components.
- protected final LSMInvertedIndexComponent mutableComponent;
+ protected final LSMInvertedIndexMutableComponent mutableComponent;
protected final IInMemoryFreePageManager memFreePageManager;
protected final IBinaryTokenizerFactory tokenizerFactory;
@@ -96,10 +97,11 @@
ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) throws IndexException {
- super(memFreePageManager, diskInvIndexFactory.getBufferCache(), fileManager, diskFileMapProvider,
- flushController, mergePolicy, opTrackerFactory, ioScheduler);
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider)
+ throws IndexException {
+ super(memFreePageManager, diskInvIndexFactory.getBufferCache(), fileManager, diskFileMapProvider, mergePolicy,
+ opTrackerFactory, ioScheduler, ioOpCallbackProvider);
this.memFreePageManager = memFreePageManager;
this.tokenizerFactory = tokenizerFactory;
this.invListTypeTraits = invListTypeTraits;
@@ -111,16 +113,10 @@
BTree deleteKeysBTree = BTreeUtils.createBTree(memBufferCache, memFreePageManager,
((InMemoryBufferCache) memBufferCache).getFileMapProvider(), invListTypeTraits, invListCmpFactories,
BTreeLeafFrameType.REGULAR_NSM, new FileReference(new File("membtree")));
- mutableComponent = new LSMInvertedIndexComponent(memInvIndex, deleteKeysBTree);
+ mutableComponent = new LSMInvertedIndexMutableComponent(memInvIndex, deleteKeysBTree, memFreePageManager);
componentFactory = new LSMInvertedIndexComponentFactory(diskInvIndexFactory, deletedKeysBTreeFactory);
}
- protected InMemoryInvertedIndex createInMemoryInvertedIndex(IInMemoryBufferCache memBufferCache)
- throws IndexException {
- return InvertedIndexUtils.createInMemoryBTreeInvertedindex(memBufferCache, memFreePageManager,
- invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory);
- }
-
@Override
public synchronized void create() throws HyracksDataException {
if (isActivated) {
@@ -129,7 +125,7 @@
fileManager.deleteDirs();
fileManager.createDirs();
- immutableComponents.clear();
+ componentsRef.get().clear();
}
@Override
@@ -138,6 +134,7 @@
return;
}
try {
+ List<ILSMComponent> immutableComponents = componentsRef.get();
((InMemoryBufferCache) mutableComponent.getInvIndex().getBufferCache()).open();
mutableComponent.getInvIndex().create();
mutableComponent.getInvIndex().activate();
@@ -146,7 +143,7 @@
immutableComponents.clear();
List<LSMComponentFileReferences> validFileReferences = fileManager.cleanupAndGetValidFiles();
for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
- LSMInvertedIndexComponent component;
+ LSMInvertedIndexImmutableComponent component;
try {
component = createDiskInvIndexComponent(componentFactory,
lsmComonentFileReference.getInsertIndexFileReference(),
@@ -164,52 +161,15 @@
}
@Override
- public List<ILSMComponent> getOperationalComponents(IIndexOperationContext ctx) {
- List<ILSMComponent> operationalComponents = new ArrayList<ILSMComponent>();
- switch (ctx.getOperation()) {
- case SEARCH:
- // TODO: We should add the mutable component at some point.
- operationalComponents.addAll(immutableComponents);
- break;
- case MERGE:
- // TODO: determining the participating components in a merge should probably the task of the merge policy.
- if (immutableComponents.size() > 1) {
- for (ILSMComponent c : immutableComponents) {
- if (c.negativeCompareAndSet(LSMComponentState.MERGING, LSMComponentState.MERGING)) {
- operationalComponents.add(c);
- }
- }
- }
- break;
- default:
- throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
- }
- return operationalComponents;
- }
-
- protected LSMInvertedIndexComponent createDiskInvIndexComponent(ILSMComponentFactory factory,
- FileReference dictBTreeFileRef, FileReference btreeFileRef, boolean create) throws HyracksDataException,
- IndexException {
- LSMInvertedIndexComponent component = (LSMInvertedIndexComponent) factory
- .createLSMComponentInstance(new LSMComponentFileReferences(dictBTreeFileRef, btreeFileRef));
- if (create) {
- component.getInvIndex().create();
- component.getDeletedKeysBTree().create();
- }
- // Will be closed during cleanup of merge().
- component.getInvIndex().activate();
- component.getDeletedKeysBTree().activate();
- return component;
- }
-
- @Override
public void clear() throws HyracksDataException {
if (!isActivated) {
throw new HyracksDataException("Failed to clear the index since it is not activated.");
}
- resetMutableComponent();
+ List<ILSMComponent> immutableComponents = componentsRef.get();
+ mutableComponent.getInvIndex().clear();
+ mutableComponent.getDeletedKeysBTree().clear();
for (ILSMComponent c : immutableComponents) {
- LSMInvertedIndexComponent component = (LSMInvertedIndexComponent) c;
+ LSMInvertedIndexImmutableComponent component = (LSMInvertedIndexImmutableComponent) c;
component.getInvIndex().deactivate();
component.getDeletedKeysBTree().deactivate();
component.getInvIndex().destroy();
@@ -219,25 +179,29 @@
}
@Override
- public synchronized void deactivate() throws HyracksDataException {
+ public synchronized void deactivate(boolean flushOnExit) throws HyracksDataException {
if (!isActivated) {
return;
}
isActivated = false;
- BlockingIOOperationCallback blockingCallBack = new BlockingIOOperationCallback();
- ILSMIndexAccessor accessor = (ILSMIndexAccessor) createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
- lsmHarness.getIOScheduler().scheduleOperation(accessor.createFlushOperation(blockingCallBack));
- try {
- blockingCallBack.waitForIO();
- } catch (InterruptedException e) {
- throw new HyracksDataException(e);
+ if (flushOnExit) {
+ BlockingIOOperationCallbackWrapper blockingCallBack = new BlockingIOOperationCallbackWrapper(
+ ioOpCallbackProvider.getIOOperationCallback(this));
+ ILSMIndexAccessor accessor = (ILSMIndexAccessor) createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ accessor.scheduleFlush(blockingCallBack);
+ try {
+ blockingCallBack.waitForIO();
+ } catch (InterruptedException e) {
+ throw new HyracksDataException(e);
+ }
}
+ List<ILSMComponent> immutableComponents = componentsRef.get();
for (ILSMComponent c : immutableComponents) {
- LSMInvertedIndexComponent component = (LSMInvertedIndexComponent) c;
+ LSMInvertedIndexImmutableComponent component = (LSMInvertedIndexImmutableComponent) c;
component.getInvIndex().deactivate();
component.getDeletedKeysBTree().deactivate();
}
@@ -249,6 +213,11 @@
}
@Override
+ public synchronized void deactivate() throws HyracksDataException {
+ deactivate(true);
+ }
+
+ @Override
public synchronized void destroy() throws HyracksDataException {
if (isActivated) {
throw new HyracksDataException("Failed to destroy the index since it is activated.");
@@ -256,8 +225,9 @@
mutableComponent.getInvIndex().destroy();
mutableComponent.getDeletedKeysBTree().destroy();
+ List<ILSMComponent> immutableComponents = componentsRef.get();
for (ILSMComponent c : immutableComponents) {
- LSMInvertedIndexComponent component = (LSMInvertedIndexComponent) c;
+ LSMInvertedIndexImmutableComponent component = (LSMInvertedIndexImmutableComponent) c;
component.getInvIndex().destroy();
component.getDeletedKeysBTree().destroy();
}
@@ -265,34 +235,28 @@
}
@Override
- public void validate() throws HyracksDataException {
- mutableComponent.getInvIndex().validate();
- mutableComponent.getDeletedKeysBTree().validate();
- for (ILSMComponent c : immutableComponents) {
- LSMInvertedIndexComponent component = (LSMInvertedIndexComponent) c;
- component.getInvIndex().validate();
- component.getDeletedKeysBTree().validate();
+ public void getOperationalComponents(ILSMIndexOperationContext ctx) {
+ List<ILSMComponent> immutableComponents = componentsRef.get();
+ List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
+ operationalComponents.clear();
+ switch (ctx.getOperation()) {
+ case FLUSH:
+ case DELETE:
+ case INSERT:
+ operationalComponents.add(mutableComponent);
+ break;
+ case SEARCH:
+ operationalComponents.add(mutableComponent);
+ operationalComponents.addAll(immutableComponents);
+ break;
+ case MERGE:
+ operationalComponents.addAll(immutableComponents);
+ break;
+ default:
+ throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
}
}
- @Override
- public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback) {
- return new LSMInvertedIndexAccessor(this, lsmHarness, fileManager, createOpContext(modificationCallback,
- searchCallback));
- }
-
- private LSMInvertedIndexOpContext createOpContext(IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback) {
- return new LSMInvertedIndexOpContext(mutableComponent.getInvIndex(), mutableComponent.getDeletedKeysBTree(),
- modificationCallback, searchCallback);
- }
-
- @Override
- public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput) throws IndexException {
- return new LSMInvertedIndexBulkLoader(fillFactor, verifyInput);
- }
-
/**
* The keys in the in-memory deleted-keys BTree only refer to on-disk components.
* We delete documents from the in-memory inverted index by deleting its entries directly,
@@ -308,10 +272,16 @@
* - Insert key into deleted-keys BTree.
*/
@Override
- public void insertUpdateOrDelete(ITupleReference tuple, IIndexOperationContext ictx) throws HyracksDataException,
- IndexException {
+ public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
LSMInvertedIndexOpContext ctx = (LSMInvertedIndexOpContext) ictx;
+ // TODO: This is a hack to support logging properly in ASTERIX.
+ // The proper undo operations are only dependent on the after image so
+ // it is correct to say we found nothing (null) as the before image (at least
+ // in the perspective of ASTERIX). The semantics for the operation callbacks
+ // are violated here (and they are somewhat unclear in the first place as to
+ // what they should be for an inverted index).
ctx.modificationCallback.before(tuple);
+ ctx.modificationCallback.found(null, tuple);
switch (ctx.getOperation()) {
case INSERT: {
// Insert into the in-memory inverted index.
@@ -319,7 +289,6 @@
break;
}
case DELETE: {
- ctx.modificationCallback.before(tuple);
// First remove all entries in the in-memory inverted index (if any).
ctx.memInvIndexAccessor.delete(tuple);
// Insert key into the deleted-keys BTree.
@@ -338,9 +307,12 @@
}
@Override
- public void search(IIndexCursor cursor, List<ILSMComponent> immutableComponents, ISearchPredicate pred,
- IIndexOperationContext ictx, boolean includeMutableComponent) throws HyracksDataException, IndexException {
- int numComponents = (includeMutableComponent) ? immutableComponents.size() : immutableComponents.size() + 1;
+ public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred)
+ throws HyracksDataException, IndexException {
+ List<ILSMComponent> operationalComponents = ictx.getComponentHolder();
+ int numComponents = operationalComponents.size();
+ assert numComponents > 0;
+ boolean includeMutableComponent = operationalComponents.get(0) == mutableComponent;
ArrayList<IIndexAccessor> indexAccessors = new ArrayList<IIndexAccessor>(numComponents);
ArrayList<IIndexAccessor> deletedKeysBTreeAccessors = new ArrayList<IIndexAccessor>(numComponents);
if (includeMutableComponent) {
@@ -351,8 +323,10 @@
NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
deletedKeysBTreeAccessors.add(deletedKeysAccessor);
}
- for (int i = 0; i < immutableComponents.size(); i++) {
- LSMInvertedIndexComponent component = (LSMInvertedIndexComponent) immutableComponents.get(i);
+
+ for (int i = includeMutableComponent ? 1 : 0; i < operationalComponents.size(); i++) {
+ LSMInvertedIndexImmutableComponent component = (LSMInvertedIndexImmutableComponent) operationalComponents
+ .get(i);
IIndexAccessor invIndexAccessor = component.getInvIndex().createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
indexAccessors.add(invIndexAccessor);
@@ -369,12 +343,13 @@
private ICursorInitialState createCursorInitialState(ISearchPredicate pred, IIndexOperationContext ictx,
boolean includeMutableComponent, ArrayList<IIndexAccessor> indexAccessors,
ArrayList<IIndexAccessor> deletedKeysBTreeAccessors) {
+ List<ILSMComponent> immutableComponents = componentsRef.get();
ICursorInitialState initState = null;
PermutingTupleReference keysOnlyTuple = createKeysOnlyTupleReference();
MultiComparator keyCmp = MultiComparator.createIgnoreFieldLength(invListCmpFactories);
List<ILSMComponent> operationalComponents = new ArrayList<ILSMComponent>();
if (includeMutableComponent) {
- operationalComponents.add(getMutableComponent());
+ operationalComponents.add(mutableComponent);
}
operationalComponents.addAll(immutableComponents);
@@ -407,67 +382,17 @@
}
@Override
- public ILSMComponent merge(List<ILSMComponent> mergedComponents, ILSMIOOperation operation)
- throws HyracksDataException, IndexException {
- LSMInvertedIndexMergeOperation mergeOp = (LSMInvertedIndexMergeOperation) operation;
-
- // Create an inverted index instance.
- LSMInvertedIndexComponent component = createDiskInvIndexComponent(componentFactory,
- mergeOp.getDictBTreeMergeTarget(), mergeOp.getDeletedKeysBTreeMergeTarget(), true);
-
- IInvertedIndex mergedDiskInvertedIndex = component.getInvIndex();
- IIndexCursor cursor = mergeOp.getCursor();
- IIndexBulkLoader invIndexBulkLoader = mergedDiskInvertedIndex.createBulkLoader(1.0f, true);
- try {
- while (cursor.hasNext()) {
- cursor.next();
- ITupleReference tuple = cursor.getTuple();
- invIndexBulkLoader.add(tuple);
- }
- } finally {
- cursor.close();
- }
- invIndexBulkLoader.end();
-
- // Create an empty deleted keys BTree (do nothing with the returned index).
- BTree deletedKeysBTree = component.getDeletedKeysBTree();
-
- // Add the merged components for cleanup.
- mergedComponents.addAll(mergeOp.getMergingComponents());
-
- return new LSMInvertedIndexComponent(mergedDiskInvertedIndex, deletedKeysBTree);
- }
-
- @Override
- public ILSMIOOperation createMergeOperation(ILSMIOOperationCallback callback) throws HyracksDataException,
- IndexException {
- LSMInvertedIndexOpContext ctx = createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- ctx.setOperation(IndexOperation.MERGE);
- IIndexCursor cursor = new LSMInvertedIndexRangeSearchCursor(ctx);
- RangePredicate mergePred = new RangePredicate(null, null, true, true, null, null);
-
- // Scan diskInvertedIndexes ignoring the memoryInvertedIndex.
- List<ILSMComponent> mergingComponents = lsmHarness.search(cursor, mergePred, ctx, false);
- if (mergingComponents.size() <= 1) {
- cursor.close();
- return null;
- }
-
- LSMInvertedIndexComponent firstComponent = (LSMInvertedIndexComponent) mergingComponents.get(0);
- OnDiskInvertedIndex firstInvIndex = (OnDiskInvertedIndex) firstComponent.getInvIndex();
- String firstFileName = firstInvIndex.getBTree().getFileReference().getFile().getName();
-
- LSMInvertedIndexComponent lastComponent = (LSMInvertedIndexComponent) mergingComponents.get(mergingComponents
- .size() - 1);
- OnDiskInvertedIndex lastInvIndex = (OnDiskInvertedIndex) lastComponent.getInvIndex();
- String lastFileName = lastInvIndex.getBTree().getFileReference().getFile().getName();
-
- LSMComponentFileReferences relMergeFileRefs = fileManager.getRelMergeFileReference(firstFileName, lastFileName);
- LSMInvertedIndexMergeOperation mergeOp = new LSMInvertedIndexMergeOperation(this, mergingComponents, cursor,
- relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(),
- callback);
-
- return mergeOp;
+ public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ throws HyracksDataException {
+ LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
+ LSMInvertedIndexOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
+ opCtx.setOperation(IndexOperation.FLUSH);
+ opCtx.getComponentHolder().add(flushingComponent);
+ ioScheduler.scheduleOperation(new LSMInvertedIndexFlushOperation(new LSMInvertedIndexAccessor(this, lsmHarness,
+ fileManager, opCtx), mutableComponent, componentFileRefs.getInsertIndexFileReference(),
+ componentFileRefs.getDeleteIndexFileReference(), callback));
}
@Override
@@ -475,12 +400,13 @@
LSMInvertedIndexFlushOperation flushOp = (LSMInvertedIndexFlushOperation) operation;
// Create an inverted index instance to be bulk loaded.
- LSMInvertedIndexComponent component = createDiskInvIndexComponent(componentFactory,
+ LSMInvertedIndexImmutableComponent component = createDiskInvIndexComponent(componentFactory,
flushOp.getDictBTreeFlushTarget(), flushOp.getDeletedKeysBTreeFlushTarget(), true);
IInvertedIndex diskInvertedIndex = component.getInvIndex();
// Create a scan cursor on the BTree underlying the in-memory inverted index.
- InMemoryInvertedIndexAccessor memInvIndexAccessor = (InMemoryInvertedIndexAccessor) mutableComponent
+ LSMInvertedIndexMutableComponent flushingComponent = flushOp.getFlushingComponent();
+ InMemoryInvertedIndexAccessor memInvIndexAccessor = (InMemoryInvertedIndexAccessor) flushingComponent
.getInvIndex().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
BTreeAccessor memBTreeAccessor = memInvIndexAccessor.getBTreeAccessor();
RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null);
@@ -503,7 +429,7 @@
BTree diskDeletedKeysBTree = component.getDeletedKeysBTree();
// Create a scan cursor on the deleted keys BTree underlying the in-memory inverted index.
- IIndexAccessor deletedKeysBTreeAccessor = mutableComponent.getDeletedKeysBTree().createAccessor(
+ IIndexAccessor deletedKeysBTreeAccessor = flushingComponent.getDeletedKeysBTree().createAccessor(
NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
IIndexCursor deletedKeysScanCursor = deletedKeysBTreeAccessor.createSearchCursor();
deletedKeysBTreeAccessor.search(deletedKeysScanCursor, nullPred);
@@ -520,7 +446,69 @@
}
deletedKeysBTreeBulkLoader.end();
- return new LSMInvertedIndexComponent(diskInvertedIndex, diskDeletedKeysBTree);
+ return new LSMInvertedIndexImmutableComponent(diskInvertedIndex, diskDeletedKeysBTree);
+ }
+
+ @Override
+ public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ throws HyracksDataException, IndexException {
+ LSMInvertedIndexOpContext ictx = createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
+ ictx.getComponentHolder().addAll(mergingComponents);
+ IIndexCursor cursor = new LSMInvertedIndexRangeSearchCursor(ictx);
+ RangePredicate mergePred = new RangePredicate(null, null, true, true, null, null);
+
+ // Scan diskInvertedIndexes ignoring the memoryInvertedIndex.
+ search(ictx, cursor, mergePred);
+
+ ictx.setOperation(IndexOperation.MERGE);
+ LSMInvertedIndexImmutableComponent firstComponent = (LSMInvertedIndexImmutableComponent) mergingComponents
+ .get(0);
+ OnDiskInvertedIndex firstInvIndex = (OnDiskInvertedIndex) firstComponent.getInvIndex();
+ String firstFileName = firstInvIndex.getBTree().getFileReference().getFile().getName();
+
+ LSMInvertedIndexImmutableComponent lastComponent = (LSMInvertedIndexImmutableComponent) mergingComponents
+ .get(mergingComponents.size() - 1);
+ OnDiskInvertedIndex lastInvIndex = (OnDiskInvertedIndex) lastComponent.getInvIndex();
+ String lastFileName = lastInvIndex.getBTree().getFileReference().getFile().getName();
+
+ LSMComponentFileReferences relMergeFileRefs = fileManager.getRelMergeFileReference(firstFileName, lastFileName);
+ ILSMIndexAccessorInternal accessor = new LSMInvertedIndexAccessor(this, lsmHarness, fileManager, ictx);
+ ioScheduler.scheduleOperation(new LSMInvertedIndexMergeOperation(accessor, mergingComponents, cursor,
+ relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(),
+ callback));
+ }
+
+ @Override
+ public ILSMComponent merge(List<ILSMComponent> mergedComponents, ILSMIOOperation operation)
+ throws HyracksDataException, IndexException {
+ LSMInvertedIndexMergeOperation mergeOp = (LSMInvertedIndexMergeOperation) operation;
+
+ // Create an inverted index instance.
+ LSMInvertedIndexImmutableComponent component = createDiskInvIndexComponent(componentFactory,
+ mergeOp.getDictBTreeMergeTarget(), mergeOp.getDeletedKeysBTreeMergeTarget(), true);
+
+ IInvertedIndex mergedDiskInvertedIndex = component.getInvIndex();
+ IIndexCursor cursor = mergeOp.getCursor();
+ IIndexBulkLoader invIndexBulkLoader = mergedDiskInvertedIndex.createBulkLoader(1.0f, true);
+ try {
+ while (cursor.hasNext()) {
+ cursor.next();
+ ITupleReference tuple = cursor.getTuple();
+ invIndexBulkLoader.add(tuple);
+ }
+ } finally {
+ cursor.close();
+ }
+ invIndexBulkLoader.end();
+
+ // Create an empty deleted keys BTree (do nothing with the returned index).
+ BTree deletedKeysBTree = component.getDeletedKeysBTree();
+
+ // Add the merged components for cleanup.
+ mergedComponents.addAll(mergeOp.getMergingComponents());
+
+ return new LSMInvertedIndexImmutableComponent(mergedDiskInvertedIndex, deletedKeysBTree);
}
private ILSMComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
@@ -529,6 +517,11 @@
componentFileRefs.getDeleteIndexFileReference(), true);
}
+ @Override
+ public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput) throws IndexException {
+ return new LSMInvertedIndexBulkLoader(fillFactor, verifyInput);
+ }
+
public class LSMInvertedIndexBulkLoader implements IIndexBulkLoader {
private final ILSMComponent component;
private final IIndexBulkLoader invIndexBulkLoader;
@@ -543,8 +536,8 @@
} catch (IndexException e) {
throw new TreeIndexException(e);
}
- invIndexBulkLoader = ((LSMInvertedIndexComponent) component).getInvIndex().createBulkLoader(fillFactor,
- verifyInput);
+ invIndexBulkLoader = ((LSMInvertedIndexImmutableComponent) component).getInvIndex().createBulkLoader(
+ fillFactor, verifyInput);
}
@Override
@@ -564,10 +557,10 @@
}
protected void handleException() throws HyracksDataException {
- ((LSMInvertedIndexComponent) component).getInvIndex().deactivate();
- ((LSMInvertedIndexComponent) component).getInvIndex().destroy();
- ((LSMInvertedIndexComponent) component).getDeletedKeysBTree().deactivate();
- ((LSMInvertedIndexComponent) component).getDeletedKeysBTree().destroy();
+ ((LSMInvertedIndexImmutableComponent) component).getInvIndex().deactivate();
+ ((LSMInvertedIndexImmutableComponent) component).getInvIndex().destroy();
+ ((LSMInvertedIndexImmutableComponent) component).getDeletedKeysBTree().deactivate();
+ ((LSMInvertedIndexImmutableComponent) component).getDeletedKeysBTree().destroy();
}
@Override
@@ -577,17 +570,38 @@
}
}
- @Override
- public void resetMutableComponent() throws HyracksDataException {
- memFreePageManager.reset();
- mutableComponent.getInvIndex().clear();
- mutableComponent.getDeletedKeysBTree().clear();
+ protected InMemoryInvertedIndex createInMemoryInvertedIndex(IInMemoryBufferCache memBufferCache)
+ throws IndexException {
+ return InvertedIndexUtils.createInMemoryBTreeInvertedindex(memBufferCache, memFreePageManager,
+ invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory);
+ }
+
+ protected LSMInvertedIndexImmutableComponent createDiskInvIndexComponent(ILSMComponentFactory factory,
+ FileReference dictBTreeFileRef, FileReference btreeFileRef, boolean create) throws HyracksDataException,
+ IndexException {
+ LSMInvertedIndexImmutableComponent component = (LSMInvertedIndexImmutableComponent) factory
+ .createLSMComponentInstance(new LSMComponentFileReferences(dictBTreeFileRef, btreeFileRef));
+ if (create) {
+ component.getInvIndex().create();
+ component.getDeletedKeysBTree().create();
+ }
+ // Will be closed during cleanup of merge().
+ component.getInvIndex().activate();
+ component.getDeletedKeysBTree().activate();
+ return component;
}
@Override
- public long getMemoryAllocationSize() {
- InMemoryBufferCache memBufferCache = (InMemoryBufferCache) mutableComponent.getInvIndex().getBufferCache();
- return memBufferCache.getNumPages() * memBufferCache.getPageSize();
+ public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
+ ISearchOperationCallback searchCallback) {
+ return new LSMInvertedIndexAccessor(this, lsmHarness, fileManager, createOpContext(modificationCallback,
+ searchCallback));
+ }
+
+ private LSMInvertedIndexOpContext createOpContext(IModificationOperationCallback modificationCallback,
+ ISearchOperationCallback searchCallback) {
+ return new LSMInvertedIndexOpContext(mutableComponent.getInvIndex(), mutableComponent.getDeletedKeysBTree(),
+ modificationCallback, searchCallback);
}
@Override
@@ -612,6 +626,12 @@
}
@Override
+ public long getMemoryAllocationSize() {
+ InMemoryBufferCache memBufferCache = (InMemoryBufferCache) mutableComponent.getInvIndex().getBufferCache();
+ return memBufferCache.getNumPages() * memBufferCache.getPageSize();
+ }
+
+ @Override
public ITypeTraits[] getTokenTypeTraits() {
return tokenTypeTraits;
}
@@ -625,9 +645,17 @@
return tokenizerFactory;
}
+ protected void forceFlushInvListsFileDirtyPages(OnDiskInvertedIndex invIndex) throws HyracksDataException {
+ int fileId = invIndex.getInvListsFileId();
+ IBufferCache bufferCache = invIndex.getBufferCache();
+ int startPageId = 0;
+ int maxPageId = invIndex.getInvListsMaxPageId();
+ forceFlushDirtyPages(bufferCache, fileId, startPageId, maxPageId);
+ }
+
@Override
public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException {
- LSMInvertedIndexComponent invIndexComponent = (LSMInvertedIndexComponent) lsmComponent;
+ LSMInvertedIndexImmutableComponent invIndexComponent = (LSMInvertedIndexImmutableComponent) lsmComponent;
OnDiskInvertedIndex invIndex = (OnDiskInvertedIndex) invIndexComponent.getInvIndex();
ITreeIndex treeIndex = invIndex.getBTree();
// Flush inverted index first.
@@ -639,16 +667,15 @@
markAsValidInternal(treeIndex);
}
- protected void forceFlushInvListsFileDirtyPages(OnDiskInvertedIndex invIndex) throws HyracksDataException {
- int fileId = invIndex.getInvListsFileId();
- IBufferCache bufferCache = invIndex.getBufferCache();
- int startPageId = 0;
- int maxPageId = invIndex.getInvListsMaxPageId();
- forceFlushDirtyPages(bufferCache, fileId, startPageId, maxPageId);
- }
-
@Override
- public ILSMComponent getMutableComponent() {
- return mutableComponent;
+ public void validate() throws HyracksDataException {
+ mutableComponent.getInvIndex().validate();
+ mutableComponent.getDeletedKeysBTree().validate();
+ List<ILSMComponent> immutableComponents = componentsRef.get();
+ for (ILSMComponent c : immutableComponents) {
+ LSMInvertedIndexImmutableComponent component = (LSMInvertedIndexImmutableComponent) c;
+ component.getInvIndex().validate();
+ component.getDeletedKeysBTree().validate();
+ }
}
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
index 5d329a0..7f7d3cd 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
@@ -24,14 +24,13 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
-public class LSMInvertedIndexAccessor implements ILSMIndexAccessor, IInvertedIndexAccessor {
+public class LSMInvertedIndexAccessor implements ILSMIndexAccessorInternal, IInvertedIndexAccessor {
protected final ILSMHarness lsmHarness;
protected final ILSMIndexFileManager fileManager;
@@ -49,30 +48,30 @@
@Override
public void insert(ITupleReference tuple) throws HyracksDataException, IndexException {
ctx.setOperation(IndexOperation.INSERT);
- lsmHarness.insertUpdateOrDelete(tuple, ctx, false);
+ lsmHarness.modify(ctx, false, tuple);
}
@Override
public void delete(ITupleReference tuple) throws HyracksDataException, IndexException {
ctx.setOperation(IndexOperation.DELETE);
- lsmHarness.insertUpdateOrDelete(tuple, ctx, false);
+ lsmHarness.modify(ctx, false, tuple);
}
@Override
public boolean tryInsert(ITupleReference tuple) throws HyracksDataException, IndexException {
ctx.setOperation(IndexOperation.INSERT);
- return lsmHarness.insertUpdateOrDelete(tuple, ctx, true);
+ return lsmHarness.modify(ctx, true, tuple);
}
@Override
public boolean tryDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
ctx.setOperation(IndexOperation.DELETE);
- return lsmHarness.insertUpdateOrDelete(tuple, ctx, true);
+ return lsmHarness.modify(ctx, true, tuple);
}
public void search(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException, IndexException {
ctx.setOperation(IndexOperation.SEARCH);
- lsmHarness.search(cursor, searchPred, ctx, true);
+ lsmHarness.search(ctx, cursor, searchPred);
}
public IIndexCursor createSearchCursor() {
@@ -80,27 +79,25 @@
}
@Override
- public ILSMIOOperation createFlushOperation(ILSMIOOperationCallback callback) {
- LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- return new LSMInvertedIndexFlushOperation(lsmHarness.getIndex(),
- componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(),
- callback);
+ public void scheduleFlush(ILSMIOOperationCallback callback) throws HyracksDataException {
+ ctx.setOperation(IndexOperation.FLUSH);
+ lsmHarness.scheduleFlush(ctx, callback);
}
@Override
public void flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
- lsmHarness.flush(operation);
+ lsmHarness.flush(ctx, operation);
}
@Override
- public ILSMIOOperation createMergeOperation(ILSMIOOperationCallback callback) throws HyracksDataException,
- IndexException {
- return lsmHarness.createMergeOperation(callback);
+ public void scheduleMerge(ILSMIOOperationCallback callback) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.MERGE);
+ lsmHarness.scheduleMerge(ctx, callback);
}
@Override
public void merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
- lsmHarness.merge(operation);
+ lsmHarness.merge(ctx, operation);
}
@Override
@@ -115,13 +112,25 @@
}
@Override
- public boolean tryNoOp() throws HyracksDataException {
- return lsmHarness.noOp(ctx, true);
+ public void noOp() throws HyracksDataException {
+ lsmHarness.noOp(ctx);
}
@Override
- public void noOp() throws HyracksDataException {
- lsmHarness.noOp(ctx, false);
+ public void forcePhysicalDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ throw new UnsupportedOperationException("Physical delete not supported by lsm inverted index.");
+ }
+
+ @Override
+ public void forceInsert(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.INSERT);
+ lsmHarness.forceModify(ctx, tuple);
+ }
+
+ @Override
+ public void forceDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.DELETE);
+ lsmHarness.forceModify(ctx, tuple);
}
@Override
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexComponentFactory.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexComponentFactory.java
index 23fc0ae..f856f90 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexComponentFactory.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexComponentFactory.java
@@ -36,9 +36,8 @@
@Override
public ILSMComponent createLSMComponentInstance(LSMComponentFileReferences cfr) throws IndexException {
- return new LSMInvertedIndexComponent(
- diskInvIndexFactory.createIndexInstance(cfr.getInsertIndexFileReference()),
- btreeFactory.createIndexInstance(cfr.getDeleteIndexFileReference()));
+ return new LSMInvertedIndexImmutableComponent(diskInvIndexFactory.createIndexInstance(cfr
+ .getInsertIndexFileReference()), btreeFactory.createIndexInstance(cfr.getDeleteIndexFileReference()));
}
@Override
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
index 3055f52..fb55ce0 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
@@ -16,46 +16,45 @@
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls;
import java.util.Collections;
-import java.util.List;
+import java.util.Set;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
-import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
public class LSMInvertedIndexFlushOperation implements ILSMIOOperation {
- private final ILSMIndex index;
+ private final ILSMIndexAccessorInternal accessor;
+ private final LSMInvertedIndexMutableComponent flushingComponent;
private final FileReference dictBTreeFlushTarget;
private final FileReference deletedKeysBTreeFlushTarget;
private final ILSMIOOperationCallback callback;
- public LSMInvertedIndexFlushOperation(ILSMIndex index, FileReference dictBTreeFlushTarget,
+ public LSMInvertedIndexFlushOperation(ILSMIndexAccessorInternal accessor,
+ LSMInvertedIndexMutableComponent flushingComponent, FileReference dictBTreeFlushTarget,
FileReference deletedKeysBTreeFlushTarget, ILSMIOOperationCallback callback) {
- this.index = index;
+ this.accessor = accessor;
+ this.flushingComponent = flushingComponent;
this.dictBTreeFlushTarget = dictBTreeFlushTarget;
this.deletedKeysBTreeFlushTarget = deletedKeysBTreeFlushTarget;
this.callback = callback;
}
@Override
- public List<IODeviceHandle> getReadDevices() {
- return Collections.emptyList();
+ public Set<IODeviceHandle> getReadDevices() {
+ return Collections.emptySet();
}
@Override
- public List<IODeviceHandle> getWriteDevices() {
- return Collections.singletonList(dictBTreeFlushTarget.getDevideHandle());
+ public Set<IODeviceHandle> getWriteDevices() {
+ return Collections.singleton(dictBTreeFlushTarget.getDeviceHandle());
}
@Override
public void perform() throws HyracksDataException, IndexException {
- ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
accessor.flush(this);
}
@@ -71,4 +70,8 @@
public FileReference getDeletedKeysBTreeFlushTarget() {
return deletedKeysBTreeFlushTarget;
}
+
+ public LSMInvertedIndexMutableComponent getFlushingComponent() {
+ return flushingComponent;
+ }
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexImmutableComponent.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexImmutableComponent.java
new file mode 100644
index 0000000..5099a7b
--- /dev/null
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexImmutableComponent.java
@@ -0,0 +1,34 @@
+package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractImmutableLSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
+
+public class LSMInvertedIndexImmutableComponent extends AbstractImmutableLSMComponent {
+
+ private final IInvertedIndex invIndex;
+ private final BTree deletedKeysBTree;
+
+ public LSMInvertedIndexImmutableComponent(IInvertedIndex invIndex, BTree deletedKeysBTree) {
+ this.invIndex = invIndex;
+ this.deletedKeysBTree = deletedKeysBTree;
+ }
+
+ @Override
+ public void destroy() throws HyracksDataException {
+ invIndex.deactivate();
+ invIndex.destroy();
+ deletedKeysBTree.deactivate();
+ deletedKeysBTree.destroy();
+ }
+
+ public IInvertedIndex getInvIndex() {
+ return invIndex;
+ }
+
+ public BTree getDeletedKeysBTree() {
+ return deletedKeysBTree;
+ }
+
+}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
index e6baf9d..63b604e 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
@@ -15,34 +15,33 @@
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls;
-import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
-import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex;
public class LSMInvertedIndexMergeOperation implements ILSMIOOperation {
- private final ILSMIndex index;
+ private final ILSMIndexAccessorInternal accessor;
private final List<ILSMComponent> mergingComponents;
private final IIndexCursor cursor;
private final FileReference dictBTreeMergeTarget;
private final FileReference deletedKeysBTreeMergeTarget;
private final ILSMIOOperationCallback callback;
- public LSMInvertedIndexMergeOperation(ILSMIndex index, List<ILSMComponent> mergingComponents, IIndexCursor cursor,
- FileReference dictBTreeMergeTarget, FileReference deletedKeysBTreeMergeTarget,
+ public LSMInvertedIndexMergeOperation(ILSMIndexAccessorInternal accessor, List<ILSMComponent> mergingComponents,
+ IIndexCursor cursor, FileReference dictBTreeMergeTarget, FileReference deletedKeysBTreeMergeTarget,
ILSMIOOperationCallback callback) {
- this.index = index;
+ this.accessor = accessor;
this.mergingComponents = mergingComponents;
this.cursor = cursor;
this.dictBTreeMergeTarget = dictBTreeMergeTarget;
@@ -51,29 +50,27 @@
}
@Override
- public List<IODeviceHandle> getReadDevices() {
- List<IODeviceHandle> devs = new ArrayList<IODeviceHandle>();
+ public Set<IODeviceHandle> getReadDevices() {
+ Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
for (Object o : mergingComponents) {
- LSMInvertedIndexComponent component = (LSMInvertedIndexComponent) o;
+ LSMInvertedIndexImmutableComponent component = (LSMInvertedIndexImmutableComponent) o;
OnDiskInvertedIndex invIndex = (OnDiskInvertedIndex) component.getInvIndex();
- devs.add(invIndex.getBTree().getFileReference().getDevideHandle());
- devs.add(component.getDeletedKeysBTree().getFileReference().getDevideHandle());
+ devs.add(invIndex.getBTree().getFileReference().getDeviceHandle());
+ devs.add(component.getDeletedKeysBTree().getFileReference().getDeviceHandle());
}
return devs;
}
@Override
- public List<IODeviceHandle> getWriteDevices() {
- List<IODeviceHandle> devs = new ArrayList<IODeviceHandle>(2);
- devs.add(dictBTreeMergeTarget.getDevideHandle());
- devs.add(deletedKeysBTreeMergeTarget.getDevideHandle());
+ public Set<IODeviceHandle> getWriteDevices() {
+ Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>(2);
+ devs.add(dictBTreeMergeTarget.getDeviceHandle());
+ devs.add(deletedKeysBTreeMergeTarget.getDeviceHandle());
return devs;
}
@Override
public void perform() throws HyracksDataException, IndexException {
- ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
accessor.merge(this);
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexComponent.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMutableComponent.java
similarity index 67%
rename from hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexComponent.java
rename to hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMutableComponent.java
index d9a65b9..c36319d 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexComponent.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMutableComponent.java
@@ -17,33 +17,21 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractLSMComponent;
+import edu.uci.ics.hyracks.storage.am.common.api.IInMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractMutableLSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
-public class LSMInvertedIndexComponent extends AbstractLSMComponent {
+public class LSMInvertedIndexMutableComponent extends AbstractMutableLSMComponent {
private final IInvertedIndex invIndex;
private final BTree deletedKeysBTree;
+ private final IInMemoryFreePageManager mfpm;
- public LSMInvertedIndexComponent(IInvertedIndex invIndex, BTree deletedKeysBTree) {
+ public LSMInvertedIndexMutableComponent(IInvertedIndex invIndex, BTree deletedKeysBTree,
+ IInMemoryFreePageManager mfpm) {
this.invIndex = invIndex;
this.deletedKeysBTree = deletedKeysBTree;
- }
-
- @Override
- public void destroy() throws HyracksDataException {
- invIndex.deactivate();
- invIndex.destroy();
- deletedKeysBTree.deactivate();
- deletedKeysBTree.destroy();
- }
-
- @Override
- public void reset() throws HyracksDataException {
- ((InMemoryFreePageManager) deletedKeysBTree.getFreePageManager()).reset();
- invIndex.clear();
- deletedKeysBTree.clear();
+ this.mfpm = mfpm;
}
public IInvertedIndex getInvIndex() {
@@ -53,4 +41,15 @@
public BTree getDeletedKeysBTree() {
return deletedKeysBTree;
}
+
+ @Override
+ protected boolean isFull() {
+ return mfpm.isFull();
+ }
+
+ @Override
+ protected void reset() throws HyracksDataException {
+ invIndex.clear();
+ deletedKeysBTree.clear();
+ }
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
index 625ecd4..b961b7a 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
@@ -15,6 +15,9 @@
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls;
+import java.util.LinkedList;
+import java.util.List;
+
import edu.uci.ics.hyracks.storage.am.common.api.IIndex;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
@@ -22,6 +25,7 @@
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.common.tuples.PermutingTupleReference;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor;
@@ -29,17 +33,18 @@
public class LSMInvertedIndexOpContext implements ILSMIndexOperationContext {
private static final int NUM_DOCUMENT_FIELDS = 1;
-
+
private IndexOperation op;
private final IInvertedIndex memInvIndex;
private final IIndex memDeletedKeysBTree;
+ private final List<ILSMComponent> componentHolder;
public final IModificationOperationCallback modificationCallback;
public final ISearchOperationCallback searchCallback;
-
+
// Tuple that only has the inverted-index elements (aka keys), projecting away the document fields.
public PermutingTupleReference keysOnlyTuple;
-
+
// Accessor to the in-memory inverted index.
public IInvertedIndexAccessor memInvIndexAccessor;
// Accessor to the deleted-keys BTree.
@@ -49,17 +54,20 @@
IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) {
this.memInvIndex = memInvIndex;
this.memDeletedKeysBTree = memDeletedKeysBTree;
+ this.componentHolder = new LinkedList<ILSMComponent>();
this.modificationCallback = modificationCallback;
this.searchCallback = searchCallback;
}
@Override
public void reset() {
+ componentHolder.clear();
}
@Override
// TODO: Ignore opcallback for now.
public void setOperation(IndexOperation newOp) {
+ reset();
switch (newOp) {
case INSERT:
case DELETE:
@@ -82,13 +90,18 @@
}
op = newOp;
}
-
+
@Override
public IndexOperation getOperation() {
return op;
}
@Override
+ public List<ILSMComponent> getComponentHolder() {
+ return componentHolder;
+ }
+
+ @Override
public ISearchOperationCallback getSearchOperationCallback() {
return searchCallback;
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexSearchCursor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexSearchCursor.java
index b6d3bb7..1ea66f7 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexSearchCursor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexSearchCursor.java
@@ -153,7 +153,7 @@
public void close() throws HyracksDataException {
reset();
accessorIndex = -1;
- harness.closeSearchCursor(operationalComponents, includeMemComponent, opCtx);
+ harness.endSearch(opCtx);
}
@Override
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
index d963dbc..3cc5deb 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
@@ -20,7 +20,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IInMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -40,11 +40,12 @@
IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) throws IndexException {
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider)
+ throws IndexException {
super(memBufferCache, memFreePageManager, diskInvIndexFactory, deletedKeysBTreeFactory, fileManager,
diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories,
- tokenizerFactory, flushController, mergePolicy, opTrackerFactory, ioScheduler);
+ tokenizerFactory, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackProvider);
}
protected InMemoryInvertedIndex createInMemoryInvertedIndex(IInMemoryBufferCache memBufferCache)
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
index d129fa9..8a4f989 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
@@ -34,7 +34,7 @@
import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -121,11 +121,12 @@
ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, IIOManager ioManager,
- String onDiskDir, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) throws IndexException {
+ String onDiskDir, ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider)
+ throws IndexException {
return createLSMInvertedIndex(memBufferCache, memFreePageManager, diskFileMapProvider, invListTypeTraits,
invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, diskBufferCache, ioManager,
- onDiskDir, flushController, mergePolicy, opTrackerFactory, ioScheduler, 0);
+ onDiskDir, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackProvider, 0);
}
public static LSMInvertedIndex createLSMInvertedIndex(IInMemoryBufferCache memBufferCache,
@@ -133,9 +134,9 @@
ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, IIOManager ioManager,
- String onDiskDir, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler, int startIODeviceIndex)
- throws IndexException {
+ String onDiskDir, ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
+ int startIODeviceIndex) throws IndexException {
BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(diskFileMapProvider, invListTypeTraits,
invListCmpFactories, diskBufferCache);
@@ -152,8 +153,8 @@
LSMInvertedIndex invIndex = new LSMInvertedIndex(memBufferCache, memFreePageManager, invIndexFactory,
deletedKeysBTreeFactory, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories,
- tokenTypeTraits, tokenCmpFactories, tokenizerFactory, flushController, mergePolicy, opTrackerFactory,
- ioScheduler);
+ tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTrackerFactory, ioScheduler,
+ ioOpCallbackProvider);
return invIndex;
}
@@ -162,11 +163,13 @@
ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, IIOManager ioManager,
- String onDiskDir, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) throws IndexException {
+ String onDiskDir, ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider)
+ throws IndexException {
return createPartitionedLSMInvertedIndex(memBufferCache, memFreePageManager, diskFileMapProvider,
invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
- diskBufferCache, ioManager, onDiskDir, flushController, mergePolicy, opTrackerFactory, ioScheduler, 0);
+ diskBufferCache, ioManager, onDiskDir, mergePolicy, opTrackerFactory, ioScheduler,
+ ioOpCallbackProvider, 0);
}
public static PartitionedLSMInvertedIndex createPartitionedLSMInvertedIndex(IInMemoryBufferCache memBufferCache,
@@ -174,9 +177,9 @@
ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, IIOManager ioManager,
- String onDiskDir, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler, int startIODeviceIndex)
- throws IndexException {
+ String onDiskDir, ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
+ int startIODeviceIndex) throws IndexException {
BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(diskFileMapProvider, invListTypeTraits,
invListCmpFactories, diskBufferCache);
@@ -193,8 +196,8 @@
PartitionedLSMInvertedIndex invIndex = new PartitionedLSMInvertedIndex(memBufferCache, memFreePageManager,
invIndexFactory, deletedKeysBTreeFactory, fileManager, diskFileMapProvider, invListTypeTraits,
- invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, flushController,
- mergePolicy, opTrackerFactory, ioScheduler);
+ invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy,
+ opTrackerFactory, ioScheduler, ioOpCallbackProvider);
return invIndex;
}
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
index 92b443b..c363c99 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
@@ -30,7 +30,7 @@
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -52,22 +52,22 @@
public AbstractLSMRTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
IBinaryComparatorFactory[] btreeComparatorFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
ILinearizeComparatorFactory linearizeCmpFactory) {
this(opDesc, ctx, partition, DEFAULT_MEM_PAGE_SIZE, DEFAULT_MEM_NUM_PAGES, btreeComparatorFactories,
- valueProviderFactories, rtreePolicyType, flushController, mergePolicy, opTrackerFactory, ioScheduler,
- linearizeCmpFactory);
+ valueProviderFactories, rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler,
+ ioOpCallbackProvider, linearizeCmpFactory);
}
public AbstractLSMRTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
int memPageSize, int memNumPages, IBinaryComparatorFactory[] btreeComparatorFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
ILinearizeComparatorFactory linearizeCmpFactory) {
- super(opDesc, ctx, partition, memPageSize, memNumPages, flushController, mergePolicy, opTrackerFactory,
- ioScheduler);
+ super(opDesc, ctx, partition, memPageSize, memNumPages, mergePolicy, opTrackerFactory, ioScheduler,
+ ioOpCallbackProvider);
this.btreeComparatorFactories = btreeComparatorFactories;
this.valueProviderFactories = valueProviderFactories;
this.rtreePolicyType = rtreePolicyType;
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
index 7eae769..1df914e 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
@@ -28,7 +28,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -42,21 +42,21 @@
public LSMRTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
IBinaryComparatorFactory[] btreeComparatorFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
ILinearizeComparatorFactory linearizeCmpFactory) {
- super(opDesc, ctx, partition, btreeComparatorFactories, valueProviderFactories, rtreePolicyType,
- flushController, mergePolicy, opTrackerFactory, ioScheduler, linearizeCmpFactory);
+ super(opDesc, ctx, partition, btreeComparatorFactories, valueProviderFactories, rtreePolicyType, mergePolicy,
+ opTrackerFactory, ioScheduler, ioOpCallbackProvider, linearizeCmpFactory);
}
public LSMRTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
int memPageSize, int memNumPages, IBinaryComparatorFactory[] btreeComparatorFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
ILinearizeComparatorFactory linearizeCmpFactory) {
super(opDesc, ctx, partition, memPageSize, memNumPages, btreeComparatorFactories, valueProviderFactories,
- rtreePolicyType, flushController, mergePolicy, opTrackerFactory, ioScheduler, linearizeCmpFactory);
+ rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackProvider, linearizeCmpFactory);
}
@Override
@@ -69,8 +69,8 @@
try {
return LSMRTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ioManager, file, diskBufferCache,
diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories,
- rtreePolicyType, flushController, mergePolicy, opTrackerFactory, ioScheduler, linearizeCmpFactory,
- startIODeviceIndex);
+ rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackProvider,
+ linearizeCmpFactory, startIODeviceIndex);
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
index 5ba724c..a730895 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
@@ -21,7 +21,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushControllerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicyProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -39,10 +39,11 @@
public LSMRTreeDataflowHelperFactory(IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, IBinaryComparatorFactory[] btreeComparatorFactories,
- ILSMFlushControllerProvider flushControllerProvider, ILSMMergePolicyProvider mergePolicyProvider,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
- ILinearizeComparatorFactory linearizeCmpFactory) {
- super(flushControllerProvider, mergePolicyProvider, opTrackerFactory, ioSchedulerProvider);
+ ILSMMergePolicyProvider mergePolicyProvider, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationSchedulerProvider ioSchedulerProvider, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
+ ILinearizeComparatorFactory linearizeCmpFactory, int memPageSize, int memNumPages) {
+ super(mergePolicyProvider, opTrackerFactory, ioSchedulerProvider, ioOpCallbackProvider, memPageSize,
+ memNumPages);
this.btreeComparatorFactories = btreeComparatorFactories;
this.valueProviderFactories = valueProviderFactories;
this.rtreePolicyType = rtreePolicyType;
@@ -52,9 +53,8 @@
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition) {
- return new LSMRTreeDataflowHelper(opDesc, ctx, partition, btreeComparatorFactories, valueProviderFactories,
- rtreePolicyType, flushControllerProvider.getFlushController(ctx),
- mergePolicyProvider.getMergePolicy(ctx), opTrackerFactory, ioSchedulerProvider.getIOScheduler(ctx),
- linearizeCmpFactory);
+ return new LSMRTreeDataflowHelper(opDesc, ctx, partition, memPageSize, memNumPages, btreeComparatorFactories, valueProviderFactories,
+ rtreePolicyType, mergePolicyProvider.getMergePolicy(ctx), opTrackerFactory,
+ ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackProvider, linearizeCmpFactory);
}
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
index d44817c..6f5ecb1 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
@@ -28,7 +28,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -41,21 +41,21 @@
public LSMRTreeWithAntiMatterTuplesDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, IBinaryComparatorFactory[] btreeComparatorFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
ILinearizeComparatorFactory linearizeCmpFactory) {
- super(opDesc, ctx, partition, btreeComparatorFactories, valueProviderFactories, rtreePolicyType,
- flushController, mergePolicy, opTrackerFactory, ioScheduler, linearizeCmpFactory);
+ super(opDesc, ctx, partition, btreeComparatorFactories, valueProviderFactories, rtreePolicyType, mergePolicy,
+ opTrackerFactory, ioScheduler, ioOpCallbackProvider, linearizeCmpFactory);
}
public LSMRTreeWithAntiMatterTuplesDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, int memPageSize, int memNumPages, IBinaryComparatorFactory[] btreeComparatorFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
ILinearizeComparatorFactory linearizeCmpFactory) {
super(opDesc, ctx, partition, memPageSize, memNumPages, btreeComparatorFactories, valueProviderFactories,
- rtreePolicyType, flushController, mergePolicy, opTrackerFactory, ioScheduler, linearizeCmpFactory);
+ rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackProvider, linearizeCmpFactory);
}
@Override
@@ -68,8 +68,8 @@
try {
return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(memBufferCache, memFreePageManager, ioManager, file,
diskBufferCache, diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, rtreePolicyType, flushController, mergePolicy, opTrackerFactory,
- ioScheduler, linearizeCmpFactory, startIODeviceIndex);
+ valueProviderFactories, rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler,
+ ioOpCallbackProvider, linearizeCmpFactory, startIODeviceIndex);
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java
index dddbccc..b27e84f 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java
@@ -22,7 +22,7 @@
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushControllerProvider;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicyProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -35,24 +35,24 @@
private final IBinaryComparatorFactory[] btreeComparatorFactories;
private final IPrimitiveValueProviderFactory[] valueProviderFactories;
private final RTreePolicyType rtreePolicyType;
- private final ILSMFlushControllerProvider flushControllerProvider;
private final ILSMMergePolicyProvider mergePolicyProvider;
private final ILSMOperationTrackerFactory opTrackerProvider;
private final ILSMIOOperationSchedulerProvider ioSchedulerProvider;
+ private final ILSMIOOperationCallbackProvider ioOpCallbackProvider;
private final ILinearizeComparatorFactory linearizeCmpFactory;
public LSMRTreeWithAntiMatterTuplesDataflowHelperFactory(IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, IBinaryComparatorFactory[] btreeComparatorFactories,
- ILSMFlushControllerProvider flushControllerProvider, ILSMMergePolicyProvider mergePolicyProvider,
- ILSMOperationTrackerFactory opTrackerProvider, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
+ ILSMMergePolicyProvider mergePolicyProvider, ILSMOperationTrackerFactory opTrackerProvider,
+ ILSMIOOperationSchedulerProvider ioSchedulerProvider, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
ILinearizeComparatorFactory linearizeCmpFactory) {
this.btreeComparatorFactories = btreeComparatorFactories;
this.valueProviderFactories = valueProviderFactories;
this.rtreePolicyType = rtreePolicyType;
- this.flushControllerProvider = flushControllerProvider;
this.mergePolicyProvider = mergePolicyProvider;
this.ioSchedulerProvider = ioSchedulerProvider;
this.opTrackerProvider = opTrackerProvider;
+ this.ioOpCallbackProvider = ioOpCallbackProvider;
this.linearizeCmpFactory = linearizeCmpFactory;
}
@@ -60,8 +60,7 @@
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition) {
return new LSMRTreeWithAntiMatterTuplesDataflowHelper(opDesc, ctx, partition, btreeComparatorFactories,
- valueProviderFactories, rtreePolicyType, flushControllerProvider.getFlushController(ctx),
- mergePolicyProvider.getMergePolicy(ctx), opTrackerProvider, ioSchedulerProvider.getIOScheduler(ctx),
- linearizeCmpFactory);
+ valueProviderFactories, rtreePolicyType, mergePolicyProvider.getMergePolicy(ctx), opTrackerProvider,
+ ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackProvider, linearizeCmpFactory);
}
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index 13fc738..5a1482f 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -16,7 +16,6 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
import java.io.File;
-import java.util.ArrayList;
import java.util.List;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -31,6 +30,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IInMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOperationContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
@@ -40,17 +40,17 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentState;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
@@ -64,7 +64,7 @@
protected final IBinaryComparatorFactory[] linearizerArray;
// In-memory components.
- protected final LSMRTreeComponent mutableComponent;
+ protected final LSMRTreeMutableComponent mutableComponent;
protected final IInMemoryBufferCache memBufferCache;
// This is used to estimate number of tuples in the memory RTree and BTree
@@ -93,17 +93,17 @@
ILSMComponentFactory componentFactory, IFileMapProvider diskFileMapProvider, int fieldCount,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) {
- super(memFreePageManager, diskRTreeFactory.getBufferCache(), fileManager, diskFileMapProvider, flushController,
- mergePolicy, opTrackerFactory, ioScheduler);
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
+ super(memFreePageManager, diskRTreeFactory.getBufferCache(), fileManager, diskFileMapProvider, mergePolicy,
+ opTrackerFactory, ioScheduler, ioOpCallbackProvider);
RTree memRTree = new RTree(memBufferCache, ((InMemoryBufferCache) memBufferCache).getFileMapProvider(),
memFreePageManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories, fieldCount,
new FileReference(new File("memrtree")));
BTree memBTree = new BTree(memBufferCache, ((InMemoryBufferCache) memBufferCache).getFileMapProvider(),
memFreePageManager, btreeInteriorFrameFactory, btreeLeafFrameFactory, btreeCmpFactories, fieldCount,
new FileReference(new File("membtree")));
- mutableComponent = new LSMRTreeComponent(memRTree, memBTree);
+ mutableComponent = new LSMRTreeMutableComponent(memRTree, memBTree, memFreePageManager);
this.memBufferCache = memBufferCache;
this.rtreeInteriorFrameFactory = rtreeInteriorFrameFactory;
this.rtreeLeafFrameFactory = rtreeLeafFrameFactory;
@@ -128,7 +128,7 @@
fileManager.deleteDirs();
fileManager.createDirs();
- immutableComponents.clear();
+ componentsRef.get().clear();
}
@Override
@@ -145,19 +145,22 @@
}
@Override
- public synchronized void deactivate() throws HyracksDataException {
+ public synchronized void deactivate(boolean flushOnExit) throws HyracksDataException {
if (!isActivated) {
return;
}
- BlockingIOOperationCallback cb = new BlockingIOOperationCallback();
- ILSMIndexAccessor accessor = (ILSMIndexAccessor) createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
- lsmHarness.getIOScheduler().scheduleOperation(accessor.createFlushOperation(cb));
- try {
- cb.waitForIO();
- } catch (InterruptedException e) {
- throw new HyracksDataException(e);
+ if (flushOnExit) {
+ BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(
+ ioOpCallbackProvider.getIOOperationCallback(this));
+ ILSMIndexAccessor accessor = (ILSMIndexAccessor) createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ accessor.scheduleFlush(cb);
+ try {
+ cb.waitForIO();
+ } catch (InterruptedException e) {
+ throw new HyracksDataException(e);
+ }
}
mutableComponent.getRTree().deactivate();
@@ -188,33 +191,33 @@
}
@Override
- public List<ILSMComponent> getOperationalComponents(IIndexOperationContext ctx) {
- List<ILSMComponent> operationalComponents = new ArrayList<ILSMComponent>();
+ public void getOperationalComponents(ILSMIndexOperationContext ctx) {
+ List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
+ operationalComponents.clear();
+ List<ILSMComponent> immutableComponents = componentsRef.get();
switch (ctx.getOperation()) {
+ case INSERT:
+ case DELETE:
+ case FLUSH:
+ operationalComponents.add(mutableComponent);
+ break;
case SEARCH:
- // TODO: We should add the mutable component at some point.
+ operationalComponents.add(mutableComponent);
operationalComponents.addAll(immutableComponents);
break;
case MERGE:
- // TODO: determining the participating components in a merge should probably the task of the merge policy.
- if (immutableComponents.size() > 1) {
- for (ILSMComponent c : immutableComponents) {
- if (c.negativeCompareAndSet(LSMComponentState.MERGING, LSMComponentState.MERGING)) {
- operationalComponents.add(c);
- }
- }
- }
+ operationalComponents.addAll(immutableComponents);
break;
default:
throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
}
- return operationalComponents;
}
protected LSMComponentFileReferences getMergeTargetFileName(List<ILSMComponent> mergingDiskComponents)
throws HyracksDataException {
- RTree firstTree = ((LSMRTreeComponent) mergingDiskComponents.get(0)).getRTree();
- RTree lastTree = ((LSMRTreeComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1)).getRTree();
+ RTree firstTree = ((LSMRTreeImmutableComponent) mergingDiskComponents.get(0)).getRTree();
+ RTree lastTree = ((LSMRTreeImmutableComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1))
+ .getRTree();
FileReference firstFile = diskFileMapProvider.lookupFileName(firstTree.getFileId());
FileReference lastFile = diskFileMapProvider.lookupFileName(lastTree.getFileId());
LSMComponentFileReferences fileRefs = fileManager.getRelMergeFileReference(firstFile.getFile().getName(),
@@ -222,10 +225,10 @@
return fileRefs;
}
- protected LSMRTreeComponent createDiskComponent(ILSMComponentFactory factory, FileReference insertFileRef,
+ protected LSMRTreeImmutableComponent createDiskComponent(ILSMComponentFactory factory, FileReference insertFileRef,
FileReference deleteFileRef, boolean createComponent) throws HyracksDataException, IndexException {
// Create new tree instance.
- LSMRTreeComponent component = (LSMRTreeComponent) factory
+ LSMRTreeImmutableComponent component = (LSMRTreeImmutableComponent) factory
.createLSMComponentInstance(new LSMComponentFileReferences(insertFileRef, deleteFileRef));
if (createComponent) {
component.getRTree().create();
@@ -272,11 +275,10 @@
}
@Override
- public void insertUpdateOrDelete(ITupleReference tuple, IIndexOperationContext ictx) throws HyracksDataException,
- IndexException {
+ public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
LSMRTreeOpContext ctx = (LSMRTreeOpContext) ictx;
if (ctx.getOperation() == IndexOperation.PHYSICALDELETE) {
- throw new UnsupportedOperationException("Physical delete not yet supported in LSM R-tree");
+ throw new UnsupportedOperationException("Physical delete not supported in the LSM-RTree");
}
if (ctx.getOperation() == IndexOperation.INSERT) {
@@ -324,24 +326,14 @@
}
}
- @Override
- public void resetMutableComponent() throws HyracksDataException {
- memFreePageManager.reset();
- mutableComponent.getRTree().clear();
- mutableComponent.getBTree().clear();
- memRTreeTuples = 0;
- memBTreeTuples = 0;
- }
-
- protected LSMRTreeOpContext createOpContext() {
- return new LSMRTreeOpContext((RTree.RTreeAccessor) mutableComponent.getRTree().createAccessor(
- NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE),
- (IRTreeLeafFrame) rtreeLeafFrameFactory.createFrame(),
+ protected LSMRTreeOpContext createOpContext(IModificationOperationCallback modCallback) {
+ return new LSMRTreeOpContext((RTree.RTreeAccessor) mutableComponent.getRTree().createAccessor(modCallback,
+ NoOpOperationCallback.INSTANCE), (IRTreeLeafFrame) rtreeLeafFrameFactory.createFrame(),
(IRTreeInteriorFrame) rtreeInteriorFrameFactory.createFrame(), memFreePageManager
.getMetaDataFrameFactory().createFrame(), 4, (BTree.BTreeAccessor) mutableComponent.getBTree()
- .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE),
- btreeLeafFrameFactory, btreeInteriorFrameFactory, memFreePageManager.getMetaDataFrameFactory()
- .createFrame(), rtreeCmpFactories, btreeCmpFactories, null, null);
+ .createAccessor(modCallback, NoOpOperationCallback.INSTANCE), btreeLeafFrameFactory,
+ btreeInteriorFrameFactory, memFreePageManager.getMetaDataFrameFactory().createFrame(),
+ rtreeCmpFactories, btreeCmpFactories, null, null);
}
@Override
@@ -350,7 +342,7 @@
}
public boolean isEmptyIndex() throws HyracksDataException {
- return immutableComponents.isEmpty()
+ return componentsRef.get().isEmpty()
&& mutableComponent.getBTree().isEmptyTree(
mutableComponent.getBTree().getInteriorFrameFactory().createFrame())
&& mutableComponent.getRTree().isEmptyTree(
@@ -367,9 +359,4 @@
InMemoryBufferCache memBufferCache = (InMemoryBufferCache) mutableComponent.getRTree().getBufferCache();
return memBufferCache.getNumPages() * memBufferCache.getPageSize();
}
-
- @Override
- public ILSMComponent getMutableComponent() {
- return mutableComponent;
- }
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 2019d43..e43e525 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -15,7 +15,6 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
-import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
@@ -26,10 +25,8 @@
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IInMemoryFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
@@ -43,11 +40,12 @@
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -69,13 +67,13 @@
TreeIndexFactory<BTree> diskBTreeFactory, IFileMapProvider diskFileMapProvider, int fieldCount,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) {
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
super(memBufferCache, memFreePageManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager, diskRTreeFactory,
new LSMRTreeComponentFactory(diskRTreeFactory, diskBTreeFactory), diskFileMapProvider, fieldCount,
- rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields, linearizerArray, flushController,
- mergePolicy, opTrackerFactory, ioScheduler);
+ rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields, linearizerArray, mergePolicy,
+ opTrackerFactory, ioScheduler, ioOpCallbackProvider);
}
/**
@@ -89,6 +87,7 @@
@Override
public synchronized void activate() throws HyracksDataException {
super.activate();
+ List<ILSMComponent> immutableComponents = componentsRef.get();
List<LSMComponentFileReferences> validFileReferences;
try {
validFileReferences = fileManager.cleanupAndGetValidFiles();
@@ -97,7 +96,7 @@
}
immutableComponents.clear();
for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
- LSMRTreeComponent component;
+ LSMRTreeImmutableComponent component;
try {
component = createDiskComponent(componentFactory,
lsmComonentFileReference.getInsertIndexFileReference(),
@@ -111,10 +110,11 @@
}
@Override
- public synchronized void deactivate() throws HyracksDataException {
- super.deactivate();
+ public synchronized void deactivate(boolean flushOnExit) throws HyracksDataException {
+ super.deactivate(flushOnExit);
+ List<ILSMComponent> immutableComponents = componentsRef.get();
for (ILSMComponent c : immutableComponents) {
- LSMRTreeComponent component = (LSMRTreeComponent) c;
+ LSMRTreeImmutableComponent component = (LSMRTreeImmutableComponent) c;
RTree rtree = component.getRTree();
BTree btree = component.getBTree();
rtree.deactivate();
@@ -124,10 +124,16 @@
}
@Override
+ public synchronized void deactivate() throws HyracksDataException {
+ deactivate(true);
+ }
+
+ @Override
public synchronized void destroy() throws HyracksDataException {
super.destroy();
+ List<ILSMComponent> immutableComponents = componentsRef.get();
for (ILSMComponent c : immutableComponents) {
- LSMRTreeComponent component = (LSMRTreeComponent) c;
+ LSMRTreeImmutableComponent component = (LSMRTreeImmutableComponent) c;
component.getBTree().destroy();
component.getRTree().destroy();
}
@@ -137,8 +143,9 @@
@Override
public synchronized void clear() throws HyracksDataException {
super.clear();
+ List<ILSMComponent> immutableComponents = componentsRef.get();
for (ILSMComponent c : immutableComponents) {
- LSMRTreeComponent component = (LSMRTreeComponent) c;
+ LSMRTreeImmutableComponent component = (LSMRTreeImmutableComponent) c;
component.getBTree().deactivate();
component.getRTree().deactivate();
component.getBTree().destroy();
@@ -148,12 +155,14 @@
}
@Override
- public void search(IIndexCursor cursor, List<ILSMComponent> immutableComponents, ISearchPredicate pred,
- IIndexOperationContext ictx, boolean includeMutableComponent) throws HyracksDataException, IndexException {
+ public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred)
+ throws HyracksDataException, IndexException {
LSMRTreeOpContext ctx = (LSMRTreeOpContext) ictx;
- int numDiskComponents = immutableComponents.size();
- int numTrees = (includeMutableComponent) ? numDiskComponents + 1 : numDiskComponents;
+ List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
+ boolean includeMutableComponent = operationalComponents.get(0) == mutableComponent;
+ int numTrees = operationalComponents.size();
+ ListIterator<ILSMComponent> diskComponentIter = operationalComponents.listIterator();
ITreeIndexAccessor[] rTreeAccessors = new ITreeIndexAccessor[numTrees];
ITreeIndexAccessor[] bTreeAccessors = new ITreeIndexAccessor[numTrees];
int diskComponentIx = 0;
@@ -161,11 +170,11 @@
rTreeAccessors[0] = ctx.memRTreeAccessor;
bTreeAccessors[0] = ctx.memBTreeAccessor;
diskComponentIx++;
+ diskComponentIter.next();
}
- ListIterator<ILSMComponent> diskComponentIter = immutableComponents.listIterator();
while (diskComponentIter.hasNext()) {
- LSMRTreeComponent component = (LSMRTreeComponent) diskComponentIter.next();
+ LSMRTreeImmutableComponent component = (LSMRTreeImmutableComponent) diskComponentIter.next();
RTree diskRTree = component.getRTree();
BTree diskBTree = component.getBTree();
rTreeAccessors[diskComponentIx] = diskRTree.createAccessor(NoOpOperationCallback.INSTANCE,
@@ -175,11 +184,6 @@
diskComponentIx++;
}
- List<ILSMComponent> operationalComponents = new ArrayList<ILSMComponent>();
- if (includeMutableComponent) {
- operationalComponents.add(getMutableComponent());
- }
- operationalComponents.addAll(immutableComponents);
LSMRTreeCursorInitialState initialState = new LSMRTreeCursorInitialState(numTrees, rtreeLeafFrameFactory,
rtreeInteriorFrameFactory, btreeLeafFrameFactory, ctx.getBTreeMultiComparator(), rTreeAccessors,
bTreeAccessors, includeMutableComponent, lsmHarness, comparatorFields, linearizerArray,
@@ -188,19 +192,33 @@
}
@Override
+ public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ throws HyracksDataException {
+ LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
+ ILSMIndexOperationContext rctx = createOpContext(NoOpOperationCallback.INSTANCE);
+ LSMRTreeMutableComponent flushingComponent = (LSMRTreeMutableComponent) ctx.getComponentHolder().get(0);
+ rctx.setOperation(IndexOperation.FLUSH);
+ rctx.getComponentHolder().addAll(ctx.getComponentHolder());
+ LSMRTreeAccessor accessor = new LSMRTreeAccessor(lsmHarness, rctx);
+ ioScheduler.scheduleOperation(new LSMRTreeFlushOperation(accessor, flushingComponent, componentFileRefs
+ .getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(), callback));
+ }
+
+ @Override
public ILSMComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
LSMRTreeFlushOperation flushOp = (LSMRTreeFlushOperation) operation;
+ LSMRTreeMutableComponent flushingComponent = flushOp.getFlushingComponent();
// Renaming order is critical because we use assume ordering when we
// read the file names when we open the tree.
// The RTree should be renamed before the BTree.
// scan the memory RTree
- ITreeIndexAccessor memRTreeAccessor = mutableComponent.getRTree().createAccessor(
+ ITreeIndexAccessor memRTreeAccessor = flushingComponent.getRTree().createAccessor(
NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
RTreeSearchCursor rtreeScanCursor = (RTreeSearchCursor) memRTreeAccessor.createSearchCursor();
SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
- LSMRTreeComponent component = createDiskComponent(componentFactory, flushOp.getRTreeFlushTarget(),
+ LSMRTreeImmutableComponent component = createDiskComponent(componentFactory, flushOp.getRTreeFlushTarget(),
flushOp.getBTreeFlushTarget(), true);
RTree diskRTree = component.getRTree();
IIndexBulkLoader rTreeBulkloader;
@@ -209,9 +227,9 @@
IBinaryComparatorFactory[] linearizerArray = { linearizer };
if (rTreeTupleSorter == null) {
- rTreeTupleSorter = new TreeTupleSorter(memRTreeTuples, mutableComponent.getRTree().getFileId(),
+ rTreeTupleSorter = new TreeTupleSorter(memRTreeTuples, flushingComponent.getRTree().getFileId(),
linearizerArray, rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(),
- mutableComponent.getRTree().getBufferCache(), comparatorFields);
+ flushingComponent.getRTree().getBufferCache(), comparatorFields);
} else {
rTreeTupleSorter.reset();
}
@@ -248,7 +266,7 @@
rTreeBulkloader.end();
// scan the memory BTree
- ITreeIndexAccessor memBTreeAccessor = mutableComponent.getBTree().createAccessor(
+ ITreeIndexAccessor memBTreeAccessor = flushingComponent.getBTree().createAccessor(
NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
IIndexCursor btreeScanCursor = memBTreeAccessor.createSearchCursor();
RangePredicate btreeNullPredicate = new RangePredicate(null, null, true, true, null, null);
@@ -267,7 +285,30 @@
btreeScanCursor.close();
}
bTreeBulkloader.end();
- return new LSMRTreeComponent(diskRTree, diskBTree);
+ memRTreeTuples = 0;
+ memBTreeTuples = 0;
+ return new LSMRTreeImmutableComponent(diskRTree, diskBTree);
+ }
+
+ @Override
+ public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ throws HyracksDataException, IndexException {
+ // Renaming order is critical because we use assume ordering when we
+ // read the file names when we open the tree.
+ // The RTree should be renamed before the BTree.
+ List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
+ ILSMIndexOperationContext rctx = createOpContext(NoOpOperationCallback.INSTANCE);
+ rctx.getComponentHolder().addAll(mergingComponents);
+ ITreeIndexCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer);
+ ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
+ search(rctx, cursor, rtreeSearchPred);
+
+ rctx.setOperation(IndexOperation.MERGE);
+ LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
+ ILSMIndexAccessorInternal accessor = new LSMRTreeAccessor(lsmHarness, rctx);
+ ioScheduler.scheduleOperation(new LSMRTreeMergeOperation((ILSMIndexAccessorInternal) accessor,
+ mergingComponents, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs
+ .getDeleteIndexFileReference(), callback));
}
@Override
@@ -284,7 +325,7 @@
}
// Bulk load the tuples from all on-disk RTrees into the new RTree.
- LSMRTreeComponent component = createDiskComponent(componentFactory, mergeOp.getRTreeMergeTarget(),
+ LSMRTreeImmutableComponent component = createDiskComponent(componentFactory, mergeOp.getRTreeMergeTarget(),
mergeOp.getBTreeMergeTarget(), true);
RTree mergedRTree = component.getRTree();
BTree mergedBTree = component.getBTree();
@@ -304,41 +345,13 @@
// Load an empty BTree tree.
mergedBTree.createBulkLoader(1.0f, false).end();
- return new LSMRTreeComponent(mergedRTree, mergedBTree);
+ return new LSMRTreeImmutableComponent(mergedRTree, mergedBTree);
}
@Override
- public ILSMIOOperation createMergeOperation(ILSMIOOperationCallback callback) throws HyracksDataException {
- // Renaming order is critical because we use assume ordering when we
- // read the file names when we open the tree.
- // The RTree should be renamed before the BTree.
- ILSMIndexOperationContext ctx = createOpContext();
- ctx.setOperation(IndexOperation.MERGE);
- ITreeIndexCursor cursor;
- cursor = new LSMRTreeSortedCursor(ctx, linearizer);
- ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
- // Scan the RTrees, ignoring the in-memory RTree.
- List<ILSMComponent> mergingComponents;
- try {
- mergingComponents = lsmHarness.search(cursor, rtreeSearchPred, ctx, false);
- } catch (IndexException e) {
- throw new HyracksDataException(e);
- }
- // Nothing to merge.
- if (mergingComponents.size() <= 1) {
- cursor.close();
- return null;
- }
- LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
- return new LSMRTreeMergeOperation(lsmHarness.getIndex(), mergingComponents, cursor,
- relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(),
- callback);
- }
-
- @Override
- public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
+ public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
- return new LSMRTreeAccessor(lsmHarness, createOpContext());
+ return new LSMRTreeAccessor(lsmHarness, createOpContext(modificationCallback));
}
public class LSMRTreeAccessor extends LSMTreeIndexAccessor {
@@ -355,13 +368,6 @@
LSMRTreeOpContext concreteCtx = (LSMRTreeOpContext) ctx;
return concreteCtx.rtreeOpContext.cmp;
}
-
- @Override
- public ILSMIOOperation createFlushOperation(ILSMIOOperationCallback callback) {
- LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- return new LSMRTreeFlushOperation(lsmHarness.getIndex(), componentFileRefs.getInsertIndexFileReference(),
- componentFileRefs.getDeleteIndexFileReference(), callback);
- }
}
private ILSMComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
@@ -389,7 +395,7 @@
} catch (IndexException e) {
throw new TreeIndexException(e);
}
- bulkLoader = ((LSMRTreeComponent) component).getRTree().createBulkLoader(fillFactor, verifyInput);
+ bulkLoader = ((LSMRTreeImmutableComponent) component).getRTree().createBulkLoader(fillFactor, verifyInput);
}
@Override
@@ -415,16 +421,16 @@
}
protected void handleException() throws HyracksDataException {
- ((LSMRTreeComponent) component).getRTree().deactivate();
- ((LSMRTreeComponent) component).getRTree().destroy();
- ((LSMRTreeComponent) component).getBTree().deactivate();
- ((LSMRTreeComponent) component).getBTree().destroy();
+ ((LSMRTreeImmutableComponent) component).getRTree().deactivate();
+ ((LSMRTreeImmutableComponent) component).getRTree().destroy();
+ ((LSMRTreeImmutableComponent) component).getBTree().deactivate();
+ ((LSMRTreeImmutableComponent) component).getBTree().destroy();
}
}
@Override
public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException {
- LSMRTreeComponent component = (LSMRTreeComponent) lsmComponent;
+ LSMRTreeImmutableComponent component = (LSMRTreeImmutableComponent) lsmComponent;
forceFlushDirtyPages(component.getRTree());
markAsValidInternal(component.getRTree());
forceFlushDirtyPages(component.getBTree());
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
index e5af9ee..41ffd3d 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
@@ -105,7 +105,7 @@
rtreeCursors = null;
btreeCursors = null;
} finally {
- lsmHarness.closeSearchCursor(operationalComponents, includeMemRTree, opCtx);
+ lsmHarness.endSearch(opCtx);
}
open = false;
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeComponentFactory.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeComponentFactory.java
index b875c67..1681fbd 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeComponentFactory.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeComponentFactory.java
@@ -35,7 +35,7 @@
@Override
public ILSMComponent createLSMComponentInstance(LSMComponentFileReferences cfr) throws IndexException {
- return new LSMRTreeComponent(rtreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()),
+ return new LSMRTreeImmutableComponent(rtreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()),
btreeFactory.createIndexInstance(cfr.getDeleteIndexFileReference()));
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
index 5ec4776..8698a1d 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
@@ -1,47 +1,49 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
import java.util.Collections;
-import java.util.List;
+import java.util.HashSet;
+import java.util.Set;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
-import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
public class LSMRTreeFlushOperation implements ILSMIOOperation {
- private final ILSMIndex index;
+ private final ILSMIndexAccessorInternal accessor;
+ private final LSMRTreeMutableComponent flushingComponent;
private final FileReference rtreeFlushTarget;
private final FileReference btreeFlushTarget;
private final ILSMIOOperationCallback callback;
- public LSMRTreeFlushOperation(ILSMIndex index, FileReference rtreeFlushTarget, FileReference btreeFlushTarget,
- ILSMIOOperationCallback callback) {
- this.index = index;
+ public LSMRTreeFlushOperation(ILSMIndexAccessorInternal accessor, LSMRTreeMutableComponent flushingComponent,
+ FileReference rtreeFlushTarget, FileReference btreeFlushTarget, ILSMIOOperationCallback callback) {
+ this.accessor = accessor;
+ this.flushingComponent = flushingComponent;
this.rtreeFlushTarget = rtreeFlushTarget;
this.btreeFlushTarget = btreeFlushTarget;
this.callback = callback;
}
@Override
- public List<IODeviceHandle> getReadDevices() {
- return Collections.emptyList();
+ public Set<IODeviceHandle> getReadDevices() {
+ return Collections.emptySet();
}
@Override
- public List<IODeviceHandle> getWriteDevices() {
- return Collections.singletonList(rtreeFlushTarget.getDevideHandle());
+ public Set<IODeviceHandle> getWriteDevices() {
+ Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ devs.add(rtreeFlushTarget.getDeviceHandle());
+ devs.add(btreeFlushTarget.getDeviceHandle());
+ return devs;
}
@Override
public void perform() throws HyracksDataException, IndexException {
- ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
accessor.flush(this);
}
@@ -57,4 +59,8 @@
public FileReference getBTreeFlushTarget() {
return btreeFlushTarget;
}
+
+ public LSMRTreeMutableComponent getFlushingComponent() {
+ return flushingComponent;
+ }
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeImmutableComponent.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeImmutableComponent.java
new file mode 100644
index 0000000..afba3a0
--- /dev/null
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeImmutableComponent.java
@@ -0,0 +1,35 @@
+package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractImmutableLSMComponent;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
+
+public class LSMRTreeImmutableComponent extends AbstractImmutableLSMComponent {
+ private final RTree rtree;
+ private final BTree btree;
+
+ public LSMRTreeImmutableComponent(RTree rtree, BTree btree) {
+ this.rtree = rtree;
+ this.btree = btree;
+ }
+
+ @Override
+ public void destroy() throws HyracksDataException {
+ rtree.deactivate();
+ rtree.destroy();
+ if (btree != null) {
+ btree.deactivate();
+ btree.destroy();
+ }
+ }
+
+ public RTree getRTree() {
+ return rtree;
+ }
+
+ public BTree getBTree() {
+ return btree;
+ }
+
+}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java
index 7449340..970b253 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java
@@ -1,32 +1,32 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
-import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
-import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
public class LSMRTreeMergeOperation implements ILSMIOOperation {
- private final ILSMIndex index;
+ private final ILSMIndexAccessorInternal accessor;
private final List<ILSMComponent> mergingComponents;
private final ITreeIndexCursor cursor;
private final FileReference rtreeMergeTarget;
private final FileReference btreeMergeTarget;
private final ILSMIOOperationCallback callback;
- public LSMRTreeMergeOperation(ILSMIndex index, List<ILSMComponent> mergingComponents, ITreeIndexCursor cursor,
- FileReference rtreeMergeTarget, FileReference btreeMergeTarget, ILSMIOOperationCallback callback) {
- this.index = index;
+ public LSMRTreeMergeOperation(ILSMIndexAccessorInternal accessor, List<ILSMComponent> mergingComponents,
+ ITreeIndexCursor cursor, FileReference rtreeMergeTarget, FileReference btreeMergeTarget,
+ ILSMIOOperationCallback callback) {
+ this.accessor = accessor;
this.mergingComponents = mergingComponents;
this.cursor = cursor;
this.rtreeMergeTarget = rtreeMergeTarget;
@@ -35,24 +35,23 @@
}
@Override
- public List<IODeviceHandle> getReadDevices() {
- List<IODeviceHandle> devs = new ArrayList<IODeviceHandle>();
+ public Set<IODeviceHandle> getReadDevices() {
+ Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
for (ILSMComponent o : mergingComponents) {
- LSMRTreeComponent component = (LSMRTreeComponent) o;
- devs.add(component.getRTree().getFileReference().getDevideHandle());
+ LSMRTreeImmutableComponent component = (LSMRTreeImmutableComponent) o;
+ devs.add(component.getRTree().getFileReference().getDeviceHandle());
+ devs.add(component.getBTree().getFileReference().getDeviceHandle());
}
return devs;
}
@Override
- public List<IODeviceHandle> getWriteDevices() {
- return Collections.singletonList(rtreeMergeTarget.getDevideHandle());
+ public Set<IODeviceHandle> getWriteDevices() {
+ return Collections.singleton(rtreeMergeTarget.getDeviceHandle());
}
@Override
public void perform() throws HyracksDataException, IndexException {
- ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
accessor.merge(this);
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeComponent.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMutableComponent.java
similarity index 67%
rename from hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeComponent.java
rename to hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMutableComponent.java
index 4b53055..80f76a1 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeComponent.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMutableComponent.java
@@ -17,37 +17,20 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractLSMComponent;
+import edu.uci.ics.hyracks.storage.am.common.api.IInMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractMutableLSMComponent;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
-public class LSMRTreeComponent extends AbstractLSMComponent {
+public class LSMRTreeMutableComponent extends AbstractMutableLSMComponent {
private final RTree rtree;
private final BTree btree;
+ private final IInMemoryFreePageManager mfpm;
- public LSMRTreeComponent(RTree rtree, BTree btree) {
+ public LSMRTreeMutableComponent(RTree rtree, BTree btree, IInMemoryFreePageManager mfpm) {
this.rtree = rtree;
this.btree = btree;
- }
-
- @Override
- public void destroy() throws HyracksDataException {
- rtree.deactivate();
- rtree.destroy();
- if (btree != null) {
- btree.deactivate();
- btree.destroy();
- }
- }
-
- @Override
- public void reset() throws HyracksDataException {
- ((InMemoryFreePageManager) rtree.getFreePageManager()).reset();
- rtree.clear();
- if (btree != null) {
- btree.clear();
- }
+ this.mfpm = mfpm;
}
public RTree getRTree() {
@@ -57,4 +40,17 @@
public BTree getBTree() {
return btree;
}
+
+ @Override
+ protected boolean isFull() {
+ return mfpm.isFull();
+ }
+
+ @Override
+ protected void reset() throws HyracksDataException {
+ rtree.clear();
+ if (btree != null) {
+ btree.clear();
+ }
+ }
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
index 7dc3107..b8805d1 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
@@ -15,6 +15,9 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
+import java.util.LinkedList;
+import java.util.List;
+
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
@@ -25,6 +28,7 @@
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
@@ -38,6 +42,7 @@
public final RTree.RTreeAccessor memRTreeAccessor;
public final BTree.BTreeAccessor memBTreeAccessor;
private IndexOperation op;
+ public final List<ILSMComponent> componentHolder;
public final IModificationOperationCallback modificationCallback;
public final ISearchOperationCallback searchCallback;
@@ -49,6 +54,7 @@
IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) {
this.memRTreeAccessor = memRtreeAccessor;
this.memBTreeAccessor = memBtreeAccessor;
+ this.componentHolder = new LinkedList<ILSMComponent>();
this.modificationCallback = modificationCallback;
this.searchCallback = searchCallback;
this.rtreeOpContext = new RTreeOpContext(rtreeLeafFrame, rtreeInteriorFrame, rtreeMetaFrame, rtreeCmpFactories,
@@ -58,6 +64,7 @@
}
public void setOperation(IndexOperation newOp) {
+ reset();
if (newOp == IndexOperation.INSERT) {
rtreeOpContext.setOperation(newOp);
} else if (newOp == IndexOperation.DELETE) {
@@ -68,7 +75,7 @@
@Override
public void reset() {
-
+ componentHolder.clear();
}
@Override
@@ -81,6 +88,11 @@
}
@Override
+ public List<ILSMComponent> getComponentHolder() {
+ return componentHolder;
+ }
+
+ @Override
public ISearchOperationCallback getSearchOperationCallback() {
return searchCallback;
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java
index 53f467b..f5e6f3d 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java
@@ -48,7 +48,7 @@
rtreeCursors = null;
btreeCursors = null;
} finally {
- lsmHarness.closeSearchCursor(operationalComponents, includeMemRTree, opCtx);
+ lsmHarness.endSearch(opCtx);
}
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java
index 9a4db5f..46cf050 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java
@@ -57,7 +57,7 @@
}
} finally {
if (open) {
- lsmHarness.closeSearchCursor(operationalComponents, includeMemRTree, opCtx);
+ lsmHarness.endSearch(opCtx);
}
}
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index 99eed2c..824b8ef 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -15,7 +15,6 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
-import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
@@ -26,10 +25,8 @@
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IInMemoryFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
@@ -44,11 +41,12 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -78,14 +76,14 @@
TreeIndexFactory<RTree> diskRTreeFactory, TreeIndexFactory<RTree> bulkLoadRTreeFactory,
IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
- int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMFlushController flushController,
- ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
- ILSMIOOperationScheduler ioScheduler) {
+ int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMMergePolicy mergePolicy,
+ ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
super(memBufferCache, memFreePageManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
btreeInteriorFrameFactory, btreeLeafFrameFactory, fileManager, diskRTreeFactory,
new LSMRTreeWithAntiMatterTuplesComponentFactory(diskRTreeFactory), diskFileMapProvider, fieldCount,
- rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields, linearizerArray, flushController,
- mergePolicy, opTrackerFactory, ioScheduler);
+ rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields, linearizerArray, mergePolicy,
+ opTrackerFactory, ioScheduler, ioOpCallbackProvider);
bulkLoaComponentFactory = new LSMRTreeWithAntiMatterTuplesComponentFactory(bulkLoadRTreeFactory);
this.bTreeTupleSorter = null;
}
@@ -93,6 +91,7 @@
@Override
public synchronized void activate() throws HyracksDataException {
super.activate();
+ List<ILSMComponent> immutableComponents = componentsRef.get();
immutableComponents.clear();
List<LSMComponentFileReferences> validFileReferences;
try {
@@ -101,7 +100,7 @@
throw new HyracksDataException(e);
}
for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
- LSMRTreeComponent component;
+ LSMRTreeImmutableComponent component;
try {
component = createDiskComponent(componentFactory,
lsmComonentFileReference.getInsertIndexFileReference(), null, false);
@@ -114,20 +113,27 @@
}
@Override
- public synchronized void deactivate() throws HyracksDataException {
- super.deactivate();
+ public synchronized void deactivate(boolean flushOnExit) throws HyracksDataException {
+ super.deactivate(flushOnExit);
+ List<ILSMComponent> immutableComponents = componentsRef.get();
for (ILSMComponent c : immutableComponents) {
- RTree rtree = (RTree) ((LSMRTreeComponent) c).getRTree();
+ RTree rtree = (RTree) ((LSMRTreeImmutableComponent) c).getRTree();
rtree.deactivate();
}
isActivated = false;
}
@Override
+ public synchronized void deactivate() throws HyracksDataException {
+ deactivate(true);
+ }
+
+ @Override
public synchronized void destroy() throws HyracksDataException {
super.destroy();
+ List<ILSMComponent> immutableComponents = componentsRef.get();
for (ILSMComponent c : immutableComponents) {
- RTree rtree = (RTree) ((LSMRTreeComponent) c).getRTree();
+ RTree rtree = (RTree) ((LSMRTreeImmutableComponent) c).getRTree();
rtree.destroy();
}
fileManager.deleteDirs();
@@ -136,8 +142,9 @@
@Override
public synchronized void clear() throws HyracksDataException {
super.clear();
+ List<ILSMComponent> immutableComponents = componentsRef.get();
for (ILSMComponent c : immutableComponents) {
- RTree rtree = (RTree) ((LSMRTreeComponent) c).getRTree();
+ RTree rtree = (RTree) ((LSMRTreeImmutableComponent) c).getRTree();
rtree.deactivate();
rtree.destroy();
}
@@ -145,11 +152,13 @@
}
@Override
- public void search(IIndexCursor cursor, List<ILSMComponent> immutableComponents, ISearchPredicate pred,
- IIndexOperationContext ictx, boolean includeMutableComponent) throws HyracksDataException, IndexException {
+ public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred)
+ throws HyracksDataException, IndexException {
LSMRTreeOpContext ctx = (LSMRTreeOpContext) ictx;
+ List<ILSMComponent> operationalComponents = ictx.getComponentHolder();
+ boolean includeMutableComponent = operationalComponents.get(0) == mutableComponent;
LSMRTreeWithAntiMatterTuplesSearchCursor lsmTreeCursor = (LSMRTreeWithAntiMatterTuplesSearchCursor) cursor;
- int numDiskRComponents = immutableComponents.size();
+ int numDiskRComponents = operationalComponents.size();
LSMRTreeCursorInitialState initialState;
ITreeIndexAccessor[] bTreeAccessors = null;
@@ -159,11 +168,6 @@
bTreeAccessors[0] = ctx.memBTreeAccessor;
}
- List<ILSMComponent> operationalComponents = new ArrayList<ILSMComponent>();
- if (includeMutableComponent) {
- operationalComponents.add(getMutableComponent());
- }
- operationalComponents.addAll(immutableComponents);
initialState = new LSMRTreeCursorInitialState(numDiskRComponents, rtreeLeafFrameFactory,
rtreeInteriorFrameFactory, btreeLeafFrameFactory, ctx.getBTreeMultiComparator(), null, bTreeAccessors,
includeMutableComponent, lsmHarness, comparatorFields, linearizerArray, ctx.searchCallback,
@@ -171,17 +175,19 @@
lsmTreeCursor.open(initialState, pred);
+ ListIterator<ILSMComponent> diskComponentsIter = operationalComponents.listIterator();
+ int diskComponentIx = 0;
if (includeMutableComponent) {
// Open cursor of in-memory RTree
ctx.memRTreeAccessor.search(lsmTreeCursor.getMemRTreeCursor(), pred);
+ diskComponentIx++;
+ diskComponentsIter.next();
}
// Open cursors of on-disk RTrees.
ITreeIndexAccessor[] diskRTreeAccessors = new ITreeIndexAccessor[numDiskRComponents];
- int diskComponentIx = 0;
- ListIterator<ILSMComponent> diskComponentsIter = immutableComponents.listIterator();
while (diskComponentsIter.hasNext()) {
- RTree diskRTree = (RTree) ((LSMRTreeComponent) diskComponentsIter.next()).getRTree();
+ RTree diskRTree = (RTree) ((LSMRTreeImmutableComponent) diskComponentsIter.next()).getRTree();
diskRTreeAccessors[diskComponentIx] = diskRTree.createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
diskRTreeAccessors[diskComponentIx].search(lsmTreeCursor.getCursor(diskComponentIx), pred);
@@ -191,23 +197,36 @@
}
@Override
+ public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ throws HyracksDataException {
+ LSMRTreeOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE);
+ LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference();
+ ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
+ opCtx.setOperation(IndexOperation.FLUSH);
+ opCtx.getComponentHolder().add(flushingComponent);
+ ILSMIndexAccessorInternal accessor = new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness, opCtx);
+ ioScheduler.scheduleOperation(new LSMFlushOperation(accessor, flushingComponent, relFlushFileRefs
+ .getInsertIndexFileReference(), callback));
+ }
+
+ @Override
public ILSMComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
LSMFlushOperation flushOp = (LSMFlushOperation) operation;
// Renaming order is critical because we use assume ordering when we
// read the file names when we open the tree.
// The RTree should be renamed before the BTree.
-
- // scan the memory RTree
- ITreeIndexAccessor memRTreeAccessor = mutableComponent.getRTree().createAccessor(
+ LSMRTreeMutableComponent flushingComponent = (LSMRTreeMutableComponent) flushOp.getFlushingComponent();
+ ITreeIndexAccessor memRTreeAccessor = flushingComponent.getRTree().createAccessor(
NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
RTreeSearchCursor rtreeScanCursor = (RTreeSearchCursor) memRTreeAccessor.createSearchCursor();
SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
- LSMRTreeComponent component = createDiskComponent(componentFactory, flushOp.getFlushTarget(), null, true);
+ LSMRTreeImmutableComponent component = createDiskComponent(componentFactory, flushOp.getFlushTarget(), null,
+ true);
RTree diskRTree = component.getRTree();
// scan the memory BTree
- ITreeIndexAccessor memBTreeAccessor = mutableComponent.getBTree().createAccessor(
+ ITreeIndexAccessor memBTreeAccessor = flushingComponent.getBTree().createAccessor(
NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
BTreeRangeSearchCursor btreeScanCursor = (BTreeRangeSearchCursor) memBTreeAccessor.createSearchCursor();
RangePredicate btreeNullPredicate = new RangePredicate(null, null, true, true, null, null);
@@ -216,13 +235,13 @@
// Since the LSM-RTree is used as a secondary assumption, the
// primary key will be the last comparator in the BTree comparators
if (rTreeTupleSorter == null) {
- rTreeTupleSorter = new TreeTupleSorter(memRTreeTuples, mutableComponent.getRTree().getFileId(),
+ rTreeTupleSorter = new TreeTupleSorter(memRTreeTuples, flushingComponent.getRTree().getFileId(),
linearizerArray, rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(),
- mutableComponent.getRTree().getBufferCache(), comparatorFields);
+ flushingComponent.getRTree().getBufferCache(), comparatorFields);
- bTreeTupleSorter = new TreeTupleSorter(memBTreeTuples, mutableComponent.getBTree().getFileId(),
+ bTreeTupleSorter = new TreeTupleSorter(memBTreeTuples, flushingComponent.getBTree().getFileId(),
linearizerArray, btreeLeafFrameFactory.createFrame(), btreeLeafFrameFactory.createFrame(),
- mutableComponent.getBTree().getBufferCache(), comparatorFields);
+ flushingComponent.getBTree().getBufferCache(), comparatorFields);
} else {
rTreeTupleSorter.reset();
bTreeTupleSorter.reset();
@@ -280,10 +299,28 @@
rTreeBulkloader.end();
+ memRTreeTuples = 0;
+ memBTreeTuples = 0;
return component;
}
@Override
+ public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ throws HyracksDataException, IndexException {
+ List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
+ LSMRTreeOpContext rctx = createOpContext(NoOpOperationCallback.INSTANCE);
+ rctx.getComponentHolder().addAll(mergingComponents);
+ ITreeIndexCursor cursor = new LSMRTreeWithAntiMatterTuplesSearchCursor(ctx);
+ ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
+ search(rctx, cursor, (SearchPredicate) rtreeSearchPred);
+ rctx.setOperation(IndexOperation.MERGE);
+ LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
+ ILSMIndexAccessorInternal accessor = new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness, rctx);
+ ioScheduler.scheduleOperation(new LSMRTreeMergeOperation(accessor, mergingComponents, cursor, relMergeFileRefs
+ .getInsertIndexFileReference(), null, callback));
+ }
+
+ @Override
public ILSMComponent merge(List<ILSMComponent> mergedComponents, ILSMIOOperation operation)
throws HyracksDataException, IndexException {
LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation;
@@ -297,7 +334,8 @@
}
// Bulk load the tuples from all on-disk RTrees into the new RTree.
- LSMRTreeComponent component = createDiskComponent(componentFactory, mergeOp.getRTreeMergeTarget(), null, true);
+ LSMRTreeImmutableComponent component = createDiskComponent(componentFactory, mergeOp.getRTreeMergeTarget(),
+ null, true);
RTree mergedRTree = component.getRTree();
IIndexBulkLoader bulkloader = mergedRTree.createBulkLoader(1.0f, false);
try {
@@ -314,9 +352,9 @@
}
@Override
- public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
+ public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
- return new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness, createOpContext());
+ return new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness, createOpContext(modificationCallback));
}
public class LSMRTreeWithAntiMatterTuplesAccessor extends LSMTreeIndexAccessor {
@@ -333,13 +371,6 @@
LSMRTreeOpContext concreteCtx = (LSMRTreeOpContext) ctx;
return concreteCtx.rtreeOpContext.cmp;
}
-
- @Override
- public ILSMIOOperation createFlushOperation(ILSMIOOperationCallback callback) {
- LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference();
- return new LSMFlushOperation(lsmHarness.getIndex(), relFlushFileRefs.getInsertIndexFileReference(),
- callback);
- }
}
@Override
@@ -366,7 +397,7 @@
} catch (IndexException e) {
throw new TreeIndexException(e);
}
- bulkLoader = ((LSMRTreeComponent) component).getRTree().createBulkLoader(fillFactor, verifyInput);
+ bulkLoader = ((LSMRTreeImmutableComponent) component).getRTree().createBulkLoader(fillFactor, verifyInput);
}
@Override
@@ -392,40 +423,15 @@
}
protected void handleException() throws HyracksDataException {
- ((LSMRTreeComponent) component).getRTree().deactivate();
- ((LSMRTreeComponent) component).getRTree().destroy();
+ ((LSMRTreeImmutableComponent) component).getRTree().deactivate();
+ ((LSMRTreeImmutableComponent) component).getRTree().destroy();
}
}
@Override
- public ILSMIOOperation createMergeOperation(ILSMIOOperationCallback callback) throws HyracksDataException {
- LSMRTreeOpContext ctx = createOpContext();
- ctx.setOperation(IndexOperation.MERGE);
- ITreeIndexCursor cursor = new LSMRTreeWithAntiMatterTuplesSearchCursor(ctx);
- ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
- // Ordered scan, ignoring the in-memory RTree.
- // We get back a snapshot of the on-disk RTrees that are going to be
- // merged now, so we can clean them up after the merge has completed.
- List<ILSMComponent> mergingComponents;
- try {
- mergingComponents = lsmHarness.search(cursor, (SearchPredicate) rtreeSearchPred, ctx, false);
- if (mergingComponents.size() <= 1) {
- cursor.close();
- return null;
- }
- } catch (IndexException e) {
- throw new HyracksDataException(e);
- }
-
- LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
- return new LSMRTreeMergeOperation(lsmHarness.getIndex(), mergingComponents, cursor,
- relMergeFileRefs.getInsertIndexFileReference(), null, callback);
- }
-
- @Override
public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException {
- RTree rtree = ((LSMRTreeComponent) lsmComponent).getRTree();
+ RTree rtree = ((LSMRTreeImmutableComponent) lsmComponent).getRTree();
forceFlushDirtyPages(rtree);
markAsValidInternal(rtree);
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesComponentFactory.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesComponentFactory.java
index d9573a4..0ca353b 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesComponentFactory.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesComponentFactory.java
@@ -32,7 +32,7 @@
@Override
public ILSMComponent createLSMComponentInstance(LSMComponentFileReferences cfr) throws IndexException {
- return new LSMRTreeComponent(rtreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()), null);
+ return new LSMRTreeImmutableComponent(rtreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()), null);
}
@Override
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
index 3521240..e4d2e82 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
@@ -33,7 +33,7 @@
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -64,21 +64,21 @@
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
ILinearizeComparatorFactory linearizeCmpFactory) throws TreeIndexException {
return createLSMTree(memBufferCache, memFreePageManager, ioManager, file, diskBufferCache, diskFileMapProvider,
- typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
- flushController, mergePolicy, opTrackerFactory, ioScheduler, linearizeCmpFactory, 0);
+ typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, mergePolicy,
+ opTrackerFactory, ioScheduler, ioOpCallbackProvider, linearizeCmpFactory, 0);
}
-
+
public static LSMRTree createLSMTree(IInMemoryBufferCache memBufferCache,
IInMemoryFreePageManager memFreePageManager, IIOManager ioManager, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
ILinearizeComparatorFactory linearizeCmpFactory, int startIODeviceIndex) throws TreeIndexException {
LSMTypeAwareTupleWriterFactory rtreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
LSMTypeAwareTupleWriterFactory btreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
@@ -110,8 +110,8 @@
LSMRTree lsmTree = new LSMRTree(memBufferCache, memFreePageManager, rtreeInteriorFrameFactory,
rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager,
diskRTreeFactory, diskBTreeFactory, diskFileMapProvider, typeTraits.length, rtreeCmpFactories,
- btreeCmpFactories, linearizeCmpFactory, comparatorFields, linearizerArray, flushController,
- mergePolicy, opTrackerFactory, ioScheduler);
+ btreeCmpFactories, linearizeCmpFactory, comparatorFields, linearizerArray, mergePolicy,
+ opTrackerFactory, ioScheduler, ioOpCallbackProvider);
return lsmTree;
}
@@ -120,22 +120,23 @@
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
- ILSMFlushController flushPolicy, ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
- ILSMIOOperationScheduler ioScheduler, ILinearizeComparatorFactory linearizerCmpFactory)
- throws TreeIndexException {
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
+ ILinearizeComparatorFactory linearizerCmpFactory) throws TreeIndexException {
return createLSMTreeWithAntiMatterTuples(memBufferCache, memFreePageManager, ioManager, file, diskBufferCache,
diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories,
- rtreePolicyType, flushPolicy, mergePolicy, opTrackerFactory, ioScheduler, linearizerCmpFactory, 0);
+ rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackProvider,
+ linearizerCmpFactory, 0);
}
-
+
public static LSMRTreeWithAntiMatterTuples createLSMTreeWithAntiMatterTuples(IInMemoryBufferCache memBufferCache,
IInMemoryFreePageManager memFreePageManager, IIOManager ioManager, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
- ILSMFlushController flushPolicy, ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
- ILSMIOOperationScheduler ioScheduler, ILinearizeComparatorFactory linearizerCmpFactory, int startIODeviceIndex)
- throws TreeIndexException {
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
+ ILinearizeComparatorFactory linearizerCmpFactory, int startIODeviceIndex) throws TreeIndexException {
LSMRTreeTupleWriterFactory rtreeTupleWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, false);
LSMRTreeTupleWriterFactory btreeTupleWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, true);
@@ -177,7 +178,7 @@
rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory,
fileNameManager, diskRTreeFactory, bulkLoadRTreeFactory, diskFileMapProvider, typeTraits.length,
rtreeCmpFactories, btreeCmpFactories, linearizerCmpFactory, comparatorFields, linearizerArray,
- flushPolicy, mergePolicy, opTrackerFactory, ioScheduler);
+ mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackProvider);
return lsmTree;
}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java
index 9e40208..59c047c 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java
@@ -22,9 +22,9 @@
public interface IRTreeInteriorFrame extends IRTreeFrame {
- public boolean findBestChild(ITupleReference tuple, MultiComparator cmp);
+ public int findBestChild(ITupleReference tuple, MultiComparator cmp);
- public int getBestChildPageId();
+ public boolean checkIfEnlarementIsNeeded(ITupleReference tuple, MultiComparator cmp);
public int getChildPageId(int tupleIndex);
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreePolicy.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreePolicy.java
index b1d026a..a0cc5e8 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreePolicy.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreePolicy.java
@@ -28,6 +28,6 @@
public void split(ITreeIndexFrame leftFrame, ByteBuffer buf, ITreeIndexFrame rightFrame, ISlotManager slotManager,
ITreeIndexTupleReference frameTuple, ITupleReference tuple, ISplitKey splitKey);
- public boolean findBestChild(ITreeIndexFrame frame, ITupleReference tuple, ITreeIndexTupleReference frameTuple,
+ public int findBestChildPosition(ITreeIndexFrame frame, ITupleReference tuple, ITreeIndexTupleReference frameTuple,
MultiComparator cmp);
}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RStarTreePolicy.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RStarTreePolicy.java
index 3341ace..aafecd5 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RStarTreePolicy.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RStarTreePolicy.java
@@ -64,6 +64,7 @@
}
}
+ @Override
public void split(ITreeIndexFrame leftFrame, ByteBuffer buf, ITreeIndexFrame rightFrame, ISlotManager slotManager,
ITreeIndexTupleReference frameTuple, ITupleReference tuple, ISplitKey splitKey) {
RTreeSplitKey rTreeSplitKey = ((RTreeSplitKey) splitKey);
@@ -252,7 +253,8 @@
}
}
- public boolean findBestChild(ITreeIndexFrame frame, ITupleReference tuple, ITreeIndexTupleReference frameTuple,
+ @Override
+ public int findBestChildPosition(ITreeIndexFrame frame, ITupleReference tuple, ITreeIndexTupleReference frameTuple,
MultiComparator cmp) {
cmpFrameTuple.setFieldCount(cmp.getKeyFieldCount());
frameTuple.setFieldCount(cmp.getKeyFieldCount());
@@ -347,11 +349,6 @@
}
tupleEntries1.clear();
- frameTuple.resetByTupleIndex(frame, bestChild);
- if (minEnlargedArea > 0.0) {
- return true;
- } else {
- return false;
- }
+ return bestChild;
}
}
\ No newline at end of file
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeComputationUtils.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeComputationUtils.java
index 475dc6f..f0122b3 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeComputationUtils.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeComputationUtils.java
@@ -113,4 +113,25 @@
return area;
}
+ public static boolean containsRegion(ITupleReference tuple1, ITupleReference tuple2, MultiComparator cmp,
+ IPrimitiveValueProvider[] keyValueProviders) {
+ int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ int c = cmp.getComparators()[i]
+ .compare(tuple1.getFieldData(i), tuple1.getFieldStart(i), tuple1.getFieldLength(i),
+ tuple2.getFieldData(i), tuple2.getFieldStart(i), tuple2.getFieldLength(i));
+ if (c > 0) {
+ return false;
+ }
+
+ c = cmp.getComparators()[j]
+ .compare(tuple1.getFieldData(j), tuple1.getFieldStart(j), tuple1.getFieldLength(j),
+ tuple2.getFieldData(j), tuple2.getFieldStart(j), tuple2.getFieldLength(j));
+ if (c < 0) {
+ return false;
+ }
+ }
+ return true;
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
index 954a3bb..5ab9632 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
@@ -48,8 +48,16 @@
}
@Override
- public boolean findBestChild(ITupleReference tuple, MultiComparator cmp) {
- return rtreePolicy.findBestChild(this, tuple, frameTuple, cmp);
+ public int findBestChild(ITupleReference tuple, MultiComparator cmp) {
+ int bestChild = rtreePolicy.findBestChildPosition(this, tuple, frameTuple, cmp);
+ frameTuple.resetByTupleIndex(this, bestChild);
+ return buf.getInt(getChildPointerOff(frameTuple));
+ }
+
+ // frameTuple is assumed to have the tuple to be tested against.
+ @Override
+ public boolean checkIfEnlarementIsNeeded(ITupleReference tuple, MultiComparator cmp) {
+ return !RTreeComputationUtils.containsRegion(frameTuple, tuple, cmp, keyValueProviders);
}
@Override
@@ -60,11 +68,6 @@
}
@Override
- public int getBestChildPageId() {
- return buf.getInt(getChildPointerOff(frameTuple));
- }
-
- @Override
public int findTupleByPointer(ITupleReference tuple, MultiComparator cmp) {
frameTuple.setFieldCount(cmp.getKeyFieldCount());
for (int i = 0; i < getTupleCount(); i++) {
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreePolicy.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreePolicy.java
index 1d76b0e..9d94794 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreePolicy.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreePolicy.java
@@ -55,6 +55,7 @@
}
}
+ @Override
public void split(ITreeIndexFrame leftFrame, ByteBuffer buf, ITreeIndexFrame rightFrame, ISlotManager slotManager,
ITreeIndexTupleReference frameTuple, ITupleReference tuple, ISplitKey splitKey) {
RTreeSplitKey rTreeSplitKey = ((RTreeSplitKey) splitKey);
@@ -199,7 +200,8 @@
rTreeSplitKey.getRightTuple().resetByTupleOffset(rTreeSplitKey.getRightPageBuffer(), 0);
}
- public boolean findBestChild(ITreeIndexFrame frame, ITupleReference tuple, ITreeIndexTupleReference frameTuple,
+ @Override
+ public int findBestChildPosition(ITreeIndexFrame frame, ITupleReference tuple, ITreeIndexTupleReference frameTuple,
MultiComparator cmp) {
cmpFrameTuple.setFieldCount(cmp.getKeyFieldCount());
frameTuple.setFieldCount(cmp.getKeyFieldCount());
@@ -224,12 +226,7 @@
}
}
- frameTuple.resetByTupleIndex(frame, bestChild);
- if (minEnlargedArea > 0.0) {
- return true;
- } else {
- return false;
- }
+ return bestChild;
}
}
\ No newline at end of file
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
index e39ab50..773d593 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
@@ -255,9 +255,9 @@
ctx.pathList.add(pageId, pageLsn, -1);
if (!isLeaf) {
- // findBestChild must be called *before* getBestChildPageId
- boolean enlarementIsNeeded = ctx.interiorFrame.findBestChild(ctx.getTuple(), ctx.cmp);
- int childPageId = ctx.interiorFrame.getBestChildPageId();
+ // findBestChild must be called *before* checkIfEnlarementIsNeeded
+ int childPageId = ctx.interiorFrame.findBestChild(ctx.getTuple(), ctx.cmp);
+ boolean enlarementIsNeeded = ctx.interiorFrame.checkIfEnlarementIsNeeded(ctx.getTuple(), ctx.cmp);
if (enlarementIsNeeded) {
if (!writeLatched) {
diff --git a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/ILocalResourceFactory.java b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/ILocalResourceFactory.java
index 7fed1cf..c443b0b 100644
--- a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/ILocalResourceFactory.java
+++ b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/ILocalResourceFactory.java
@@ -15,5 +15,5 @@
package edu.uci.ics.hyracks.storage.common.file;
public interface ILocalResourceFactory {
- public LocalResource createLocalResource(long resourceId, String resourceName);
+ public LocalResource createLocalResource(long resourceId, String resourceName, int partition);
}
diff --git a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/LocalResource.java b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/LocalResource.java
index 3ef3ce7..f4575f3 100644
--- a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/LocalResource.java
+++ b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/LocalResource.java
@@ -20,14 +20,21 @@
private static final long serialVersionUID = 1L;
private final long resourceId;
private final String resourceName;
+ private final int partition;
private final int resourceType;
private final Object object;
- public LocalResource(long resourceId, String resourceName, int resourceType, Object object) {
+ public static final int TransientResource = 0;
+ public static final int LSMBTreeResource = 1;
+ public static final int LSMRTreeResource = 2;
+ public static final int LSMInvertedIndexResource = 3;
+
+ public LocalResource(long resourceId, String resourceName, int partition, int resourceType, Object object) {
this.resourceId = resourceId;
this.resourceName = resourceName;
- this.object = object;
+ this.partition = partition;
this.resourceType = resourceType;
+ this.object = object;
}
public long getResourceId() {
@@ -38,6 +45,10 @@
return resourceName;
}
+ public int getPartition() {
+ return partition;
+ }
+
public int getResourceType() {
return resourceType;
}
diff --git a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/TransientLocalResourceFactory.java b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/TransientLocalResourceFactory.java
index 5d4bbbd..ffd3dff 100644
--- a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/TransientLocalResourceFactory.java
+++ b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/TransientLocalResourceFactory.java
@@ -16,10 +16,8 @@
public class TransientLocalResourceFactory implements ILocalResourceFactory {
- public static final int TransientResource = 0;
-
@Override
- public LocalResource createLocalResource(long resourceId, String resourceName) {
- return new LocalResource(resourceId, resourceName, TransientResource, null);
+ public LocalResource createLocalResource(long resourceId, String resourceName, int partition) {
+ return new LocalResource(resourceId, resourceName, partition, LocalResource.TransientResource, null);
}
}
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java
index fb2fb38..3998108 100644
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java
@@ -78,8 +78,8 @@
// Test parameters.
public static final int LSM_INVINDEX_NUM_DOCS_TO_INSERT = 100;
// Used for full-fledged search test.
- public static final int LSM_INVINDEX_NUM_DOC_QUERIES = 100;
- public static final int LSM_INVINDEX_NUM_RANDOM_QUERIES = 100;
+ public static final int LSM_INVINDEX_NUM_DOC_QUERIES = 1000;
+ public static final int LSM_INVINDEX_NUM_RANDOM_QUERIES = 1000;
// Used for non-search tests to sanity check index searches.
public static final int LSM_INVINDEX_TINY_NUM_DOC_QUERIES = 200;
public static final int LSM_INVINDEX_TINY_NUM_RANDOM_QUERIES = 200;
@@ -90,7 +90,7 @@
// Allocate a generous size to make sure we have enough elements for all tests.
public static final int LSM_INVINDEX_SCAN_COUNT_ARRAY_SIZE = 1000000;
public static final int LSM_INVINDEX_MULTITHREAD_NUM_OPERATIONS = 200;
-
+
}
/* ORIGINAL TEST PARAMETERS: DO NOT EDIT!
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
index d04fa72..4bd1910 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
@@ -53,8 +53,9 @@
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
- harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFlushController(),
- harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler());
+ harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getMergePolicy(),
+ harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
index 0eb3d36..069faad 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
@@ -53,8 +53,9 @@
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
- harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFlushController(),
- harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler());
+ harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getMergePolicy(),
+ harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
index 6250197..644f348 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
@@ -36,8 +36,9 @@
throws TreeIndexException {
return LSMBTreeUtils.createLSMTree(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
- harness.getDiskFileMapProvider(), typeTraits, cmpFactories, harness.getFlushController(),
- harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler());
+ harness.getDiskFileMapProvider(), typeTraits, cmpFactories, harness.getMergePolicy(),
+ harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
}
@Before
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
index 9a80f57..f17e3c8 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
@@ -53,8 +53,9 @@
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
- harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFlushController(),
- harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler());
+ harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getMergePolicy(),
+ harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
index 0061943..24d1f10 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
@@ -44,8 +44,9 @@
harness.setUp();
testCtx = LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
- harness.getDiskFileMapProvider(), fieldSerdes, fieldSerdes.length, harness.getFlushController(),
- harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler());
+ harness.getDiskFileMapProvider(), fieldSerdes, fieldSerdes.length, harness.getMergePolicy(),
+ harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
index = testCtx.getIndex();
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
index 62dda0d..da36c79 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
@@ -52,8 +52,9 @@
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
- harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFlushController(),
- harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler());
+ harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getMergePolicy(),
+ harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTestDriver.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTestDriver.java
index 3add02e..8b02a8e 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTestDriver.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTestDriver.java
@@ -24,7 +24,6 @@
import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexTestUtils;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.config.AccessMethodTestsConfig;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
@@ -65,10 +64,7 @@
}
ILSMIndexAccessor accessor = (ILSMIndexAccessor) ctx.getIndexAccessor();
- ILSMIOOperation ioop = accessor.createMergeOperation(NoOpIOOperationCallback.INSTANCE);
- if (ioop != null) {
- accessor.merge(ioop);
- }
+ accessor.scheduleMerge(NoOpIOOperationCallback.INSTANCE);
orderedIndexTestUtils.checkPointSearches(ctx);
orderedIndexTestUtils.checkScan(ctx);
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
index 6495fa6..978ff86 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
@@ -5,24 +5,23 @@
import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
import edu.uci.ics.hyracks.storage.am.btree.AbstractModificationOperationCallbackTest;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpOperationTrackerFactory;
public class LSMBTreeModificationOperationCallbackTest extends AbstractModificationOperationCallbackTest {
private static final int NUM_TUPLES = 11;
private final LSMBTreeTestHarness harness;
- private final BlockingIOOperationCallback ioOpCallback;
+ private final BlockingIOOperationCallbackWrapper ioOpCallback;
public LSMBTreeModificationOperationCallbackTest() {
super();
- this.ioOpCallback = new BlockingIOOperationCallback();
+ this.ioOpCallback = new BlockingIOOperationCallbackWrapper(NoOpIOOperationCallback.INSTANCE);
harness = new LSMBTreeTestHarness();
}
@@ -31,8 +30,9 @@
index = LSMBTreeUtils.createLSMTree(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), SerdeUtils.serdesToTypeTraits(keySerdes),
- SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), harness.getFlushController(),
- harness.getMergePolicy(), NoOpOperationTrackerFactory.INSTANCE, harness.getIOScheduler());
+ SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), harness.getMergePolicy(),
+ NoOpOperationTrackerFactory.INSTANCE, harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
}
@Override
@@ -49,8 +49,7 @@
@Test
public void modificationCallbackTest() throws Exception {
- IIndexAccessor accessor = index.createAccessor(cb, NoOpOperationCallback.INSTANCE);
- ILSMIOOperation flushOp = ((ILSMIndexAccessor) accessor).createFlushOperation(ioOpCallback);
+ ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(cb, NoOpOperationCallback.INSTANCE);
for (int j = 0; j < 2; j++) {
isFoundNull = true;
@@ -60,7 +59,7 @@
}
if (j == 1) {
- harness.getIOScheduler().scheduleOperation(flushOp);
+ accessor.scheduleFlush(ioOpCallback);
ioOpCallback.waitForIO();
isFoundNull = true;
} else {
@@ -73,7 +72,7 @@
}
if (j == 1) {
- harness.getIOScheduler().scheduleOperation(flushOp);
+ accessor.scheduleFlush(ioOpCallback);
ioOpCallback.waitForIO();
isFoundNull = true;
} else {
@@ -85,7 +84,7 @@
accessor.delete(tuple);
}
- harness.getIOScheduler().scheduleOperation(flushOp);
+ accessor.scheduleFlush(ioOpCallback);
ioOpCallback.waitForIO();
}
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
index 97a5781..3a99c16 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
@@ -54,8 +54,9 @@
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
- harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFlushController(),
- harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler());
+ harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getMergePolicy(),
+ harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
index 72ff348..d0d2c77 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
@@ -35,8 +35,9 @@
index = LSMBTreeUtils.createLSMTree(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), SerdeUtils.serdesToTypeTraits(keySerdes),
- SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), harness.getFlushController(),
- harness.getMergePolicy(), NoOpOperationTrackerFactory.INSTANCE, harness.getIOScheduler());
+ SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), harness.getMergePolicy(),
+ NoOpOperationTrackerFactory.INSTANCE, harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
index ea80b36..ca89512 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
@@ -53,8 +53,9 @@
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
- harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFlushController(),
- harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler());
+ harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getMergePolicy(),
+ harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
index 164c490..0a0b201 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
@@ -52,8 +52,9 @@
throws TreeIndexException {
return LSMBTreeUtils.createLSMTree(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
- harness.getDiskFileMapProvider(), typeTraits, cmpFactories, harness.getFlushController(),
- harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler());
+ harness.getDiskFileMapProvider(), typeTraits, cmpFactories, harness.getMergePolicy(),
+ harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeTestWorker.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeTestWorker.java
index 2dc3d6f..c008f90 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeTestWorker.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeTestWorker.java
@@ -33,12 +33,9 @@
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.lsm.btree.impls.LSMBTree;
import edu.uci.ics.hyracks.storage.am.lsm.btree.impls.LSMBTree.LSMBTreeAccessor;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMMergeInProgressException;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
public class LSMBTreeTestWorker extends AbstractIndexTestWorker {
-
private final LSMBTree lsmBTree;
private final int numKeyFields;
private final ArrayTupleBuilder deleteTb;
@@ -108,19 +105,7 @@
break;
case MERGE:
- try {
- ILSMIOOperation ioop = accessor.createMergeOperation(NoOpIOOperationCallback.INSTANCE);
- if (ioop != null) {
- accessor.merge(ioop);
- }
- } catch (LSMMergeInProgressException e) {
- // Ignore ongoing merges. Do an insert instead.
- try {
- accessor.insert(tuple);
- } catch (BTreeDuplicateKeyException e1) {
- // Ignore duplicate keys, since we get random tuples.
- }
- }
+ accessor.scheduleMerge(NoOpIOOperationCallback.INSTANCE);
break;
default:
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
index e9e0efb..62186c2 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
@@ -39,10 +39,10 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.FlushController;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ImmediateScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoMergePolicy;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.RefCountingOperationTrackerFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.SynchronousScheduler;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ThreadCountingOperationTrackerFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@@ -93,10 +93,10 @@
inMemNumPages, new TransientFileMapManager());
IInMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(inMemNumPages,
new LIFOMetaDataFrameFactory());
- this.ioScheduler = ImmediateScheduler.INSTANCE;
+ this.ioScheduler = SynchronousScheduler.INSTANCE;
lsmtree = LSMBTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ioManager, file, bufferCache, fmp,
- typeTraits, cmpFactories, new FlushController(), NoMergePolicy.INSTANCE,
- RefCountingOperationTrackerFactory.INSTANCE, ioScheduler);
+ typeTraits, cmpFactories, NoMergePolicy.INSTANCE, ThreadCountingOperationTrackerFactory.INSTANCE,
+ ioScheduler, NoOpIOOperationCallback.INSTANCE);
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
index 72d6c8e..e15845f 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
@@ -29,7 +29,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.lsm.btree.impls.LSMBTree;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -66,13 +66,14 @@
public static LSMBTreeTestContext create(IInMemoryBufferCache memBufferCache,
IInMemoryFreePageManager memFreePageManager, IOManager ioManager, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ISerializerDeserializer[] fieldSerdes,
- int numKeyFields, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) throws Exception {
+ int numKeyFields, ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider)
+ throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
LSMBTree lsmTree = LSMBTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ioManager, file,
- diskBufferCache, diskFileMapProvider, typeTraits, cmpFactories, flushController, mergePolicy,
- opTrackerFactory, ioScheduler);
+ diskBufferCache, diskFileMapProvider, typeTraits, cmpFactories, mergePolicy, opTrackerFactory,
+ ioScheduler, ioOpCallbackProvider);
LSMBTreeTestContext testCtx = new LSMBTreeTestContext(fieldSerdes, lsmTree);
return testCtx;
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
index 5e0f716..9128607 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
@@ -33,16 +33,16 @@
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.config.AccessMethodTestsConfig;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.FlushController;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ImmediateScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoMergePolicy;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.RefCountingOperationTrackerFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.SynchronousScheduler;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ThreadCountingOperationTrackerFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@@ -71,9 +71,9 @@
protected IInMemoryFreePageManager memFreePageManager;
protected IHyracksTaskContext ctx;
protected ILSMIOOperationScheduler ioScheduler;
- protected ILSMFlushController flushController;
protected ILSMMergePolicy mergePolicy;
protected ILSMOperationTrackerFactory opTrackerFactory;
+ protected ILSMIOOperationCallbackProvider ioOpCallbackProvider;
protected final Random rnd = new Random();
protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
@@ -88,10 +88,10 @@
this.memPageSize = AccessMethodTestsConfig.LSM_BTREE_MEM_PAGE_SIZE;
this.memNumPages = AccessMethodTestsConfig.LSM_BTREE_MEM_NUM_PAGES;
this.hyracksFrameSize = AccessMethodTestsConfig.LSM_BTREE_HYRACKS_FRAME_SIZE;
- this.ioScheduler = ImmediateScheduler.INSTANCE;
+ this.ioScheduler = SynchronousScheduler.INSTANCE;
this.mergePolicy = NoMergePolicy.INSTANCE;
- this.flushController = new FlushController();
- this.opTrackerFactory = RefCountingOperationTrackerFactory.INSTANCE;
+ this.opTrackerFactory = ThreadCountingOperationTrackerFactory.INSTANCE;
+ this.ioOpCallbackProvider = NoOpIOOperationCallback.INSTANCE;
}
public LSMBTreeTestHarness(int diskPageSize, int diskNumPages, int diskMaxOpenFiles, int memPageSize,
@@ -102,10 +102,9 @@
this.memPageSize = memPageSize;
this.memNumPages = memNumPages;
this.hyracksFrameSize = hyracksFrameSize;
- this.ioScheduler = ImmediateScheduler.INSTANCE;
+ this.ioScheduler = SynchronousScheduler.INSTANCE;
this.mergePolicy = NoMergePolicy.INSTANCE;
- this.flushController = new FlushController();
- this.opTrackerFactory = RefCountingOperationTrackerFactory.INSTANCE;
+ this.opTrackerFactory = ThreadCountingOperationTrackerFactory.INSTANCE;
}
public void setUp() throws HyracksException {
@@ -206,11 +205,11 @@
return opTrackerFactory;
}
- public ILSMFlushController getFlushController() {
- return flushController;
- }
-
public ILSMMergePolicy getMergePolicy() {
return mergePolicy;
}
+
+ public ILSMIOOperationCallbackProvider getIOOperationCallbackProvider() {
+ return ioOpCallbackProvider;
+ }
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexMergeTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexMergeTest.java
index 0a65834..811919b 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexMergeTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/LSMInvertedIndexMergeTest.java
@@ -22,7 +22,6 @@
import edu.uci.ics.hyracks.storage.am.common.datagen.TupleGenerator;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.config.AccessMethodTestsConfig;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.common.AbstractInvertedIndexLoadTest;
@@ -33,7 +32,7 @@
public class LSMInvertedIndexMergeTest extends AbstractInvertedIndexLoadTest {
private final int maxTreesToMerge = AccessMethodTestsConfig.LSM_INVINDEX_MAX_TREES_TO_MERGE;
-
+
public LSMInvertedIndexMergeTest() {
super(InvertedIndexType.LSM, true, 1);
}
@@ -41,10 +40,11 @@
@Override
protected void runTest(LSMInvertedIndexTestContext testCtx, TupleGenerator tupleGen) throws IOException,
IndexException {
- IIndex invIndex = testCtx.getIndex();
+ IIndex invIndex = testCtx.getIndex();
invIndex.create();
invIndex.activate();
- ILSMIndexAccessor invIndexAccessor = (ILSMIndexAccessor) invIndex.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ ILSMIndexAccessor invIndexAccessor = (ILSMIndexAccessor) invIndex.createAccessor(
+ NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
for (int i = 0; i < maxTreesToMerge; i++) {
for (int j = 0; j < i; j++) {
@@ -55,14 +55,11 @@
}
}
// Perform merge.
- ILSMIOOperation ioop = invIndexAccessor.createMergeOperation(NoOpIOOperationCallback.INSTANCE);
- if (ioop != null) {
- invIndexAccessor.merge(ioop);
- }
+ invIndexAccessor.scheduleMerge(NoOpIOOperationCallback.INSTANCE);
validateAndCheckIndex(testCtx);
runTinySearchWorkload(testCtx, tupleGen);
}
-
+
invIndex.deactivate();
invIndex.destroy();
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/PartitionedLSMInvertedIndexMergeTest.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/PartitionedLSMInvertedIndexMergeTest.java
index 52bb3f8..786afe1 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/PartitionedLSMInvertedIndexMergeTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/PartitionedLSMInvertedIndexMergeTest.java
@@ -22,7 +22,6 @@
import edu.uci.ics.hyracks.storage.am.common.datagen.TupleGenerator;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.config.AccessMethodTestsConfig;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.common.AbstractInvertedIndexLoadTest;
@@ -33,7 +32,7 @@
public class PartitionedLSMInvertedIndexMergeTest extends AbstractInvertedIndexLoadTest {
private final int maxTreesToMerge = AccessMethodTestsConfig.LSM_INVINDEX_MAX_TREES_TO_MERGE;
-
+
public PartitionedLSMInvertedIndexMergeTest() {
super(InvertedIndexType.PARTITIONED_LSM, true, 1);
}
@@ -41,10 +40,11 @@
@Override
protected void runTest(LSMInvertedIndexTestContext testCtx, TupleGenerator tupleGen) throws IOException,
IndexException {
- IIndex invIndex = testCtx.getIndex();
+ IIndex invIndex = testCtx.getIndex();
invIndex.create();
invIndex.activate();
- ILSMIndexAccessor invIndexAccessor = (ILSMIndexAccessor) invIndex.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ ILSMIndexAccessor invIndexAccessor = (ILSMIndexAccessor) invIndex.createAccessor(
+ NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
for (int i = 0; i < maxTreesToMerge; i++) {
for (int j = 0; j < i; j++) {
@@ -55,14 +55,11 @@
}
}
// Perform merge.
- ILSMIOOperation ioop = invIndexAccessor.createMergeOperation(NoOpIOOperationCallback.INSTANCE);
- if (ioop != null) {
- invIndexAccessor.merge(ioop);
- }
+ invIndexAccessor.scheduleMerge(NoOpIOOperationCallback.INSTANCE);
validateAndCheckIndex(testCtx);
runTinySearchWorkload(testCtx, tupleGen);
}
-
+
invIndex.deactivate();
invIndex.destroy();
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
index 71fe80f..5be1d6a 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
@@ -31,16 +31,16 @@
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.config.AccessMethodTestsConfig;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.DualIndexInMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.DualIndexInMemoryFreePageManager;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.FlushController;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ImmediateScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoMergePolicy;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.RefCountingOperationTrackerFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.SynchronousScheduler;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ThreadCountingOperationTrackerFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@@ -65,9 +65,9 @@
protected IInMemoryFreePageManager memFreePageManager;
protected IHyracksTaskContext ctx;
protected ILSMIOOperationScheduler ioScheduler;
- protected ILSMFlushController flushController;
protected ILSMMergePolicy mergePolicy;
protected ILSMOperationTrackerFactory opTrackerFactory;
+ protected ILSMIOOperationCallbackProvider ioOpCallbackProvider;
protected final Random rnd = new Random();
protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
@@ -84,10 +84,10 @@
this.memPageSize = AccessMethodTestsConfig.LSM_INVINDEX_MEM_PAGE_SIZE;
this.memNumPages = AccessMethodTestsConfig.LSM_INVINDEX_MEM_NUM_PAGES;
this.hyracksFrameSize = AccessMethodTestsConfig.LSM_INVINDEX_HYRACKS_FRAME_SIZE;
- this.ioScheduler = ImmediateScheduler.INSTANCE;
+ this.ioScheduler = SynchronousScheduler.INSTANCE;
this.mergePolicy = NoMergePolicy.INSTANCE;
- this.flushController = new FlushController();
- this.opTrackerFactory = RefCountingOperationTrackerFactory.INSTANCE;
+ this.opTrackerFactory = ThreadCountingOperationTrackerFactory.INSTANCE;
+ this.ioOpCallbackProvider = NoOpIOOperationCallback.INSTANCE;
}
public LSMInvertedIndexTestHarness(int diskPageSize, int diskNumPages, int diskMaxOpenFiles, int memPageSize,
@@ -98,10 +98,9 @@
this.memPageSize = memPageSize;
this.memNumPages = memNumPages;
this.hyracksFrameSize = hyracksFrameSize;
- this.ioScheduler = ImmediateScheduler.INSTANCE;
+ this.ioScheduler = SynchronousScheduler.INSTANCE;
this.mergePolicy = NoMergePolicy.INSTANCE;
- this.flushController = new FlushController();
- this.opTrackerFactory = RefCountingOperationTrackerFactory.INSTANCE;
+ this.opTrackerFactory = ThreadCountingOperationTrackerFactory.INSTANCE;
}
public void setUp() throws HyracksException {
@@ -207,11 +206,11 @@
return opTrackerFactory;
}
- public ILSMFlushController getFlushController() {
- return flushController;
- }
-
public ILSMMergePolicy getMergePolicy() {
return mergePolicy;
}
+
+ public ILSMIOOperationCallbackProvider getIOOperationCallbackProvider() {
+ return ioOpCallbackProvider;
+ }
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexTestWorker.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexTestWorker.java
index 0269c58..d6bb3d3 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexTestWorker.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexTestWorker.java
@@ -30,8 +30,6 @@
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMMergeInProgressException;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifier;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.exceptions.OccurrenceThresholdPanicException;
@@ -45,12 +43,12 @@
public class LSMInvertedIndexTestWorker extends AbstractIndexTestWorker {
protected final LSMInvertedIndex invIndex;
- protected final List<ITupleReference> documentCorpus = new ArrayList<ITupleReference>();
+ protected final List<ITupleReference> documentCorpus = new ArrayList<ITupleReference>();
protected final Random rnd = new Random(50);
-
+
protected final IInvertedIndexSearchModifier[] TEST_SEARCH_MODIFIERS = new IInvertedIndexSearchModifier[] {
new ConjunctiveSearchModifier(), new JaccardSearchModifier(0.8f), new JaccardSearchModifier(0.5f) };
-
+
public LSMInvertedIndexTestWorker(DataGenThread dataGen, TestOperationSelector opSelector, IIndex index,
int numBatches) {
super(dataGen, opSelector, index, numBatches);
@@ -67,7 +65,7 @@
int searchModifierIndex = Math.abs(rnd.nextInt()) % TEST_SEARCH_MODIFIERS.length;
InvertedIndexSearchPredicate searchPred = new InvertedIndexSearchPredicate(tokenizerFactory.createTokenizer(),
TEST_SEARCH_MODIFIERS[searchModifierIndex]);
-
+
switch (op) {
case INSERT: {
insert(accessor, tuple);
@@ -99,7 +97,7 @@
consumeCursorTuples(searchCursor);
} catch (OccurrenceThresholdPanicException e) {
// Ignore.
- }
+ }
break;
}
@@ -111,14 +109,7 @@
}
case MERGE: {
- try {
- ILSMIOOperation ioop = accessor.createMergeOperation(NoOpIOOperationCallback.INSTANCE);
- if (ioop != null) {
- accessor.merge(ioop);
- }
- } catch (LSMMergeInProgressException e) {
- insert(accessor, tuple);
- }
+ accessor.scheduleMerge(NoOpIOOperationCallback.INSTANCE);
break;
}
@@ -126,8 +117,9 @@
throw new HyracksDataException("Op " + op.toString() + " not supported.");
}
}
-
- private void insert(LSMInvertedIndexAccessor accessor, ITupleReference tuple) throws HyracksDataException, IndexException {
+
+ private void insert(LSMInvertedIndexAccessor accessor, ITupleReference tuple) throws HyracksDataException,
+ IndexException {
// Ignore ongoing merges. Do an insert instead.
accessor.insert(tuple);
// Add tuple to document corpus so we can delete it.
diff --git a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
index 22c010d..870e6d9 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
@@ -58,7 +58,7 @@
protected InvertedIndexTokenizingTupleIterator indexTupleIter;
protected HashSet<Comparable> allTokens = new HashSet<Comparable>();
protected List<ITupleReference> documentCorpus = new ArrayList<ITupleReference>();
-
+
public LSMInvertedIndexTestContext(ISerializerDeserializer[] fieldSerdes, IIndex index,
IBinaryTokenizerFactory tokenizerFactory, InvertedIndexType invIndexType,
InvertedIndexTokenizingTupleIterator indexTupleIter) {
@@ -146,8 +146,8 @@
harness.getMemFreePageManager(), harness.getDiskFileMapProvider(), invListTypeTraits,
invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
harness.getDiskBufferCache(), harness.getIOManager(), harness.getOnDiskDir(),
- harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTrackerFactory(),
- harness.getIOScheduler());
+ harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
break;
}
case PARTITIONED_LSM: {
@@ -155,8 +155,8 @@
harness.getMemFreePageManager(), harness.getDiskFileMapProvider(), invListTypeTraits,
invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
harness.getDiskBufferCache(), harness.getIOManager(), harness.getOnDiskDir(),
- harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTrackerFactory(),
- harness.getIOScheduler());
+ harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
break;
}
default: {
@@ -201,7 +201,7 @@
allTokens.add(checkTuple.getField(0));
}
}
-
+
public void deleteCheckTuples(ITupleReference tuple, Collection<CheckTuple> checkTuples)
throws HyracksDataException {
indexTupleIter.reset(tuple);
@@ -212,36 +212,37 @@
deleteCheckTuple(checkTuple, checkTuples);
}
}
-
+
public HashSet<Comparable> getAllTokens() {
return allTokens;
}
-
+
@SuppressWarnings("unchecked")
public CheckTuple createCheckTuple(ITupleReference tuple) throws HyracksDataException {
CheckTuple checkTuple = new CheckTuple(fieldSerdes.length, fieldSerdes.length);
for (int i = 0; i < fieldSerdes.length; i++) {
- ByteArrayInputStream bains = new ByteArrayInputStream(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
+ ByteArrayInputStream bains = new ByteArrayInputStream(tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i));
DataInput in = new DataInputStream(bains);
Comparable field = (Comparable) fieldSerdes[i].deserialize(in);
checkTuple.appendField(field);
}
return checkTuple;
}
-
+
@Override
public void upsertCheckTuple(CheckTuple checkTuple, Collection<CheckTuple> checkTuples) {
throw new UnsupportedOperationException("Upsert not supported by inverted index.");
}
-
+
public IBinaryTokenizerFactory getTokenizerFactory() {
return tokenizerFactory;
}
-
+
public List<ITupleReference> getDocumentCorpus() {
return documentCorpus;
}
-
+
public InvertedIndexType getInvertedIndexType() {
return invIndexType;
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
index bf7a5ef..995f18c 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
@@ -57,8 +57,8 @@
return LSMRTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType,
- harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTrackerFactory(),
- harness.getIOScheduler());
+ harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
index 3dc387e..d72b668 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
@@ -34,12 +34,12 @@
@SuppressWarnings("rawtypes")
public class LSMRTreeDeleteTest extends AbstractRTreeDeleteTest {
- private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
+ private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
- public LSMRTreeDeleteTest() {
- super(AccessMethodTestsConfig.LSM_RTREE_TEST_RSTAR_POLICY);
- }
-
+ public LSMRTreeDeleteTest() {
+ super(AccessMethodTestsConfig.LSM_RTREE_TEST_RSTAR_POLICY);
+ }
+
@Before
public void setUp() throws HyracksException {
harness.setUp();
@@ -57,8 +57,8 @@
return LSMRTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType,
- harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTrackerFactory(),
- harness.getIOScheduler());
+ harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
index f07dab0..cba8cee 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
@@ -40,8 +40,9 @@
return LSMRTreeUtils.createLSMTree(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, rtreePolicyType, harness.getFlushController(), harness.getMergePolicy(),
+ valueProviderFactories, rtreePolicyType, harness.getMergePolicy(),
harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider(),
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length));
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
index 6efbc72..96485f8 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
@@ -34,12 +34,12 @@
@SuppressWarnings("rawtypes")
public class LSMRTreeInsertTest extends AbstractRTreeInsertTest {
- private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
+ private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
- public LSMRTreeInsertTest() {
- super(AccessMethodTestsConfig.LSM_RTREE_TEST_RSTAR_POLICY);
- }
-
+ public LSMRTreeInsertTest() {
+ super(AccessMethodTestsConfig.LSM_RTREE_TEST_RSTAR_POLICY);
+ }
+
@Before
public void setUp() throws HyracksException {
harness.setUp();
@@ -57,8 +57,8 @@
return LSMRTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType,
- harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTrackerFactory(),
- harness.getIOScheduler());
+ harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java
index fe10850..e72b3ca 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java
@@ -55,8 +55,8 @@
testCtx = LSMRTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories, numKeys, RTreePolicyType.RTREE,
- harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTrackerFactory(),
- harness.getIOScheduler());
+ harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
index = testCtx.getIndex();
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
index 3c492d8..1d07484 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
@@ -33,12 +33,12 @@
@SuppressWarnings("rawtypes")
public class LSMRTreeMergeTest extends LSMRTreeMergeTestDriver {
- private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
+ private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
- public LSMRTreeMergeTest() {
- super(AccessMethodTestsConfig.LSM_RTREE_TEST_RSTAR_POLICY);
- }
-
+ public LSMRTreeMergeTest() {
+ super(AccessMethodTestsConfig.LSM_RTREE_TEST_RSTAR_POLICY);
+ }
+
@Before
public void setUp() throws HyracksException {
harness.setUp();
@@ -56,8 +56,8 @@
return LSMRTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType,
- harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTrackerFactory(),
- harness.getIOScheduler());
+ harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTestDriver.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTestDriver.java
index 36c3c5c..f272942 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTestDriver.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTestDriver.java
@@ -21,7 +21,6 @@
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.config.AccessMethodTestsConfig;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeTestContext;
@@ -67,10 +66,7 @@
}
ILSMIndexAccessor accessor = (ILSMIndexAccessor) ctx.getIndexAccessor();
- ILSMIOOperation mergeOperation = accessor.createMergeOperation(NoOpIOOperationCallback.INSTANCE);
- if (mergeOperation != null) {
- accessor.merge(mergeOperation);
- }
+ accessor.scheduleMerge(NoOpIOOperationCallback.INSTANCE);
rTreeTestUtils.checkScan(ctx);
rTreeTestUtils.checkDiskOrderScan(ctx);
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMultiBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMultiBulkLoadTest.java
index 16a1f3e..cc46065 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMultiBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMultiBulkLoadTest.java
@@ -33,8 +33,8 @@
@SuppressWarnings("rawtypes")
public class LSMRTreeMultiBulkLoadTest extends AbstractRTreeBulkLoadTest {
-
- public LSMRTreeMultiBulkLoadTest() {
+
+ public LSMRTreeMultiBulkLoadTest() {
super(AccessMethodTestsConfig.LSM_RTREE_BULKLOAD_ROUNDS, AccessMethodTestsConfig.LSM_RTREE_TEST_RSTAR_POLICY);
}
@@ -57,8 +57,8 @@
return LSMRTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType,
- harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTrackerFactory(),
- harness.getIOScheduler());
+ harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
index dcc6478..81a952d 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
@@ -57,8 +57,8 @@
return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getMemBufferCache(),
harness.getMemFreePageManager(), harness.getIOManager(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
- numKeys, rtreePolicyType, harness.getFlushController(), harness.getMergePolicy(),
- harness.getOperationTrackerFactory(), harness.getIOScheduler());
+ numKeys, rtreePolicyType, harness.getMergePolicy(), harness.getOperationTrackerFactory(),
+ harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
index ce90ff6..1ee92d9 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
@@ -34,12 +34,12 @@
@SuppressWarnings("rawtypes")
public class LSMRTreeWithAntiMatterTuplesDeleteTest extends AbstractRTreeDeleteTest {
- private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
+ private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
- public LSMRTreeWithAntiMatterTuplesDeleteTest() {
- super(AccessMethodTestsConfig.LSM_RTREE_TEST_RSTAR_POLICY);
- }
-
+ public LSMRTreeWithAntiMatterTuplesDeleteTest() {
+ super(AccessMethodTestsConfig.LSM_RTREE_TEST_RSTAR_POLICY);
+ }
+
@Before
public void setUp() throws HyracksException {
harness.setUp();
@@ -57,8 +57,8 @@
return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getMemBufferCache(),
harness.getMemFreePageManager(), harness.getIOManager(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
- numKeys, rtreePolicyType, harness.getFlushController(), harness.getMergePolicy(),
- harness.getOperationTrackerFactory(), harness.getIOScheduler());
+ numKeys, rtreePolicyType, harness.getMergePolicy(), harness.getOperationTrackerFactory(),
+ harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
index acab6fe..3a2537c 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
@@ -40,8 +40,9 @@
return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getMemBufferCache(),
harness.getMemFreePageManager(), harness.getIOManager(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, rtreeCmpFactories,
- btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getFlushController(),
- harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getMergePolicy(),
+ harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider(),
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length));
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
index 8188595..61d5ce7 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
@@ -34,12 +34,12 @@
@SuppressWarnings("rawtypes")
public class LSMRTreeWithAntiMatterTuplesInsertTest extends AbstractRTreeInsertTest {
- private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
+ private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
- public LSMRTreeWithAntiMatterTuplesInsertTest() {
- super(AccessMethodTestsConfig.LSM_RTREE_TEST_RSTAR_POLICY);
- }
-
+ public LSMRTreeWithAntiMatterTuplesInsertTest() {
+ super(AccessMethodTestsConfig.LSM_RTREE_TEST_RSTAR_POLICY);
+ }
+
@Before
public void setUp() throws HyracksException {
harness.setUp();
@@ -57,8 +57,8 @@
return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getMemBufferCache(),
harness.getMemFreePageManager(), harness.getIOManager(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
- numKeys, rtreePolicyType, harness.getFlushController(), harness.getMergePolicy(),
- harness.getOperationTrackerFactory(), harness.getIOScheduler());
+ numKeys, rtreePolicyType, harness.getMergePolicy(), harness.getOperationTrackerFactory(),
+ harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java
index 032f443..aee8670 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java
@@ -55,8 +55,8 @@
testCtx = LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getMemBufferCache(),
harness.getMemFreePageManager(), harness.getIOManager(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
- numKeys, RTreePolicyType.RTREE, harness.getFlushController(), harness.getMergePolicy(),
- harness.getOperationTrackerFactory(), harness.getIOScheduler());
+ numKeys, RTreePolicyType.RTREE, harness.getMergePolicy(), harness.getOperationTrackerFactory(),
+ harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
index = testCtx.getIndex();
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
index fa7ceb5..d5fecbf 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
@@ -33,12 +33,12 @@
@SuppressWarnings("rawtypes")
public class LSMRTreeWithAntiMatterTuplesMergeTest extends LSMRTreeMergeTestDriver {
- private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
+ private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
- public LSMRTreeWithAntiMatterTuplesMergeTest() {
- super(AccessMethodTestsConfig.LSM_RTREE_TEST_RSTAR_POLICY);
- }
-
+ public LSMRTreeWithAntiMatterTuplesMergeTest() {
+ super(AccessMethodTestsConfig.LSM_RTREE_TEST_RSTAR_POLICY);
+ }
+
@Before
public void setUp() throws HyracksException {
harness.setUp();
@@ -56,8 +56,8 @@
return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getMemBufferCache(),
harness.getMemFreePageManager(), harness.getIOManager(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
- numKeys, rtreePolicyType, harness.getFlushController(), harness.getMergePolicy(),
- harness.getOperationTrackerFactory(), harness.getIOScheduler());
+ numKeys, rtreePolicyType, harness.getMergePolicy(), harness.getOperationTrackerFactory(),
+ harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMultiBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMultiBulkLoadTest.java
index 237fc1d..de5f065 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMultiBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMultiBulkLoadTest.java
@@ -33,8 +33,8 @@
@SuppressWarnings("rawtypes")
public class LSMRTreeWithAntiMatterTuplesMultiBulkLoadTest extends AbstractRTreeBulkLoadTest {
-
- public LSMRTreeWithAntiMatterTuplesMultiBulkLoadTest() {
+
+ public LSMRTreeWithAntiMatterTuplesMultiBulkLoadTest() {
super(AccessMethodTestsConfig.LSM_RTREE_BULKLOAD_ROUNDS, AccessMethodTestsConfig.LSM_RTREE_TEST_RSTAR_POLICY);
}
@@ -57,8 +57,8 @@
return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getMemBufferCache(),
harness.getMemFreePageManager(), harness.getIOManager(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
- numKeys, rtreePolicyType, harness.getFlushController(), harness.getMergePolicy(),
- harness.getOperationTrackerFactory(), harness.getIOScheduler());
+ numKeys, rtreePolicyType, harness.getMergePolicy(), harness.getOperationTrackerFactory(),
+ harness.getIOScheduler(), harness.getIOOperationCallbackProvider());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
index cde9461..37be58d 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
@@ -60,8 +60,9 @@
return LSMRTreeUtils.createLSMTree(harness.getMemBufferCache(), harness.getMemFreePageManager(),
harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, rtreePolicyType, harness.getFlushController(), harness.getMergePolicy(),
+ valueProviderFactories, rtreePolicyType, harness.getMergePolicy(),
harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider(),
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length));
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeTestWorker.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeTestWorker.java
index 12e3ce8..154756e 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeTestWorker.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeTestWorker.java
@@ -27,8 +27,6 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMMergeInProgressException;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTree;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTree.LSMRTreeAccessor;
@@ -74,16 +72,7 @@
break;
case MERGE:
- try {
- ILSMIOOperation mergeOperation = accessor.createMergeOperation(NoOpIOOperationCallback.INSTANCE);
- if (mergeOperation != null) {
- accessor.merge(mergeOperation);
- }
- } catch (LSMMergeInProgressException e) {
- // Ignore ongoing merges. Do an insert instead.
- rearrangeTuple(tuple, cmp);
- accessor.insert(rearrangedTuple);
- }
+ accessor.scheduleMerge(NoOpIOOperationCallback.INSTANCE);
break;
default:
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
index ea900d7..af73676 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
@@ -60,8 +60,9 @@
return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getMemBufferCache(),
harness.getMemFreePageManager(), harness.getIOManager(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, rtreeCmpFactories,
- btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getFlushController(),
- harness.getMergePolicy(), harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getMergePolicy(),
+ harness.getOperationTrackerFactory(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackProvider(),
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length));
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesTestWorker.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesTestWorker.java
index 5f88020..661138d 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesTestWorker.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesTestWorker.java
@@ -24,8 +24,6 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMMergeInProgressException;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTreeWithAntiMatterTuples.LSMRTreeWithAntiMatterTuplesAccessor;
import edu.uci.ics.hyracks.storage.am.rtree.impls.SearchPredicate;
@@ -63,16 +61,7 @@
break;
case MERGE:
- try {
- ILSMIOOperation mergeOperation = accessor.createMergeOperation(NoOpIOOperationCallback.INSTANCE);
- if (mergeOperation != null) {
- accessor.merge(mergeOperation);
- }
- } catch (LSMMergeInProgressException e) {
- // Ignore ongoing merges. Do an insert instead.
- rearrangeTuple(tuple, cmp);
- accessor.insert(rearrangedTuple);
- }
+ accessor.scheduleMerge(NoOpIOOperationCallback.INSTANCE);
break;
default:
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
index c68f5e3..27fb9c8 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
@@ -27,7 +27,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -71,8 +71,9 @@
IInMemoryFreePageManager memFreePageManager, IOManager ioManager, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeyFields, RTreePolicyType rtreePolicyType,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) throws Exception {
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider)
+ throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] rtreeCmpFactories = SerdeUtils
.serdesToComparatorFactories(fieldSerdes, numKeyFields);
@@ -80,8 +81,8 @@
fieldSerdes.length);
LSMRTree lsmTree = LSMRTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ioManager, file,
diskBufferCache, diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, rtreePolicyType, flushController, mergePolicy, opTrackerFactory, ioScheduler,
- LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length));
+ valueProviderFactories, rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler,
+ ioOpCallbackProvider, LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length));
LSMRTreeTestContext testCtx = new LSMRTreeTestContext(fieldSerdes, lsmTree);
return testCtx;
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
index aed0d60..6fb6f9c 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
@@ -32,16 +32,16 @@
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.config.AccessMethodTestsConfig;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.DualIndexInMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.DualIndexInMemoryFreePageManager;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.FlushController;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ImmediateScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoMergePolicy;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.RefCountingOperationTrackerFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.SynchronousScheduler;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ThreadCountingOperationTrackerFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@@ -67,7 +67,7 @@
protected IInMemoryFreePageManager memFreePageManager;
protected IHyracksTaskContext ctx;
protected ILSMIOOperationScheduler ioScheduler;
- protected ILSMFlushController flushController;
+ protected ILSMIOOperationCallbackProvider ioOpCallbackProvider;
protected ILSMMergePolicy mergePolicy;
protected ILSMOperationTrackerFactory opTrackerFactory;
@@ -84,10 +84,10 @@
this.memPageSize = AccessMethodTestsConfig.LSM_RTREE_MEM_PAGE_SIZE;
this.memNumPages = AccessMethodTestsConfig.LSM_RTREE_MEM_NUM_PAGES;
this.hyracksFrameSize = AccessMethodTestsConfig.LSM_RTREE_HYRACKS_FRAME_SIZE;
- this.ioScheduler = ImmediateScheduler.INSTANCE;
+ this.ioScheduler = SynchronousScheduler.INSTANCE;
this.mergePolicy = NoMergePolicy.INSTANCE;
- this.flushController = new FlushController();
- this.opTrackerFactory = RefCountingOperationTrackerFactory.INSTANCE;
+ this.opTrackerFactory = ThreadCountingOperationTrackerFactory.INSTANCE;
+ this.ioOpCallbackProvider = NoOpIOOperationCallback.INSTANCE;
}
public LSMRTreeTestHarness(int diskPageSize, int diskNumPages, int diskMaxOpenFiles, int memPageSize,
@@ -98,10 +98,9 @@
this.memPageSize = memPageSize;
this.memNumPages = memNumPages;
this.hyracksFrameSize = hyracksFrameSize;
- this.ioScheduler = ImmediateScheduler.INSTANCE;
+ this.ioScheduler = SynchronousScheduler.INSTANCE;
this.mergePolicy = NoMergePolicy.INSTANCE;
- this.flushController = new FlushController();
- this.opTrackerFactory = RefCountingOperationTrackerFactory.INSTANCE;
+ this.opTrackerFactory = ThreadCountingOperationTrackerFactory.INSTANCE;
}
public void setUp() throws HyracksException {
@@ -205,11 +204,11 @@
return opTrackerFactory;
}
- public ILSMFlushController getFlushController() {
- return flushController;
- }
-
public ILSMMergePolicy getMergePolicy() {
return mergePolicy;
}
+
+ public ILSMIOOperationCallbackProvider getIOOperationCallbackProvider() {
+ return ioOpCallbackProvider;
+ }
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
index ebd8f72..5860236 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
@@ -27,7 +27,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMFlushController;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
@@ -71,8 +71,9 @@
IInMemoryFreePageManager memFreePageManager, IOManager ioManager, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeyFields, RTreePolicyType rtreePolicyType,
- ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationScheduler ioScheduler) throws Exception {
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerFactory opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackProvider ioOpCallbackProvider)
+ throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] rtreeCmpFactories = SerdeUtils
.serdesToComparatorFactories(fieldSerdes, numKeyFields);
@@ -80,8 +81,8 @@
fieldSerdes.length);
LSMRTreeWithAntiMatterTuples lsmTree = LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(memBufferCache,
memFreePageManager, ioManager, file, diskBufferCache, diskFileMapProvider, typeTraits,
- rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, flushController,
- mergePolicy, opTrackerFactory, ioScheduler,
+ rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, mergePolicy,
+ opTrackerFactory, ioScheduler, ioOpCallbackProvider,
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length));
LSMRTreeWithAntiMatterTuplesTestContext testCtx = new LSMRTreeWithAntiMatterTuplesTestContext(fieldSerdes,
lsmTree);
diff --git a/pom.xml b/pom.xml
index ce15c7f..49a5887 100644
--- a/pom.xml
+++ b/pom.xml
@@ -30,7 +30,7 @@
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkMode>pertest</forkMode>
- <argLine>-enableassertions -Djava.util.logging.config.file=${user.home}/logging.properties -Xdebug -Xrunjdwp:transport=dt_socket,server=y,address=8000,suspend=n ${jvm.extraargs}</argLine>
+ <argLine>-enableassertions -Djava.util.logging.config.file=${user.home}/logging.properties -Xdebug -Xrunjdwp:transport=dt_socket,server=y,address=8000,suspend=n ${jvm.extraargs} -Xmx2048m</argLine>
</configuration>
</plugin>
</plugins>