merged hyracks_lsm_tree into branch -r2707:2751
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_lsm_tree_bloom_filter@2752 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/common/TreeOperatorTestHelper.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/common/TreeOperatorTestHelper.java
index 362badb..935724b 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/common/TreeOperatorTestHelper.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/common/TreeOperatorTestHelper.java
@@ -23,6 +23,8 @@
protected final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
protected final String sep = System.getProperty("file.separator");
+ protected static int DEFAULT_MEM_PAGE_SIZE = 32768;
+ protected static int DEFAULT_MEM_NUM_PAGES = 1000;
public String getPrimaryIndexName() {
return System.getProperty("java.io.tmpdir") + sep + "primary" + simpleDateFormat.format(new Date());
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/PartitionedWordInvertedIndexTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/PartitionedWordInvertedIndexTest.java
index 09dcddd..62d4362 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/PartitionedWordInvertedIndexTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/PartitionedWordInvertedIndexTest.java
@@ -53,7 +53,8 @@
invertedIndexDataflowHelperFactory = new PartitionedLSMInvertedIndexDataflowHelperFactory(
new ConstantMergePolicyProvider(MERGE_THRESHOLD), ThreadCountingOperationTrackerFactory.INSTANCE,
- SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE);
+ SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE, DEFAULT_MEM_PAGE_SIZE,
+ DEFAULT_MEM_NUM_PAGES);
}
@Override
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java
index f0aeb4e..c35c6c9 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java
@@ -48,7 +48,7 @@
invertedIndexDataflowHelperFactory = new LSMInvertedIndexDataflowHelperFactory(new ConstantMergePolicyProvider(
MERGE_THRESHOLD), ThreadCountingOperationTrackerFactory.INSTANCE,
- SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE);
+ SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE, DEFAULT_MEM_PAGE_SIZE, DEFAULT_MEM_NUM_PAGES);
}
@Override
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java
index 6554557..fce6e69 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java
@@ -18,6 +18,7 @@
import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.AbstractLSMIndexDataflowHelper;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ConstantMergePolicyProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.SynchronousSchedulerProvider;
@@ -35,7 +36,7 @@
public IIndexDataflowHelperFactory createDataFlowHelperFactory() {
return new LSMBTreeDataflowHelperFactory(new ConstantMergePolicyProvider(MERGE_THRESHOLD),
ThreadCountingOperationTrackerFactory.INSTANCE, SynchronousSchedulerProvider.INSTANCE,
- NoOpIOOperationCallback.INSTANCE);
+ NoOpIOOperationCallback.INSTANCE, DEFAULT_MEM_PAGE_SIZE, DEFAULT_MEM_NUM_PAGES);
}
}
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
index 43b41d7..84b34b7 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
@@ -41,7 +41,7 @@
IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory) {
return new LSMRTreeDataflowHelperFactory(valueProviderFactories, rtreePolicyType, btreeComparatorFactories,
new ConstantMergePolicyProvider(MERGE_THRESHOLD), ThreadCountingOperationTrackerFactory.INSTANCE,
- SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE, linearizerCmpFactory);
+ SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE, linearizerCmpFactory,
+ DEFAULT_MEM_PAGE_SIZE, DEFAULT_MEM_NUM_PAGES);
}
-
}
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java
index 023bdd9..0e0e899 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java
@@ -50,6 +50,9 @@
private static IHyracksClientConnection hcc;
private final List<File> outputFiles;
+
+ protected static int DEFAULT_MEM_PAGE_SIZE = 32768;
+ protected static int DEFAULT_MEM_NUM_PAGES = 1000;
@Rule
public TemporaryFolder outputFolder = new TemporaryFolder();
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
index fa2d216..e46efff 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
@@ -78,7 +78,8 @@
resourceID = resourceIdFactory.createId();
ILocalResourceFactory localResourceFactory = opDesc.getLocalResourceFactoryProvider()
.getLocalResourceFactory();
- localResourceRepository.insert(localResourceFactory.createLocalResource(resourceID, file.getFile().getPath()));
+ localResourceRepository.insert(localResourceFactory.createLocalResource(resourceID, file.getFile()
+ .getPath(), partition));
} catch (IOException e) {
throw new HyracksDataException(e);
}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
index 814d8b3..f451748 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
@@ -30,14 +30,15 @@
public LSMBTreeDataflowHelperFactory(ILSMMergePolicyProvider mergePolicyProvider,
ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
- ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
- super(mergePolicyProvider, opTrackerFactory, ioSchedulerProvider, ioOpCallbackProvider);
+ ILSMIOOperationCallbackProvider ioOpCallbackProvider, int memPageSize, int memNumPages) {
+ super(mergePolicyProvider, opTrackerFactory, ioSchedulerProvider, ioOpCallbackProvider, memPageSize,
+ memNumPages);
}
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition) {
- return new LSMBTreeDataflowHelper(opDesc, ctx, partition, mergePolicyProvider.getMergePolicy(ctx),
+ return new LSMBTreeDataflowHelper(opDesc, ctx, partition, memPageSize, memNumPages, mergePolicyProvider.getMergePolicy(ctx),
opTrackerFactory, ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackProvider);
}
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMHarness.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMHarness.java
index 7b37c1a..04d7fa3 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMHarness.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMHarness.java
@@ -22,7 +22,11 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
public interface ILSMHarness {
- public boolean modify(ILSMIndexOperationContext ictx, boolean tryOperation, ITupleReference tuple)
+
+ public void forceModify(ILSMIndexOperationContext ctx, ITupleReference tuple) throws HyracksDataException,
+ IndexException;
+
+ public boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple)
throws HyracksDataException, IndexException;
public void noOp(ILSMIndexOperationContext ctx) throws HyracksDataException;
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
index f1c4c66..a85c24b 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
@@ -104,6 +104,12 @@
*/
public boolean tryUpsert(ITupleReference tuple) throws HyracksDataException, IndexException;
+ public void forcePhysicalDelete(ITupleReference tuple) throws HyracksDataException, IndexException;
+
+ public void forceInsert(ITupleReference tuple) throws HyracksDataException, IndexException;
+
+ public void forceDelete(ITupleReference tuple) throws HyracksDataException, IndexException;
+
/**
* This method can be used to increase the number of 'active' operations of an index artificially,
* without actually modifying the index.
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelperFactory.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelperFactory.java
index 07af416..a2f2a11 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelperFactory.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelperFactory.java
@@ -28,13 +28,17 @@
protected final ILSMOperationTrackerFactory opTrackerFactory;
protected final ILSMIOOperationSchedulerProvider ioSchedulerProvider;
protected final ILSMIOOperationCallbackProvider ioOpCallbackProvider;
+ protected final int memPageSize;
+ protected final int memNumPages;
public AbstractLSMIndexDataflowHelperFactory(ILSMMergePolicyProvider mergePolicyProvider,
ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
- ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
+ ILSMIOOperationCallbackProvider ioOpCallbackProvider, int memPageSize, int memNumPages) {
this.mergePolicyProvider = mergePolicyProvider;
this.opTrackerFactory = opTrackerFactory;
this.ioSchedulerProvider = ioSchedulerProvider;
this.ioOpCallbackProvider = ioOpCallbackProvider;
+ this.memPageSize = memPageSize;
+ this.memNumPages = memNumPages;
}
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractImmutableLSMComponent.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractImmutableLSMComponent.java
index 5382b81..b6fc2f7 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractImmutableLSMComponent.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractImmutableLSMComponent.java
@@ -53,6 +53,7 @@
case MODIFICATION:
case SEARCH:
readerCount--;
+
if (readerCount == 0 && state == ComponentState.READABLE_MERGING) {
destroy();
state = ComponentState.KILLED;
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMutableLSMComponent.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMutableLSMComponent.java
index 6575fea..1a6636a 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMutableLSMComponent.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMutableLSMComponent.java
@@ -25,14 +25,20 @@
@Override
public synchronized boolean threadEnter(LSMOperationType opType) throws InterruptedException {
switch (opType) {
+ case FORCE_MODIFICATION:
+ if (state != ComponentState.READABLE_WRITABLE && state != ComponentState.READABLE_UNWRITABLE) {
+ return false;
+ }
+ writerCount++;
+ break;
case MODIFICATION:
- while (state != ComponentState.READABLE_WRITABLE) {
+ if (state != ComponentState.READABLE_WRITABLE) {
return false;
}
writerCount++;
break;
case SEARCH:
- while (state == ComponentState.UNREADABLE_UNWRITABLE) {
+ if (state == ComponentState.UNREADABLE_UNWRITABLE) {
return false;
}
readerCount++;
@@ -58,6 +64,7 @@
@Override
public synchronized void threadExit(LSMOperationType opType, boolean failedOperation) throws HyracksDataException {
switch (opType) {
+ case FORCE_MODIFICATION:
case MODIFICATION:
writerCount--;
if (state == ComponentState.READABLE_WRITABLE && isFull()) {
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index d082a93..da87ed9 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -48,12 +48,6 @@
if (!lsmIndex.getFlushStatus(lsmIndex) && lsmIndex.getInMemoryFreePageManager().isFull()) {
lsmIndex.setFlushStatus(lsmIndex, true);
}
- if (opTracker == null) {
- System.out.println("break");
- }
- if (opCtx == null) {
- System.out.println("break");
- }
opTracker.afterOperation(opType, opCtx.getSearchOperationCallback(), opCtx.getModificationCallback());
}
@@ -111,9 +105,21 @@
}
@Override
+ public void forceModify(ILSMIndexOperationContext ctx, ITupleReference tuple) throws HyracksDataException,
+ IndexException {
+ LSMOperationType opType = LSMOperationType.FORCE_MODIFICATION;
+ modify(ctx, false, tuple, opType);
+ }
+
+ @Override
public boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple)
throws HyracksDataException, IndexException {
LSMOperationType opType = LSMOperationType.MODIFICATION;
+ return modify(ctx, tryOperation, tuple, opType);
+ }
+
+ private boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple,
+ LSMOperationType opType) throws HyracksDataException, IndexException {
if (!getAndEnterComponents(ctx, opType, tryOperation)) {
return false;
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMOperationType.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMOperationType.java
index 1a97cb2..981cefe 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMOperationType.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMOperationType.java
@@ -3,6 +3,7 @@
public enum LSMOperationType {
SEARCH,
MODIFICATION,
+ FORCE_MODIFICATION,
FLUSH,
MERGE,
NOOP
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
index ef0e0e7..7cc29a5 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
@@ -125,4 +125,22 @@
public void noOp() throws HyracksDataException {
lsmHarness.noOp(ctx);
}
+
+ @Override
+ public void forcePhysicalDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.PHYSICALDELETE);
+ lsmHarness.forceModify(ctx, tuple);
+ }
+
+ @Override
+ public void forceInsert(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.INSERT);
+ lsmHarness.forceModify(ctx, tuple);
+ }
+
+ @Override
+ public void forceDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.DELETE);
+ lsmHarness.forceModify(ctx, tuple);
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
index 3319b92..9796ebc 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
@@ -30,14 +30,15 @@
public LSMInvertedIndexDataflowHelperFactory(ILSMMergePolicyProvider mergePolicyProvider,
ILSMOperationTrackerFactory opTrackerProvider, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
- ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
- super(mergePolicyProvider, opTrackerProvider, ioSchedulerProvider, ioOpCallbackProvider);
+ ILSMIOOperationCallbackProvider ioOpCallbackProvider, int memPageSize, int memNumPages) {
+ super(mergePolicyProvider, opTrackerProvider, ioSchedulerProvider, ioOpCallbackProvider, memPageSize,
+ memNumPages);
}
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition) {
- return new LSMInvertedIndexDataflowHelper(opDesc, ctx, partition, mergePolicyProvider.getMergePolicy(ctx),
+ return new LSMInvertedIndexDataflowHelper(opDesc, ctx, partition, memPageSize, memNumPages, mergePolicyProvider.getMergePolicy(ctx),
opTrackerFactory, ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackProvider);
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
index 19c2a98..8a8aad2 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
@@ -30,14 +30,15 @@
public PartitionedLSMInvertedIndexDataflowHelperFactory(ILSMMergePolicyProvider mergePolicyProvider,
ILSMOperationTrackerFactory opTrackerProvider, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
- ILSMIOOperationCallbackProvider ioOpCallbackProvider) {
- super(mergePolicyProvider, opTrackerProvider, ioSchedulerProvider, ioOpCallbackProvider);
+ ILSMIOOperationCallbackProvider ioOpCallbackProvider, int memPageSize, int memNumPages) {
+ super(mergePolicyProvider, opTrackerProvider, ioSchedulerProvider, ioOpCallbackProvider, memNumPages,
+ memNumPages);
}
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition) {
- return new PartitionedLSMInvertedIndexDataflowHelper(opDesc, ctx, partition,
+ return new PartitionedLSMInvertedIndexDataflowHelper(opDesc, ctx, partition, memPageSize, memNumPages,
mergePolicyProvider.getMergePolicy(ctx), opTrackerFactory, ioSchedulerProvider.getIOScheduler(ctx),
ioOpCallbackProvider);
}
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 30a678d..159352f 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -274,7 +274,14 @@
@Override
public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
LSMInvertedIndexOpContext ctx = (LSMInvertedIndexOpContext) ictx;
+ // TODO: This is a hack to support logging properly in ASTERIX.
+ // The proper undo operations are only dependent on the after image so
+ // it is correct to say we found nothing (null) as the before image (at least
+ // in the perspective of ASTERIX). The semantics for the operation callbacks
+ // are violated here (and they are somewhat unclear in the first place as to
+ // what they should be for an inverted index).
ctx.modificationCallback.before(tuple);
+ ctx.modificationCallback.found(null, tuple);
switch (ctx.getOperation()) {
case INSERT: {
// Insert into the in-memory inverted index.
@@ -282,7 +289,6 @@
break;
}
case DELETE: {
- ctx.modificationCallback.before(tuple);
// First remove all entries in the in-memory inverted index (if any).
ctx.memInvIndexAccessor.delete(tuple);
// Insert key into the deleted-keys BTree.
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
index 6ba6c09..7f7d3cd 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
@@ -117,6 +117,23 @@
}
@Override
+ public void forcePhysicalDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ throw new UnsupportedOperationException("Physical delete not supported by lsm inverted index.");
+ }
+
+ @Override
+ public void forceInsert(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.INSERT);
+ lsmHarness.forceModify(ctx, tuple);
+ }
+
+ @Override
+ public void forceDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.DELETE);
+ lsmHarness.forceModify(ctx, tuple);
+ }
+
+ @Override
public void physicalDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
throw new UnsupportedOperationException("Physical delete not supported by lsm inverted index.");
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
index bde22a5..a730895 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
@@ -41,8 +41,9 @@
RTreePolicyType rtreePolicyType, IBinaryComparatorFactory[] btreeComparatorFactories,
ILSMMergePolicyProvider mergePolicyProvider, ILSMOperationTrackerFactory opTrackerFactory,
ILSMIOOperationSchedulerProvider ioSchedulerProvider, ILSMIOOperationCallbackProvider ioOpCallbackProvider,
- ILinearizeComparatorFactory linearizeCmpFactory) {
- super(mergePolicyProvider, opTrackerFactory, ioSchedulerProvider, ioOpCallbackProvider);
+ ILinearizeComparatorFactory linearizeCmpFactory, int memPageSize, int memNumPages) {
+ super(mergePolicyProvider, opTrackerFactory, ioSchedulerProvider, ioOpCallbackProvider, memPageSize,
+ memNumPages);
this.btreeComparatorFactories = btreeComparatorFactories;
this.valueProviderFactories = valueProviderFactories;
this.rtreePolicyType = rtreePolicyType;
@@ -52,7 +53,7 @@
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition) {
- return new LSMRTreeDataflowHelper(opDesc, ctx, partition, btreeComparatorFactories, valueProviderFactories,
+ return new LSMRTreeDataflowHelper(opDesc, ctx, partition, memPageSize, memNumPages, btreeComparatorFactories, valueProviderFactories,
rtreePolicyType, mergePolicyProvider.getMergePolicy(ctx), opTrackerFactory,
ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackProvider, linearizeCmpFactory);
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index cacac12..0e85cfd 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -30,6 +30,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IInMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOperationContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
@@ -277,7 +278,7 @@
public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
LSMRTreeOpContext ctx = (LSMRTreeOpContext) ictx;
if (ctx.getOperation() == IndexOperation.PHYSICALDELETE) {
- throw new UnsupportedOperationException("Physical delete not yet supported in LSM R-tree");
+ throw new UnsupportedOperationException("Physical delete not supported in the LSM-RTree");
}
if (ctx.getOperation() == IndexOperation.INSERT) {
@@ -325,15 +326,14 @@
}
}
- protected LSMRTreeOpContext createOpContext() {
- return new LSMRTreeOpContext((RTree.RTreeAccessor) mutableComponent.getRTree().createAccessor(
- NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE),
- (IRTreeLeafFrame) rtreeLeafFrameFactory.createFrame(),
+ protected LSMRTreeOpContext createOpContext(IModificationOperationCallback modCallback) {
+ return new LSMRTreeOpContext((RTree.RTreeAccessor) mutableComponent.getRTree().createAccessor(modCallback,
+ NoOpOperationCallback.INSTANCE), (IRTreeLeafFrame) rtreeLeafFrameFactory.createFrame(),
(IRTreeInteriorFrame) rtreeInteriorFrameFactory.createFrame(), memFreePageManager
.getMetaDataFrameFactory().createFrame(), 4, (BTree.BTreeAccessor) mutableComponent.getBTree()
- .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE),
- btreeLeafFrameFactory, btreeInteriorFrameFactory, memFreePageManager.getMetaDataFrameFactory()
- .createFrame(), rtreeCmpFactories, btreeCmpFactories, null, null);
+ .createAccessor(modCallback, NoOpOperationCallback.INSTANCE), btreeLeafFrameFactory,
+ btreeInteriorFrameFactory, memFreePageManager.getMetaDataFrameFactory().createFrame(),
+ rtreeCmpFactories, btreeCmpFactories, null, null);
}
@Override
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 1e8766a..ea6548e 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -195,7 +195,7 @@
public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException {
LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- ILSMIndexOperationContext rctx = createOpContext();
+ ILSMIndexOperationContext rctx = createOpContext(NoOpOperationCallback.INSTANCE);
LSMRTreeMutableComponent flushingComponent = (LSMRTreeMutableComponent) ctx.getComponentHolder().get(0);
rctx.setOperation(IndexOperation.FLUSH);
rctx.getComponentHolder().addAll(ctx.getComponentHolder());
@@ -297,7 +297,7 @@
// read the file names when we open the tree.
// The RTree should be renamed before the BTree.
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
- ILSMIndexOperationContext rctx = createOpContext();
+ ILSMIndexOperationContext rctx = createOpContext(NoOpOperationCallback.INSTANCE);
rctx.getComponentHolder().addAll(mergingComponents);
ITreeIndexCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer);
ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
@@ -351,7 +351,7 @@
@Override
public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
- return new LSMRTreeAccessor(lsmHarness, createOpContext());
+ return new LSMRTreeAccessor(lsmHarness, createOpContext(modificationCallback));
}
public class LSMRTreeAccessor extends LSMTreeIndexAccessor {
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index 1131ace..ea3fc46 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -198,7 +198,7 @@
@Override
public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException {
- LSMRTreeOpContext opCtx = createOpContext();
+ LSMRTreeOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE);
LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference();
ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
opCtx.setOperation(IndexOperation.FLUSH);
@@ -307,7 +307,7 @@
public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException, IndexException {
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
- LSMRTreeOpContext rctx = createOpContext();
+ LSMRTreeOpContext rctx = createOpContext(NoOpOperationCallback.INSTANCE);
rctx.getComponentHolder().addAll(mergingComponents);
ITreeIndexCursor cursor = new LSMRTreeWithAntiMatterTuplesSearchCursor(ctx);
ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
@@ -353,7 +353,7 @@
@Override
public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
- return new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness, createOpContext());
+ return new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness, createOpContext(modificationCallback));
}
public class LSMRTreeWithAntiMatterTuplesAccessor extends LSMTreeIndexAccessor {
diff --git a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/ILocalResourceFactory.java b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/ILocalResourceFactory.java
index 7fed1cf..c443b0b 100644
--- a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/ILocalResourceFactory.java
+++ b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/ILocalResourceFactory.java
@@ -15,5 +15,5 @@
package edu.uci.ics.hyracks.storage.common.file;
public interface ILocalResourceFactory {
- public LocalResource createLocalResource(long resourceId, String resourceName);
+ public LocalResource createLocalResource(long resourceId, String resourceName, int partition);
}
diff --git a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/LocalResource.java b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/LocalResource.java
index 3ef3ce7..f4575f3 100644
--- a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/LocalResource.java
+++ b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/LocalResource.java
@@ -20,14 +20,21 @@
private static final long serialVersionUID = 1L;
private final long resourceId;
private final String resourceName;
+ private final int partition;
private final int resourceType;
private final Object object;
- public LocalResource(long resourceId, String resourceName, int resourceType, Object object) {
+ public static final int TransientResource = 0;
+ public static final int LSMBTreeResource = 1;
+ public static final int LSMRTreeResource = 2;
+ public static final int LSMInvertedIndexResource = 3;
+
+ public LocalResource(long resourceId, String resourceName, int partition, int resourceType, Object object) {
this.resourceId = resourceId;
this.resourceName = resourceName;
- this.object = object;
+ this.partition = partition;
this.resourceType = resourceType;
+ this.object = object;
}
public long getResourceId() {
@@ -38,6 +45,10 @@
return resourceName;
}
+ public int getPartition() {
+ return partition;
+ }
+
public int getResourceType() {
return resourceType;
}
diff --git a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/TransientLocalResourceFactory.java b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/TransientLocalResourceFactory.java
index 5d4bbbd..ffd3dff 100644
--- a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/TransientLocalResourceFactory.java
+++ b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/TransientLocalResourceFactory.java
@@ -16,10 +16,8 @@
public class TransientLocalResourceFactory implements ILocalResourceFactory {
- public static final int TransientResource = 0;
-
@Override
- public LocalResource createLocalResource(long resourceId, String resourceName) {
- return new LocalResource(resourceId, resourceName, TransientResource, null);
+ public LocalResource createLocalResource(long resourceId, String resourceName, int partition) {
+ return new LocalResource(resourceId, resourceName, partition, LocalResource.TransientResource, null);
}
}