added force/undo operations to the lsm indexes; enabled proper logging callbacks for secondary indexes
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_lsm_tree@2740 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMHarness.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMHarness.java
index 7b37c1a..04d7fa3 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMHarness.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMHarness.java
@@ -22,7 +22,11 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
public interface ILSMHarness {
- public boolean modify(ILSMIndexOperationContext ictx, boolean tryOperation, ITupleReference tuple)
+
+ public void forceModify(ILSMIndexOperationContext ctx, ITupleReference tuple) throws HyracksDataException,
+ IndexException;
+
+ public boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple)
throws HyracksDataException, IndexException;
public void noOp(ILSMIndexOperationContext ctx) throws HyracksDataException;
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
index f1c4c66..a85c24b 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
@@ -104,6 +104,12 @@
*/
public boolean tryUpsert(ITupleReference tuple) throws HyracksDataException, IndexException;
+ public void forcePhysicalDelete(ITupleReference tuple) throws HyracksDataException, IndexException;
+
+ public void forceInsert(ITupleReference tuple) throws HyracksDataException, IndexException;
+
+ public void forceDelete(ITupleReference tuple) throws HyracksDataException, IndexException;
+
/**
* This method can be used to increase the number of 'active' operations of an index artificially,
* without actually modifying the index.
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractImmutableLSMComponent.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractImmutableLSMComponent.java
index 5382b81..b6fc2f7 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractImmutableLSMComponent.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractImmutableLSMComponent.java
@@ -53,6 +53,7 @@
case MODIFICATION:
case SEARCH:
readerCount--;
+
if (readerCount == 0 && state == ComponentState.READABLE_MERGING) {
destroy();
state = ComponentState.KILLED;
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMutableLSMComponent.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMutableLSMComponent.java
index 6575fea..1a6636a 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMutableLSMComponent.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMutableLSMComponent.java
@@ -25,14 +25,20 @@
@Override
public synchronized boolean threadEnter(LSMOperationType opType) throws InterruptedException {
switch (opType) {
+ case FORCE_MODIFICATION:
+ if (state != ComponentState.READABLE_WRITABLE && state != ComponentState.READABLE_UNWRITABLE) {
+ return false;
+ }
+ writerCount++;
+ break;
case MODIFICATION:
- while (state != ComponentState.READABLE_WRITABLE) {
+ if (state != ComponentState.READABLE_WRITABLE) {
return false;
}
writerCount++;
break;
case SEARCH:
- while (state == ComponentState.UNREADABLE_UNWRITABLE) {
+ if (state == ComponentState.UNREADABLE_UNWRITABLE) {
return false;
}
readerCount++;
@@ -58,6 +64,7 @@
@Override
public synchronized void threadExit(LSMOperationType opType, boolean failedOperation) throws HyracksDataException {
switch (opType) {
+ case FORCE_MODIFICATION:
case MODIFICATION:
writerCount--;
if (state == ComponentState.READABLE_WRITABLE && isFull()) {
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index d082a93..da87ed9 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -48,12 +48,6 @@
if (!lsmIndex.getFlushStatus(lsmIndex) && lsmIndex.getInMemoryFreePageManager().isFull()) {
lsmIndex.setFlushStatus(lsmIndex, true);
}
- if (opTracker == null) {
- System.out.println("break");
- }
- if (opCtx == null) {
- System.out.println("break");
- }
opTracker.afterOperation(opType, opCtx.getSearchOperationCallback(), opCtx.getModificationCallback());
}
@@ -111,9 +105,21 @@
}
@Override
+ public void forceModify(ILSMIndexOperationContext ctx, ITupleReference tuple) throws HyracksDataException,
+ IndexException {
+ LSMOperationType opType = LSMOperationType.FORCE_MODIFICATION;
+ modify(ctx, false, tuple, opType);
+ }
+
+ @Override
public boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple)
throws HyracksDataException, IndexException {
LSMOperationType opType = LSMOperationType.MODIFICATION;
+ return modify(ctx, tryOperation, tuple, opType);
+ }
+
+ private boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple,
+ LSMOperationType opType) throws HyracksDataException, IndexException {
if (!getAndEnterComponents(ctx, opType, tryOperation)) {
return false;
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMOperationType.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMOperationType.java
index 1a97cb2..981cefe 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMOperationType.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMOperationType.java
@@ -3,6 +3,7 @@
public enum LSMOperationType {
SEARCH,
MODIFICATION,
+ FORCE_MODIFICATION,
FLUSH,
MERGE,
NOOP
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
index ef0e0e7..7cc29a5 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
@@ -125,4 +125,22 @@
public void noOp() throws HyracksDataException {
lsmHarness.noOp(ctx);
}
+
+ @Override
+ public void forcePhysicalDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.PHYSICALDELETE);
+ lsmHarness.forceModify(ctx, tuple);
+ }
+
+ @Override
+ public void forceInsert(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.INSERT);
+ lsmHarness.forceModify(ctx, tuple);
+ }
+
+ @Override
+ public void forceDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.DELETE);
+ lsmHarness.forceModify(ctx, tuple);
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index ced1aa9..da6bc28 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -274,7 +274,14 @@
@Override
public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
LSMInvertedIndexOpContext ctx = (LSMInvertedIndexOpContext) ictx;
+ // TODO: This is a hack to support logging properly in ASTERIX.
+ // The proper undo operations are only dependent on the after image so
+ // it is correct to say we found nothing (null) as the before image (at least
+ // in the perspective of ASTERIX). The semantics for the operation callbacks
+ // are violated here (and they are somewhat unclear in the first place as to
+ // what they should be for an inverted index).
ctx.modificationCallback.before(tuple);
+ ctx.modificationCallback.found(null, tuple);
switch (ctx.getOperation()) {
case INSERT: {
// Insert into the in-memory inverted index.
@@ -282,7 +289,6 @@
break;
}
case DELETE: {
- ctx.modificationCallback.before(tuple);
// First remove all entries in the in-memory inverted index (if any).
ctx.memInvIndexAccessor.delete(tuple);
// Insert key into the deleted-keys BTree.
diff --git a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
index 6ba6c09..7f7d3cd 100644
--- a/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
+++ b/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
@@ -117,6 +117,23 @@
}
@Override
+ public void forcePhysicalDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ throw new UnsupportedOperationException("Physical delete not supported by lsm inverted index.");
+ }
+
+ @Override
+ public void forceInsert(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.INSERT);
+ lsmHarness.forceModify(ctx, tuple);
+ }
+
+ @Override
+ public void forceDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.DELETE);
+ lsmHarness.forceModify(ctx, tuple);
+ }
+
+ @Override
public void physicalDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
throw new UnsupportedOperationException("Physical delete not supported by lsm inverted index.");
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index 6cafbbb..5a1482f 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -30,6 +30,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IInMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOperationContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
@@ -277,7 +278,7 @@
public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
LSMRTreeOpContext ctx = (LSMRTreeOpContext) ictx;
if (ctx.getOperation() == IndexOperation.PHYSICALDELETE) {
- throw new UnsupportedOperationException("Physical delete not yet supported in LSM R-tree");
+ throw new UnsupportedOperationException("Physical delete not supported in the LSM-RTree");
}
if (ctx.getOperation() == IndexOperation.INSERT) {
@@ -325,15 +326,14 @@
}
}
- protected LSMRTreeOpContext createOpContext() {
- return new LSMRTreeOpContext((RTree.RTreeAccessor) mutableComponent.getRTree().createAccessor(
- NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE),
- (IRTreeLeafFrame) rtreeLeafFrameFactory.createFrame(),
+ protected LSMRTreeOpContext createOpContext(IModificationOperationCallback modCallback) {
+ return new LSMRTreeOpContext((RTree.RTreeAccessor) mutableComponent.getRTree().createAccessor(modCallback,
+ NoOpOperationCallback.INSTANCE), (IRTreeLeafFrame) rtreeLeafFrameFactory.createFrame(),
(IRTreeInteriorFrame) rtreeInteriorFrameFactory.createFrame(), memFreePageManager
.getMetaDataFrameFactory().createFrame(), 4, (BTree.BTreeAccessor) mutableComponent.getBTree()
- .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE),
- btreeLeafFrameFactory, btreeInteriorFrameFactory, memFreePageManager.getMetaDataFrameFactory()
- .createFrame(), rtreeCmpFactories, btreeCmpFactories, null, null);
+ .createAccessor(modCallback, NoOpOperationCallback.INSTANCE), btreeLeafFrameFactory,
+ btreeInteriorFrameFactory, memFreePageManager.getMetaDataFrameFactory().createFrame(),
+ rtreeCmpFactories, btreeCmpFactories, null, null);
}
@Override
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 39cd5d6..e43e525 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -195,7 +195,7 @@
public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException {
LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- ILSMIndexOperationContext rctx = createOpContext();
+ ILSMIndexOperationContext rctx = createOpContext(NoOpOperationCallback.INSTANCE);
LSMRTreeMutableComponent flushingComponent = (LSMRTreeMutableComponent) ctx.getComponentHolder().get(0);
rctx.setOperation(IndexOperation.FLUSH);
rctx.getComponentHolder().addAll(ctx.getComponentHolder());
@@ -297,7 +297,7 @@
// read the file names when we open the tree.
// The RTree should be renamed before the BTree.
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
- ILSMIndexOperationContext rctx = createOpContext();
+ ILSMIndexOperationContext rctx = createOpContext(NoOpOperationCallback.INSTANCE);
rctx.getComponentHolder().addAll(mergingComponents);
ITreeIndexCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer);
ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
@@ -351,7 +351,7 @@
@Override
public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
- return new LSMRTreeAccessor(lsmHarness, createOpContext());
+ return new LSMRTreeAccessor(lsmHarness, createOpContext(modificationCallback));
}
public class LSMRTreeAccessor extends LSMTreeIndexAccessor {
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index 47df7df..824b8ef 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -199,7 +199,7 @@
@Override
public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException {
- LSMRTreeOpContext opCtx = createOpContext();
+ LSMRTreeOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE);
LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference();
ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
opCtx.setOperation(IndexOperation.FLUSH);
@@ -308,7 +308,7 @@
public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException, IndexException {
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
- LSMRTreeOpContext rctx = createOpContext();
+ LSMRTreeOpContext rctx = createOpContext(NoOpOperationCallback.INSTANCE);
rctx.getComponentHolder().addAll(mergingComponents);
ITreeIndexCursor cursor = new LSMRTreeWithAntiMatterTuplesSearchCursor(ctx);
ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
@@ -354,7 +354,7 @@
@Override
public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
- return new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness, createOpContext());
+ return new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness, createOpContext(modificationCallback));
}
public class LSMRTreeWithAntiMatterTuplesAccessor extends LSMTreeIndexAccessor {