Improve reading from and writing to Metadata pages
This change introduces a new interface IComponentMetadata.
Each LSM component is associated with a metadata object
which can be used to read and write arbitrary data to
the metadata pages of components. When flushing a
component, data in its metadata component is automatically
flushed to the disk component. For merge operations,
the IO Callback is responsible for merging the components'
metadata pages.
Change-Id: Id95ef33c0a0bc1abb3fc3ecdea5611ee4acd6dfa
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1476
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <imaxon@apache.org>
Integration-Tests: Ian Maxon <imaxon@apache.org>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorApiLetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorApiServletTest.java
similarity index 99%
rename from asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorApiLetTest.java
rename to asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorApiServletTest.java
index d9a0a79..7da3b32 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorApiLetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorApiServletTest.java
@@ -57,7 +57,7 @@
import io.netty.handler.codec.http.HttpMethod;
import junit.extensions.PA;
-public class ConnectorApiLetTest {
+public class ConnectorApiServletTest {
@Test
public void testGet() throws Exception {
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiLetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiServletTest.java
similarity index 99%
rename from asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiLetTest.java
rename to asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiServletTest.java
index 619e7a5..7fed010 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiLetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiServletTest.java
@@ -46,7 +46,7 @@
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.codec.http.HttpMethod;
-public class VersionApiLetTest {
+public class VersionApiServletTest {
@Test
public void testGet() throws Exception {
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
index edd1848..1467dbf 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
@@ -51,6 +51,7 @@
import org.apache.hyracks.api.test.FrameWriterTestUtils;
import org.apache.hyracks.api.test.FrameWriterTestUtils.FrameWriterOperation;
import org.apache.hyracks.api.util.HyracksConstants;
+import org.apache.hyracks.data.std.primitive.LongPointable;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.dataflow.common.io.MessagingFrameTupleAppender;
@@ -58,6 +59,7 @@
import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree;
import org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory;
+import org.apache.hyracks.storage.am.lsm.common.utils.ComponentMetadataUtil;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -68,8 +70,8 @@
private static final IAType[] KEY_TYPES = { BuiltinType.AINT32 };
private static final ARecordType RECORD_TYPE = new ARecordType("TestRecordType", new String[] { "key", "value" },
new IAType[] { BuiltinType.AINT32, BuiltinType.AINT64 }, false);
- private static final GenerationFunction[] RECORD_GEN_FUNCTION = { GenerationFunction.DETERMINISTIC,
- GenerationFunction.DETERMINISTIC };
+ private static final GenerationFunction[] RECORD_GEN_FUNCTION =
+ { GenerationFunction.DETERMINISTIC, GenerationFunction.DETERMINISTIC };
private static final boolean[] UNIQUE_RECORD_FIELDS = { true, false };
private static final ARecordType META_TYPE = null;
private static final GenerationFunction[] META_GEN_FUNCTION = null;
@@ -146,7 +148,9 @@
KEY_INDICATORS_LIST);
dataflowHelper.open();
LSMBTree btree = (LSMBTree) dataflowHelper.getIndexInstance();
- long lsn = btree.getMostRecentMarkerLSN();
+ LongPointable longPointable = LongPointable.FACTORY.createPointable();
+ ComponentMetadataUtil.get(btree, ComponentMetadataUtil.MARKER_LSN_KEY, longPointable);
+ long lsn = longPointable.getLong();
int numOfMarkers = 0;
LogReader logReader = (LogReader) nc.getTransactionSubsystem().getLogManager().getLogReader(false);
long expectedMarkerId = markerId - 1;
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java
index 70339f3..e16ce78 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicy.java
@@ -30,12 +30,12 @@
import org.apache.hyracks.storage.am.common.api.IResourceLifecycleManager;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.ComponentState;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
-import org.apache.hyracks.storage.am.lsm.common.impls.AbstractDiskLSMComponent;
public class CorrelatedPrefixMergePolicy implements ILSMMergePolicy {
@@ -60,18 +60,18 @@
// all such components for which the sum of their sizes exceeds MaxMrgCompSz. Schedule a merge of those components into a new component.
// 2. If a merge from 1 doesn't happen, see if the set of candidate components for merging exceeds MaxTolCompCnt. If so, schedule
// a merge all of the current candidates into a new single component.
- List<ILSMComponent> immutableComponents = new ArrayList<ILSMComponent>(index.getImmutableComponents());
+ List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getImmutableComponents());
// Reverse the components order so that we look at components from oldest to newest.
Collections.reverse(immutableComponents);
- for (ILSMComponent c : immutableComponents) {
+ for (ILSMDiskComponent c : immutableComponents) {
if (c.getState() != ComponentState.READABLE_UNWRITABLE) {
return;
}
}
if (fullMergeIsRequested) {
- ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
+ ILSMIndexAccessor accessor =
+ index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
accessor.scheduleFullMerge(index.getIOOperationCallback());
return;
}
@@ -89,7 +89,7 @@
for (int i = 0; i < minNumComponents; i++) {
ILSMComponent c = immutableComponents.get(i);
- long componentSize = ((AbstractDiskLSMComponent) c).getComponentSize();
+ long componentSize = ((ILSMDiskComponent) c).getComponentSize();
if (componentSize > maxMergableComponentSize) {
startIndex = i;
totalSize = 0;
@@ -101,20 +101,20 @@
|| (isLastComponent && i - startIndex >= maxToleranceComponentCount)) {
for (ILSMIndex lsmIndex : indexes) {
- List<ILSMComponent> reversedImmutableComponents = new ArrayList<ILSMComponent>(
- lsmIndex.getImmutableComponents());
+ List<ILSMDiskComponent> reversedImmutableComponents =
+ new ArrayList<>(lsmIndex.getImmutableComponents());
// Reverse the components order so that we look at components from oldest to newest.
Collections.reverse(reversedImmutableComponents);
- List<ILSMComponent> mergableComponents = new ArrayList<ILSMComponent>();
+ List<ILSMDiskComponent> mergableComponents = new ArrayList<>();
for (int j = startIndex + 1; j <= i; j++) {
mergableComponents.add(reversedImmutableComponents.get(j));
}
// Reverse the components order back to its original order
Collections.reverse(mergableComponents);
- ILSMIndexAccessor accessor = lsmIndex.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
+ ILSMIndexAccessor accessor =
+ lsmIndex.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
accessor.scheduleMerge(lsmIndex.getIOOperationCallback(), mergableComponents);
}
break;
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
index 6350d73..7fca039 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
@@ -35,7 +35,6 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.ComponentState;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexInternal;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
@@ -101,8 +100,7 @@
if (!flushOnExit) {
for (ILSMIndex lsmIndex : indexes) {
- ILSMIndexInternal lsmIndexInternal = (ILSMIndexInternal) lsmIndex;
- if (lsmIndexInternal.hasFlushRequestForCurrentMutableComponent()) {
+ if (lsmIndex.hasFlushRequestForCurrentMutableComponent()) {
needsFlush = true;
break;
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMIndexUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMIndexUtil.java
index cd40179..04090bb 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMIndexUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMIndexUtil.java
@@ -21,7 +21,8 @@
import org.apache.asterix.common.ioopcallbacks.AbstractLSMIOOperationCallback;
import org.apache.asterix.common.transactions.ILogManager;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
public class LSMIndexUtil {
@@ -33,18 +34,18 @@
//prevent transactions from incorrectly setting the first LSN on a modified component by checking the index is still empty
synchronized (lsmIndex.getOperationTracker()) {
if (lsmIndex.isCurrentMutableComponentEmpty()) {
- AbstractLSMIOOperationCallback ioOpCallback = (AbstractLSMIOOperationCallback) lsmIndex
- .getIOOperationCallback();
+ AbstractLSMIOOperationCallback ioOpCallback =
+ (AbstractLSMIOOperationCallback) lsmIndex.getIOOperationCallback();
ioOpCallback.setFirstLSN(logManager.getAppendLSN());
}
}
}
}
- public static long getComponentFileLSNOffset(AbstractLSMIndex lsmIndex, ILSMComponent lsmComponent,
+ public static long getComponentFileLSNOffset(ILSMIndex lsmIndex, ILSMDiskComponent lsmComponent,
String componentFilePath) throws HyracksDataException {
- AbstractLSMIOOperationCallback ioOpCallback = (AbstractLSMIOOperationCallback) lsmIndex
- .getIOOperationCallback();
+ AbstractLSMIOOperationCallback ioOpCallback =
+ (AbstractLSMIOOperationCallback) lsmIndex.getIOOperationCallback();
return ioOpCallback.getComponentFileLSNOffset(lsmComponent, componentFilePath);
}
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/RuntimeDataException.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/RuntimeDataException.java
index fded0d9..0c099fb 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/RuntimeDataException.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/RuntimeDataException.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
public class RuntimeDataException extends HyracksDataException {
+ private static final long serialVersionUID = 1L;
public RuntimeDataException(int errorCode, Serializable... params) {
super(ErrorCode.ASTERIX, errorCode, ErrorCode.getErrorMessage(errorCode), params);
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
index 29af7c9..f903b65 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
@@ -27,13 +27,13 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.freepage.MutableArrayValueReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
// A single LSMIOOperationCallback per LSM index used to perform actions around Flush and Merge operations
public abstract class AbstractLSMIOOperationCallback implements ILSMIOOperationCallback {
- public static final MutableArrayValueReference LSN_KEY = new MutableArrayValueReference("LSN"
- .getBytes());
+ public static final MutableArrayValueReference LSN_KEY = new MutableArrayValueReference("LSN".getBytes());
public static final long INVALID = -1L;
// First LSN per mutable component
@@ -78,7 +78,7 @@
}
@Override
- public void afterFinalize(LSMOperationType opType, ILSMComponent newComponent) {
+ public void afterFinalize(LSMOperationType opType, ILSMDiskComponent newComponent) {
// The operation was complete and the next I/O operation for the LSM index didn't start yet
if (opType == LSMOperationType.FLUSH && newComponent != null) {
synchronized (this) {
@@ -93,15 +93,11 @@
}
}
- public abstract long getComponentLSN(List<ILSMComponent> oldComponents) throws HyracksDataException;
+ public abstract long getComponentLSN(List<? extends ILSMComponent> oldComponents) throws HyracksDataException;
- public void putLSNIntoMetadata(ITreeIndex treeIndex, List<ILSMComponent> oldComponents)
+ public void putLSNIntoMetadata(ILSMDiskComponent index, List<ILSMComponent> oldComponents)
throws HyracksDataException {
- byte[] lsn = new byte[Long.BYTES];
- LongPointable.setLong(lsn, 0, getComponentLSN(oldComponents));
- IMetadataPageManager metadataPageManager = (IMetadataPageManager) treeIndex.getPageManager();
- metadataPageManager.put(metadataPageManager.createMetadataFrame(), LSN_KEY, new MutableArrayValueReference(
- lsn));
+ index.getMetadata().put(LSN_KEY, LongPointable.FACTORY.createPointable(getComponentLSN(oldComponents)));
}
public static long getTreeIndexLSN(ITreeIndex treeIndex) throws HyracksDataException {
@@ -144,6 +140,6 @@
* otherwise {@link IMetadataPageManager#INVALID_LSN_OFFSET}.
* @throws HyracksDataException
*/
- public abstract long getComponentFileLSNOffset(ILSMComponent component, String componentFilePath)
+ public abstract long getComponentFileLSNOffset(ILSMDiskComponent component, String componentFilePath)
throws HyracksDataException;
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java
index 142b84f..173c962 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java
@@ -22,12 +22,15 @@
import java.util.List;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.LongPointable;
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeDiskComponent;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.apache.hyracks.storage.am.lsm.common.utils.ComponentMetadataUtil;
public class LSMBTreeIOOperationCallback extends AbstractLSMIOOperationCallback {
@@ -36,16 +39,24 @@
}
@Override
- public void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents, ILSMComponent newComponent)
- throws HyracksDataException {
+ public void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents,
+ ILSMDiskComponent newComponent) throws HyracksDataException {
+ //TODO: Copying Filters and all content of the metadata pages for flush operation should be done here
if (newComponent != null) {
LSMBTreeDiskComponent btreeComponent = (LSMBTreeDiskComponent) newComponent;
- putLSNIntoMetadata(btreeComponent.getBTree(), oldComponents);
+ putLSNIntoMetadata(btreeComponent, oldComponents);
+ if (opType == LSMOperationType.MERGE) {
+ LongPointable markerLsn = LongPointable.FACTORY
+ .createPointable(ComponentMetadataUtil.getLong(oldComponents.get(0).getMetadata(),
+ ComponentMetadataUtil.MARKER_LSN_KEY, ComponentMetadataUtil.NOT_FOUND));
+ btreeComponent.getMetadata().put(ComponentMetadataUtil.MARKER_LSN_KEY, markerLsn);
+ }
+
}
}
@Override
- public long getComponentLSN(List<ILSMComponent> diskComponents) throws HyracksDataException {
+ public long getComponentLSN(List<? extends ILSMComponent> diskComponents) throws HyracksDataException {
if (diskComponents == null) {
// Implies a flush IO operation. --> moves the flush pointer
// Flush operation of an LSM index are executed sequentially.
@@ -64,12 +75,12 @@
}
@Override
- public long getComponentFileLSNOffset(ILSMComponent diskComponent, String diskComponentFilePath)
+ public long getComponentFileLSNOffset(ILSMDiskComponent diskComponent, String diskComponentFilePath)
throws HyracksDataException {
if (diskComponentFilePath.endsWith(LSMBTreeFileManager.BTREE_STRING)) {
LSMBTreeDiskComponent btreeComponent = (LSMBTreeDiskComponent) diskComponent;
- IMetadataPageManager metadataPageManager = (IMetadataPageManager) btreeComponent.getBTree()
- .getPageManager();
+ IMetadataPageManager metadataPageManager =
+ (IMetadataPageManager) btreeComponent.getBTree().getPageManager();
return metadataPageManager.getFileOffset(metadataPageManager.createMetadataFrame(), LSN_KEY);
}
return INVALID;
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallbackFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallbackFactory.java
index 21882dd..322b5ef 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallbackFactory.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallbackFactory.java
@@ -32,7 +32,7 @@
}
@Override
- public ILSMIOOperationCallback createIOOperationCallback() {
+ public ILSMIOOperationCallback createIoOpCallback() {
return new LSMBTreeIOOperationCallback();
}
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java
index a8c545d..6c987d6 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java
@@ -25,21 +25,22 @@
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyDiskComponent;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
public class LSMBTreeWithBuddyIOOperationCallback extends AbstractLSMIOOperationCallback {
@Override
- public void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents, ILSMComponent newComponent)
- throws HyracksDataException {
+ public void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents,
+ ILSMDiskComponent newComponent) throws HyracksDataException {
if (newComponent != null) {
LSMBTreeWithBuddyDiskComponent btreeComponent = (LSMBTreeWithBuddyDiskComponent) newComponent;
- putLSNIntoMetadata(btreeComponent.getBTree(), oldComponents);
+ putLSNIntoMetadata(btreeComponent, oldComponents);
}
}
@Override
- public long getComponentLSN(List<ILSMComponent> diskComponents) throws HyracksDataException {
+ public long getComponentLSN(List<? extends ILSMComponent> diskComponents) throws HyracksDataException {
if (diskComponents == null) {
// Implies a flush IO operation <Will never happen currently as Btree with buddy btree is only used with external datasets>
synchronized (this) {
@@ -57,12 +58,12 @@
}
@Override
- public long getComponentFileLSNOffset(ILSMComponent diskComponent, String diskComponentFilePath)
+ public long getComponentFileLSNOffset(ILSMDiskComponent diskComponent, String diskComponentFilePath)
throws HyracksDataException {
if (diskComponentFilePath.endsWith(LSMBTreeWithBuddyFileManager.BTREE_STRING)) {
LSMBTreeWithBuddyDiskComponent btreeComponent = (LSMBTreeWithBuddyDiskComponent) diskComponent;
- IMetadataPageManager metadataPageManager = (IMetadataPageManager) btreeComponent.getBTree()
- .getPageManager();
+ IMetadataPageManager metadataPageManager =
+ (IMetadataPageManager) btreeComponent.getBTree().getPageManager();
return metadataPageManager.getFileOffset(metadataPageManager.createMetadataFrame(), LSN_KEY);
}
return INVALID;
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackFactory.java
index 848de29..1055fa6 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackFactory.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackFactory.java
@@ -32,7 +32,7 @@
}
@Override
- public ILSMIOOperationCallback createIOOperationCallback() {
+ public ILSMIOOperationCallback createIoOpCallback() {
return new LSMBTreeWithBuddyIOOperationCallback();
}
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java
index ec09d65..657d908 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
import org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexDiskComponent;
import org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexFileManager;
@@ -35,16 +36,16 @@
}
@Override
- public void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents, ILSMComponent newComponent)
- throws HyracksDataException {
+ public void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents,
+ ILSMDiskComponent newComponent) throws HyracksDataException {
if (newComponent != null) {
LSMInvertedIndexDiskComponent invIndexComponent = (LSMInvertedIndexDiskComponent) newComponent;
- putLSNIntoMetadata(invIndexComponent.getDeletedKeysBTree(), oldComponents);
+ putLSNIntoMetadata(invIndexComponent, oldComponents);
}
}
@Override
- public long getComponentLSN(List<ILSMComponent> diskComponents) throws HyracksDataException {
+ public long getComponentLSN(List<? extends ILSMComponent> diskComponents) throws HyracksDataException {
if (diskComponents == null) {
// Implies a flush IO operation.
synchronized (this) {
@@ -56,18 +57,19 @@
long maxLSN = -1;
for (Object o : diskComponents) {
LSMInvertedIndexDiskComponent invIndexComponent = (LSMInvertedIndexDiskComponent) o;
- maxLSN = Math.max(AbstractLSMIOOperationCallback.getTreeIndexLSN(invIndexComponent.getDeletedKeysBTree()), maxLSN);
+ maxLSN = Math.max(AbstractLSMIOOperationCallback.getTreeIndexLSN(invIndexComponent.getDeletedKeysBTree()),
+ maxLSN);
}
return maxLSN;
}
@Override
- public long getComponentFileLSNOffset(ILSMComponent diskComponent, String diskComponentFilePath)
+ public long getComponentFileLSNOffset(ILSMDiskComponent diskComponent, String diskComponentFilePath)
throws HyracksDataException {
if (diskComponentFilePath.endsWith(LSMInvertedIndexFileManager.DELETED_KEYS_BTREE_SUFFIX)) {
LSMInvertedIndexDiskComponent invIndexComponent = (LSMInvertedIndexDiskComponent) diskComponent;
- IMetadataPageManager metadataPageManager = (IMetadataPageManager) invIndexComponent.getDeletedKeysBTree()
- .getPageManager();
+ IMetadataPageManager metadataPageManager =
+ (IMetadataPageManager) invIndexComponent.getDeletedKeysBTree().getPageManager();
return metadataPageManager.getFileOffset(metadataPageManager.createMetadataFrame(), LSN_KEY);
}
return INVALID;
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallbackFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallbackFactory.java
index 8951cb4..0fa0167 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallbackFactory.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallbackFactory.java
@@ -33,7 +33,7 @@
}
@Override
- public ILSMIOOperationCallback createIOOperationCallback() {
+ public ILSMIOOperationCallback createIoOpCallback() {
return new LSMInvertedIndexIOOperationCallback();
}
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java
index 175250d..2dc06f7 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeDiskComponent;
import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeFileManager;
@@ -35,16 +36,16 @@
}
@Override
- public void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents, ILSMComponent newComponent)
- throws HyracksDataException {
+ public void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents,
+ ILSMDiskComponent newComponent) throws HyracksDataException {
if (newComponent != null) {
LSMRTreeDiskComponent rtreeComponent = (LSMRTreeDiskComponent) newComponent;
- putLSNIntoMetadata(rtreeComponent.getRTree(), oldComponents);
+ putLSNIntoMetadata(rtreeComponent, oldComponents);
}
}
@Override
- public long getComponentLSN(List<ILSMComponent> diskComponents) throws HyracksDataException {
+ public long getComponentLSN(List<? extends ILSMComponent> diskComponents) throws HyracksDataException {
if (diskComponents == null) {
// Implies a flush IO operation.
synchronized (this) {
@@ -62,12 +63,12 @@
}
@Override
- public long getComponentFileLSNOffset(ILSMComponent diskComponent, String diskComponentFilePath)
+ public long getComponentFileLSNOffset(ILSMDiskComponent diskComponent, String diskComponentFilePath)
throws HyracksDataException {
if (diskComponentFilePath.endsWith(LSMRTreeFileManager.RTREE_STRING)) {
LSMRTreeDiskComponent rtreeComponent = (LSMRTreeDiskComponent) diskComponent;
- IMetadataPageManager metadataPageManager = (IMetadataPageManager) rtreeComponent.getRTree()
- .getPageManager();
+ IMetadataPageManager metadataPageManager =
+ (IMetadataPageManager) rtreeComponent.getRTree().getPageManager();
return metadataPageManager.getFileOffset(metadataPageManager.createMetadataFrame(), LSN_KEY);
}
return INVALID;
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallbackFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallbackFactory.java
index 954c6e1..83db16a 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallbackFactory.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallbackFactory.java
@@ -32,7 +32,7 @@
}
@Override
- public ILSMIOOperationCallback createIOOperationCallback() {
+ public ILSMIOOperationCallback createIoOpCallback() {
return new LSMRTreeIOOperationCallback();
}
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/PrimaryIndexLogMarkerCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/PrimaryIndexLogMarkerCallback.java
index 7dae65f..b977c4d 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/PrimaryIndexLogMarkerCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/PrimaryIndexLogMarkerCallback.java
@@ -19,33 +19,104 @@
package org.apache.asterix.common.transactions;
import java.nio.ByteBuffer;
+import java.util.List;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import org.apache.hyracks.data.std.primitive.LongPointable;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
+import org.apache.hyracks.storage.am.lsm.common.utils.ComponentMetadataUtil;
/**
* A basic callback used to write marker to transaction logs
*/
public class PrimaryIndexLogMarkerCallback implements ILogMarkerCallback {
-
- private AbstractLSMIndex index;
+ private final LongPointable pointable = LongPointable.FACTORY.createPointable();
+ private final ILSMIndex index;
/**
* @param index:
* a pointer to the primary index used to store marker log info
* @throws HyracksDataException
*/
- public PrimaryIndexLogMarkerCallback(AbstractLSMIndex index) throws HyracksDataException {
+ public PrimaryIndexLogMarkerCallback(ILSMIndex index) throws HyracksDataException {
this.index = index;
}
@Override
public void before(ByteBuffer buffer) {
- buffer.putLong(index.getCurrentMemoryComponent().getMostRecentMarkerLSN());
+ buffer.putLong(getLsn());
+ }
+
+ private long getLsn() {
+ long lsn;
+ try {
+ lsn = ComponentMetadataUtil.getLong(index.getCurrentMemoryComponent().getMetadata(),
+ ComponentMetadataUtil.MARKER_LSN_KEY, ComponentMetadataUtil.NOT_FOUND);
+ } catch (HyracksDataException e) {
+ // Should never happen since this is a memory component
+ throw new IllegalStateException(e);
+ }
+ if (lsn == ComponentMetadataUtil.NOT_FOUND) {
+ synchronized (index.getOperationTracker()) {
+ // look for it in previous memory component if exists
+ lsn = lsnFromImmutableMemoryComponents();
+ if (lsn == ComponentMetadataUtil.NOT_FOUND) {
+ // look for it in disk component
+ lsn = lsnFromDiskComponents();
+ }
+ }
+ }
+ return lsn;
+ }
+
+ private long lsnFromDiskComponents() {
+ List<ILSMDiskComponent> diskComponents = index.getImmutableComponents();
+ for (ILSMDiskComponent c : diskComponents) {
+ try {
+ long lsn = ComponentMetadataUtil.getLong(c.getMetadata(), ComponentMetadataUtil.MARKER_LSN_KEY,
+ ComponentMetadataUtil.NOT_FOUND);
+ if (lsn != ComponentMetadataUtil.NOT_FOUND) {
+ return lsn;
+ }
+ } catch (HyracksDataException e) {
+ throw new IllegalStateException("Unable to read metadata page. Disk Error?", e);
+ }
+ }
+ return ComponentMetadataUtil.NOT_FOUND;
+ }
+
+ private long lsnFromImmutableMemoryComponents() {
+ List<ILSMMemoryComponent> memComponents = index.getMemoryComponents();
+ int numOtherMemComponents = memComponents.size() - 1;
+ int next = index.getCurrentMemoryComponentIndex();
+ long lsn = ComponentMetadataUtil.NOT_FOUND;
+ for (int i = 0; i < numOtherMemComponents; i++) {
+ next = next - 1;
+ if (next < 0) {
+ next = memComponents.size() - 1;
+ }
+ ILSMMemoryComponent c = index.getMemoryComponents().get(next);
+ if (c.isReadable()) {
+ try {
+ lsn = ComponentMetadataUtil.getLong(c.getMetadata(), ComponentMetadataUtil.MARKER_LSN_KEY,
+ ComponentMetadataUtil.NOT_FOUND);
+ } catch (HyracksDataException e) {
+ // Should never happen since this is a memory component
+ throw new IllegalStateException(e);
+ }
+ if (lsn != ComponentMetadataUtil.NOT_FOUND) {
+ return lsn;
+ }
+ }
+ }
+ return lsn;
}
@Override
public void after(long lsn) {
- index.getCurrentMemoryComponent().setMostRecentMarkerLSN(lsn);
+ pointable.setLong(lsn);
+ index.getCurrentMemoryComponent().getMetadata().put(ComponentMetadataUtil.MARKER_LSN_KEY, pointable);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFileIndexAccessor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFileIndexAccessor.java
index d01d1a8..a3f277a 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFileIndexAccessor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFileIndexAccessor.java
@@ -21,7 +21,6 @@
import java.io.ByteArrayInputStream;
import java.io.DataInput;
import java.io.DataInputStream;
-import java.io.Serializable;
import java.util.Date;
import org.apache.asterix.common.exceptions.ErrorCode;
@@ -44,20 +43,20 @@
import org.apache.hyracks.storage.am.btree.util.BTreeUtils;
import org.apache.hyracks.storage.am.common.api.IIndexCursor;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
+import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeDataflowHelper;
import org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
/*
* This class was created specifically to facilitate accessing
* external file index when doing external lookup during runtime
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
-public class ExternalFileIndexAccessor implements Serializable {
+public class ExternalFileIndexAccessor {
private final FilesIndexDescription filesIndexDescription = new FilesIndexDescription();
- private static final long serialVersionUID = 1L;
private ExternalBTreeDataflowHelper indexDataflowHelper;
private ExternalLookupOperatorDescriptor opDesc;
@@ -65,12 +64,11 @@
private ExternalBTree index;
private ArrayTupleBuilder searchKeyTupleBuilder;
private ArrayTupleReference searchKey;
- private MultiComparator searchCmp;
private AMutableInt32 currentFileNumber = new AMutableInt32(-1);
- private ISerializerDeserializer intSerde = SerializerDeserializerProvider.INSTANCE
- .getSerializerDeserializer(BuiltinType.AINT32);
+ private ISerializerDeserializer intSerde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT32);
private RangePredicate searchPredicate;
- private ILSMIndexAccessorInternal fileIndexAccessor;
+ private ILSMIndexAccessor fileIndexAccessor;
private IIndexCursor fileIndexSearchCursor;
public ExternalFileIndexAccessor(ExternalBTreeDataflowHelper indexDataflowHelper,
@@ -89,7 +87,7 @@
searchKeyTupleBuilder.reset();
searchKeyTupleBuilder.addField(intSerde, currentFileNumber);
searchKey.reset(searchKeyTupleBuilder.getFieldEndOffsets(), searchKeyTupleBuilder.getByteArray());
- searchCmp = BTreeUtils.getSearchMultiComparator(index.getComparatorFactories(), searchKey);
+ MultiComparator searchCmp = BTreeUtils.getSearchMultiComparator(index.getComparatorFactories(), searchKey);
searchPredicate = new RangePredicate(searchKey, searchKey, true, true, searchCmp, searchCmp);
// create the accessor and the cursor using the passed version
@@ -99,7 +97,7 @@
fileIndexSearchCursor = fileIndexAccessor.createSearchCursor(false);
}
- public void lookup(int fileId, ExternalFile file) throws Exception {
+ public void lookup(int fileId, ExternalFile file) throws HyracksDataException, IndexException {
// Set search parameters
currentFileNumber.setValue(fileId);
searchKeyTupleBuilder.reset();
@@ -122,8 +120,7 @@
setFile(externalFileRecord, file);
} else {
// This should never happen
- throw new RuntimeDataException(
- ErrorCode.INDEXING_EXTERNAL_FILE_INDEX_ACCESSOR_UNABLE_TO_FIND_FILE_INDEX);
+ throw new RuntimeDataException(ErrorCode.INDEXING_EXTERNAL_FILE_INDEX_ACCESSOR_UNABLE_TO_FIND_FILE_INDEX);
}
}
@@ -133,9 +130,8 @@
.getStringValue());
file.setSize(((AInt64) externalFileRecord.getValueByPos(FilesIndexDescription.EXTERNAL_FILE_SIZE_FIELD_INDEX))
.getLongValue());
- file.setLastModefiedTime((new Date(
- ((ADateTime) externalFileRecord.getValueByPos(FilesIndexDescription.EXTERNAL_FILE_MOD_DATE_FIELD_INDEX))
- .getChrononTime())));
+ file.setLastModefiedTime(new Date(((ADateTime) externalFileRecord
+ .getValueByPos(FilesIndexDescription.EXTERNAL_FILE_MOD_DATE_FIELD_INDEX)).getChrononTime()));
}
public void close() throws HyracksDataException {
diff --git a/asterixdb/asterix-lang-sqlpp/pom.xml b/asterixdb/asterix-lang-sqlpp/pom.xml
index 51a4191..6369ee2 100644
--- a/asterixdb/asterix-lang-sqlpp/pom.xml
+++ b/asterixdb/asterix-lang-sqlpp/pom.xml
@@ -23,7 +23,6 @@
<groupId>org.apache.asterix</groupId>
<version>0.9.1-SNAPSHOT</version>
</parent>
-
<licenses>
<license>
<name>Apache License, Version 2.0</name>
@@ -32,7 +31,6 @@
<comments>A business-friendly OSS license</comments>
</license>
</licenses>
-
<artifactId>asterix-lang-sqlpp</artifactId>
<build>
<plugins>
@@ -93,15 +91,9 @@
<pluginExecutions>
<pluginExecution>
<pluginExecutionFilter>
- <groupId>
- org.codehaus.mojo
- </groupId>
- <artifactId>
- javacc-maven-plugin
- </artifactId>
- <versionRange>
- [2.6,)
- </versionRange>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>javacc-maven-plugin</artifactId>
+ <versionRange>[2.6,)</versionRange>
<goals>
<goal>jjdoc</goal>
<goal>javacc</goal>
@@ -118,7 +110,6 @@
</plugins>
</pluginManagement>
</build>
-
<dependencies>
<dependency>
<groupId>org.apache.asterix</groupId>
@@ -162,5 +153,4 @@
<scope>test</scope>
</dependency>
</dependencies>
-
-</project>
+</project>
\ No newline at end of file
diff --git a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml
index 4bcdff1..0064565 100644
--- a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml
+++ b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml
@@ -11,7 +11,6 @@
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-plugin-api</artifactId>
- <version>3.3.9</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
@@ -31,6 +30,10 @@
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>algebricks-common</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.asterix</groupId>
<artifactId>asterix-external-data</artifactId>
<version>${project.version}</version>
@@ -47,10 +50,6 @@
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>algebricks-common</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.asterix</groupId>
<artifactId>asterix-common</artifactId>
<version>${project.version}</version>
@@ -97,7 +96,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
- <version>2.10</version>
<configuration>
<usedDependencies>
<usedDependency>org.apache.maven:maven-core</usedDependency>
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index 15f0a35..6cd1f8b 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -361,8 +361,8 @@
runtimeContext.getMetadataMergePolicyFactory().createMergePolicy(
GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES, dataLifecycleManager),
opTrackerProvider.getOperationTracker(appCtx), runtimeContext.getLSMIOScheduler(),
- ioOpCallbackFactory.createIOOperationCallback(), index.isPrimaryIndex(), null, null, null, null,
- true, runtimeContext.getStorageComponentProvider().getMetadataPageManagerFactory());
+ ioOpCallbackFactory.createIoOpCallback(), index.isPrimaryIndex(), null, null, null, null, true,
+ runtimeContext.getStorageComponentProvider().getMetadataPageManagerFactory());
lsmBtree.create();
resourceID = index.getResourceID();
Resource localResourceMetadata = new LSMBTreeLocalResourceMetadata(typeTraits, comparatorFactories,
@@ -396,8 +396,8 @@
runtimeContext.getMetadataMergePolicyFactory().createMergePolicy(
GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES, dataLifecycleManager),
opTrackerProvider.getOperationTracker(appCtx), runtimeContext.getLSMIOScheduler(),
- LSMBTreeIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(),
- index.isPrimaryIndex(), null, null, null, null, true,
+ LSMBTreeIOOperationCallbackFactory.INSTANCE.createIoOpCallback(), index.isPrimaryIndex(), null,
+ null, null, null, true,
runtimeContext.getStorageComponentProvider().getMetadataPageManagerFactory());
dataLifecycleManager.register(file.getRelativePath(), lsmBtree);
}
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java
index ce1e315..936f5a0 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java
@@ -83,9 +83,8 @@
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationExecutionType;
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationJobType;
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexReplicationJob;
-import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
import org.apache.hyracks.util.StorageUtil;
import org.apache.hyracks.util.StorageUtil.StorageUnit;
@@ -123,8 +122,8 @@
private ReplicationJobsProccessor replicationJobsProcessor;
private final ReplicasEventsMonitor replicationMonitor;
//dummy job used to stop ReplicationJobsProccessor thread.
- private static final IReplicationJob REPLICATION_JOB_POISON_PILL = new ReplicationJob(
- ReplicationJobType.METADATA, ReplicationOperation.REPLICATE, ReplicationExecutionType.ASYNC, null);
+ private static final IReplicationJob REPLICATION_JOB_POISON_PILL = new ReplicationJob(ReplicationJobType.METADATA,
+ ReplicationOperation.REPLICATE, ReplicationExecutionType.ASYNC, null);
//used to identify the correct IP address when the node has multiple network interfaces
private String hostIPAddressFirstOctet = null;
@@ -163,8 +162,9 @@
replicationJobsProcessor = new ReplicationJobsProccessor();
replicationMonitor = new ReplicasEventsMonitor();
- Map<String, ClusterPartition[]> nodePartitions = ((IPropertiesProvider) asterixAppRuntimeContextProvider
- .getAppContext()).getMetadataProperties().getNodePartitions();
+ Map<String, ClusterPartition[]> nodePartitions =
+ ((IPropertiesProvider) asterixAppRuntimeContextProvider.getAppContext()).getMetadataProperties()
+ .getNodePartitions();
//add list of replicas from configurations (To be read from another source e.g. Zookeeper)
Set<Replica> replicaNodes = replicationProperties.getRemoteReplicas(nodeId);
replica2PartitionsMap = new HashMap<>(replicaNodes.size());
@@ -300,8 +300,8 @@
//send LSMComponent properties
LSMComponentJob = (ILSMIndexReplicationJob) job;
LSMComponentProperties lsmCompProp = new LSMComponentProperties(LSMComponentJob, nodeId);
- requestBuffer = ReplicationProtocol.writeLSMComponentPropertiesRequest(lsmCompProp,
- requestBuffer);
+ requestBuffer =
+ ReplicationProtocol.writeLSMComponentPropertiesRequest(lsmCompProp, requestBuffer);
sendRequest(replicasSockets, requestBuffer);
}
@@ -325,10 +325,10 @@
* since this is LSM_COMPONENT REPLICATE job, the job will contain
* only the component being replicated.
*/
- ILSMComponent diskComponent = LSMComponentJob.getLSMIndexOperationContext()
+ ILSMDiskComponent diskComponent = LSMComponentJob.getLSMIndexOperationContext()
.getComponentsToBeReplicated().get(0);
- long lsnOffset = LSMIndexUtil.getComponentFileLSNOffset(
- (AbstractLSMIndex) LSMComponentJob.getLSMIndex(), diskComponent, filePath);
+ long lsnOffset = LSMIndexUtil.getComponentFileLSNOffset(LSMComponentJob.getLSMIndex(),
+ diskComponent, filePath);
asterixFileProperties.initialize(filePath, fileSize, nodeId, isLSMComponentFile,
lsnOffset, remainingFiles == 0);
} else {
@@ -338,7 +338,8 @@
requestBuffer = ReplicationProtocol.writeFileReplicationRequest(requestBuffer,
asterixFileProperties, ReplicationRequestType.REPLICATE_FILE);
- Iterator<Map.Entry<String, SocketChannel>> iterator = replicasSockets.entrySet().iterator();
+ Iterator<Map.Entry<String, SocketChannel>> iterator =
+ replicasSockets.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, SocketChannel> entry = iterator.next();
//if the remote replica is not interested in this partition, skip it.
@@ -351,8 +352,8 @@
NetworkingUtil.transferBufferToChannel(socketChannel, requestBuffer);
NetworkingUtil.sendFile(fileChannel, socketChannel);
if (asterixFileProperties.requiresAck()) {
- ReplicationRequestType responseType = waitForResponse(socketChannel,
- responseBuffer);
+ ReplicationRequestType responseType =
+ waitForResponse(socketChannel, responseBuffer);
if (responseType != ReplicationRequestType.ACK) {
throw new IOException(
"Could not receive ACK from replica " + entry.getKey());
@@ -788,10 +789,11 @@
//if got ACKs from all remote replicas, notify pending jobs if any
- if (jobCommitAcks.get(jobId).size() == replicationFactor && replicationJobsPendingAcks.containsKey(jobId)) {
+ if (jobCommitAcks.get(jobId).size() == replicationFactor
+ && replicationJobsPendingAcks.containsKey(jobId)) {
ILogRecord pendingLog = replicationJobsPendingAcks.get(jobId);
synchronized (pendingLog) {
- pendingLog.notify();
+ pendingLog.notifyAll();
}
}
}
@@ -997,17 +999,17 @@
ByteBuffer requestBuffer = ByteBuffer.allocate(INITIAL_BUFFER_SIZE);
for (String replicaId : replicaIds) {
//1. identify replica indexes with LSN less than nonSharpCheckpointTargetLSN.
- Map<Long, String> laggingIndexes = replicaResourcesManager.getLaggingReplicaIndexesId2PathMap(replicaId,
- nonSharpCheckpointTargetLSN);
+ Map<Long, String> laggingIndexes =
+ replicaResourcesManager.getLaggingReplicaIndexesId2PathMap(replicaId, nonSharpCheckpointTargetLSN);
if (laggingIndexes.size() > 0) {
//2. send request to remote replicas that have lagging indexes.
ReplicaIndexFlushRequest laggingIndexesResponse = null;
try (SocketChannel socketChannel = getReplicaSocket(replicaId)) {
- ReplicaIndexFlushRequest laggingIndexesRequest = new ReplicaIndexFlushRequest(
- laggingIndexes.keySet());
- requestBuffer = ReplicationProtocol.writeGetReplicaIndexFlushRequest(requestBuffer,
- laggingIndexesRequest);
+ ReplicaIndexFlushRequest laggingIndexesRequest =
+ new ReplicaIndexFlushRequest(laggingIndexes.keySet());
+ requestBuffer =
+ ReplicationProtocol.writeGetReplicaIndexFlushRequest(requestBuffer, laggingIndexesRequest);
NetworkingUtil.transferBufferToChannel(socketChannel, requestBuffer);
//3. remote replicas will respond with indexes that were not flushed.
@@ -1298,8 +1300,8 @@
Thread.currentThread().setName("TxnLogs Replication Listener Thread");
LOGGER.log(Level.INFO, "Started listening on socket: " + replicaSocket.socket().getRemoteSocketAddress());
- try (BufferedReader incomingResponse = new BufferedReader(
- new InputStreamReader(replicaSocket.socket().getInputStream()))) {
+ try (BufferedReader incomingResponse =
+ new BufferedReader(new InputStreamReader(replicaSocket.socket().getInputStream()))) {
while (true) {
String responseLine = incomingResponse.readLine();
if (responseLine == null) {
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
index 2d6f3c9..06ca8e6 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
@@ -41,9 +41,8 @@
public ExternalBTreeLocalResourceMetadata(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
int[] bloomFilterKeyFields, boolean isPrimary, int datasetID, int partition,
- ILSMMergePolicyFactory mergePolicyFactory,
- Map<String, String> mergePolicyProperties, ILSMOperationTrackerFactory opTrackerProvider,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties,
+ ILSMOperationTrackerFactory opTrackerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
IMetadataPageManagerFactory metadataPageManagerFactory) {
super(typeTraits, cmpFactories, bloomFilterKeyFields, isPrimary, datasetID, partition, mergePolicyFactory,
mergePolicyProperties, null, null, null, null, opTrackerProvider, ioOpCallbackFactory,
@@ -51,8 +50,8 @@
}
@Override
- public ILSMIndex createIndexInstance(INCApplicationContext appCtx,
- LocalResource resource) throws HyracksDataException {
+ public ILSMIndex createIndexInstance(INCApplicationContext appCtx, LocalResource resource)
+ throws HyracksDataException {
IAppRuntimeContext runtimeContextProvider = (IAppRuntimeContext) appCtx.getApplicationObject();
IIOManager ioManager = runtimeContextProvider.getIOManager();
FileReference file = ioManager.resolve(resource.getPath());
@@ -61,9 +60,7 @@
runtimeContextProvider.getBloomFilterFalsePositiveRate(),
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
runtimeContextProvider.getDatasetLifecycleManager()),
- opTrackerProvider.getOperationTracker(appCtx),
- runtimeContextProvider.getLSMIOScheduler(),
- ioOpCallbackFactory.createIOOperationCallback(), -1, true,
- metadataPageManagerFactory);
+ opTrackerProvider.getOperationTracker(appCtx), runtimeContextProvider.getLSMIOScheduler(),
+ ioOpCallbackFactory.createIoOpCallback(), -1, true, metadataPageManagerFactory);
}
}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
index e411f8b..1a080b2 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
@@ -50,10 +50,10 @@
private final int[] buddyBtreeFields;
public ExternalBTreeWithBuddyLocalResourceMetadata(int datasetID, int partition,
- IBinaryComparatorFactory[] btreeCmpFactories,
- ITypeTraits[] typeTraits, ILSMMergePolicyFactory mergePolicyFactory,
- Map<String, String> mergePolicyProperties, int[] buddyBtreeFields,
- ILSMOperationTrackerFactory opTrackerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ IBinaryComparatorFactory[] btreeCmpFactories, ITypeTraits[] typeTraits,
+ ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties,
+ int[] buddyBtreeFields, ILSMOperationTrackerFactory opTrackerProvider,
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory,
IMetadataPageManagerFactory metadataPageManagerFactory) {
super(datasetID, partition, null, null, null, opTrackerProvider, ioOpCallbackFactory,
metadataPageManagerFactory);
@@ -70,16 +70,12 @@
IAppRuntimeContext appRuntimeCtx = (IAppRuntimeContext) appCtx.getApplicationObject();
IIOManager ioManager = appCtx.getIoManager();
FileReference file = ioManager.resolve(resource.getPath());
- return LSMBTreeUtil.createExternalBTreeWithBuddy(
- ioManager, file, appRuntimeCtx.getBufferCache(),
+ return LSMBTreeUtil.createExternalBTreeWithBuddy(ioManager, file, appRuntimeCtx.getBufferCache(),
appRuntimeCtx.getFileMapManager(), typeTraits, btreeCmpFactories,
appRuntimeCtx.getBloomFilterFalsePositiveRate(),
- mergePolicyFactory.createMergePolicy(
- mergePolicyProperties,
+ mergePolicyFactory.createMergePolicy(mergePolicyProperties,
appRuntimeCtx.getDatasetLifecycleManager()),
- opTrackerProvider.getOperationTracker(appCtx),
- appRuntimeCtx.getLSMIOScheduler(),
- ioOpCallbackFactory.createIOOperationCallback(), buddyBtreeFields, -1,
- true, metadataPageManagerFactory);
+ opTrackerProvider.getOperationTracker(appCtx), appRuntimeCtx.getLSMIOScheduler(),
+ ioOpCallbackFactory.createIoOpCallback(), buddyBtreeFields, -1, true, metadataPageManagerFactory);
}
}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
index 3de3a58..c923046 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
@@ -59,8 +59,8 @@
}
@Override
- public ILSMIndex createIndexInstance(INCApplicationContext appCtx,
- LocalResource resource) throws HyracksDataException {
+ public ILSMIndex createIndexInstance(INCApplicationContext appCtx, LocalResource resource)
+ throws HyracksDataException {
IAppRuntimeContext runtimeContextProvider = (IAppRuntimeContext) appCtx.getApplicationObject();
IIOManager ioManager = runtimeContextProvider.getIOManager();
FileReference file = ioManager.resolve(resource.getPath());
@@ -70,10 +70,9 @@
valueProviderFactories, rtreePolicyType, runtimeContextProvider.getBloomFilterFalsePositiveRate(),
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
runtimeContextProvider.getDatasetLifecycleManager()),
- opTrackerProvider.getOperationTracker(appCtx),
- runtimeContextProvider.getLSMIOScheduler(),
- ioOpCallbackFactory.createIOOperationCallback(), linearizeCmpFactory,
- btreeFields, -1, true, isPointMBR, metadataPageManagerFactory);
+ opTrackerProvider.getOperationTracker(appCtx), runtimeContextProvider.getLSMIOScheduler(),
+ ioOpCallbackFactory.createIoOpCallback(), linearizeCmpFactory, btreeFields, -1, true, isPointMBR,
+ metadataPageManagerFactory);
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
index 0ecb396..f38711c 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
@@ -69,9 +69,9 @@
@Override
public String toString() {
- return new StringBuilder().append(" { \"").append(LSMBTreeLocalResourceMetadata.class.getName()).append(
- "\" : {").append("\"datasetId\" : ").append(datasetId()).append(", \"partition\" : ").append(
- partition()).append(" } ").append(" }").toString();
+ return new StringBuilder().append(" { \"").append(LSMBTreeLocalResourceMetadata.class.getName())
+ .append("\" : {").append("\"datasetId\" : ").append(datasetId()).append(", \"partition\" : ")
+ .append(partition()).append(" } ").append(" }").toString();
}
@Override
@@ -82,15 +82,13 @@
FileReference file = ioManager.resolve(resource.getPath());
int ioDeviceNum = Resource.getIoDeviceNum(ioManager, file.getDeviceHandle());
final IDatasetLifecycleManager datasetLifecycleManager = appRuntimeCtx.getDatasetLifecycleManager();
- return LSMBTreeUtil.createLSMTree(ioManager, datasetLifecycleManager.getVirtualBufferCaches(
- datasetId(), ioDeviceNum), file, appRuntimeCtx.getBufferCache(), appRuntimeCtx
- .getFileMapManager(),
- typeTraits, cmpFactories, bloomFilterKeyFields, appRuntimeCtx
- .getBloomFilterFalsePositiveRate(),
+ return LSMBTreeUtil.createLSMTree(ioManager,
+ datasetLifecycleManager.getVirtualBufferCaches(datasetId(), ioDeviceNum), file,
+ appRuntimeCtx.getBufferCache(), appRuntimeCtx.getFileMapManager(), typeTraits, cmpFactories,
+ bloomFilterKeyFields, appRuntimeCtx.getBloomFilterFalsePositiveRate(),
mergePolicyFactory.createMergePolicy(mergePolicyProperties, datasetLifecycleManager),
- opTrackerProvider.getOperationTracker(appCtx),
- appRuntimeCtx.getLSMIOScheduler(),
- ioOpCallbackFactory.createIOOperationCallback(), isPrimary, filterTypeTraits,
- filterCmpFactories, btreeFields, filterFields, true, metadataPageManagerFactory);
+ opTrackerProvider.getOperationTracker(appCtx), appRuntimeCtx.getLSMIOScheduler(),
+ ioOpCallbackFactory.createIoOpCallback(), isPrimary, filterTypeTraits, filterCmpFactories, btreeFields,
+ filterFields, true, metadataPageManagerFactory);
}
}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
index 097c922..acb087a 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
@@ -98,7 +98,7 @@
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
runtimeContextProvider.getDatasetLifecycleManager()),
opTrackerProvider.getOperationTracker(appCtx), runtimeContextProvider.getLSMIOScheduler(),
- ioOpCallbackFactory.createIOOperationCallback(), invertedIndexFields, filterTypeTraits,
+ ioOpCallbackFactory.createIoOpCallback(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
invertedIndexFieldsForNonBulkLoadOps, true, metadataPageManagerFactory);
} else {
@@ -109,7 +109,7 @@
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
runtimeContextProvider.getDatasetLifecycleManager()),
opTrackerProvider.getOperationTracker(appCtx), runtimeContextProvider.getLSMIOScheduler(),
- ioOpCallbackFactory.createIOOperationCallback(), invertedIndexFields, filterTypeTraits,
+ ioOpCallbackFactory.createIoOpCallback(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
invertedIndexFieldsForNonBulkLoadOps, true, metadataPageManagerFactory);
}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
index c9c5d25..500ca30 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
@@ -97,8 +97,8 @@
mergePolicyFactory.createMergePolicy(mergePolicyProperties,
runtimeContextProvider.getDatasetLifecycleManager()),
opTrackerProvider.getOperationTracker(appCtx), runtimeContextProvider.getLSMIOScheduler(),
- ioOpCallbackFactory.createIOOperationCallback(), linearizeCmpFactory, rtreeFields,
- filterTypeTraits, filterCmpFactories, filterFields, true, isPointMBR, metadataPageManagerFactory);
+ ioOpCallbackFactory.createIoOpCallback(), linearizeCmpFactory, rtreeFields, filterTypeTraits,
+ filterCmpFactories, filterFields, true, isPointMBR, metadataPageManagerFactory);
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
}
diff --git a/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/Pair.java b/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/Pair.java
index 77e6f4c..2dd71cd 100644
--- a/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/Pair.java
+++ b/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/Pair.java
@@ -19,7 +19,9 @@
package org.apache.hyracks.algebricks.common.utils;
import java.io.Serializable;
+import java.util.Objects;
+//TODO: Remove and use apache commons lang3 instead
public class Pair<T1, T2> implements Serializable {
private static final long serialVersionUID = 1L;
@@ -38,17 +40,15 @@
@Override
public boolean equals(Object obj) {
- if (!(obj instanceof Pair<?, ?>)) {
+ if (!(obj instanceof Pair)) {
return false;
- } else {
- Pair<?, ?> p = (Pair<?, ?>) obj;
- return this.first.equals(p.first) && this.second.equals(p.second);
}
+ Pair<?, ?> p = (Pair<?, ?>) obj;
+ return Objects.equals(first, p.first) && Objects.equals(second, p.second);
}
@Override
public int hashCode() {
- return first.hashCode() * 31 + second.hashCode();
+ return Objects.hash(first, second);
}
-
}
diff --git a/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/Triple.java b/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/Triple.java
index d5b188f..a30ee12 100644
--- a/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/Triple.java
+++ b/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/Triple.java
@@ -18,6 +18,9 @@
*/
package org.apache.hyracks.algebricks.common.utils;
+import java.util.Objects;
+
+//TODO: Remove and use apache commons lang3 instead
public class Triple<T1, T2, T3> {
public T1 first;
public T2 second;
@@ -36,15 +39,17 @@
@Override
public int hashCode() {
- return first.hashCode() * 31 + second.hashCode() * 15 + third.hashCode();
+ return Objects.hash(first, second, third);
}
@Override
public boolean equals(Object o) {
- if (!(o instanceof Triple<?, ?, ?>))
+ if (!(o instanceof Triple<?, ?, ?>)) {
return false;
+ }
Triple<?, ?, ?> triple = (Triple<?, ?, ?>) o;
- return first.equals(triple.first) && second.equals(triple.second) && third.equals(triple.third);
+ return Objects.equals(first, triple.first) && Objects.equals(second, triple.second)
+ && Objects.equals(third, triple.third);
}
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java
index 3759956..78e96a4 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java
@@ -23,7 +23,6 @@
import java.util.List;
import org.apache.commons.lang3.mutable.Mutable;
-
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
index 083e4d3..18da96b 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
@@ -18,8 +18,8 @@
*/
package org.apache.hyracks.algebricks.core.algebra.operators.physical;
-import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java
index aedf046..650c9e0 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java
@@ -20,8 +20,8 @@
import java.util.ArrayList;
-import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml b/hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml
index 2488c3f..386b824 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml
@@ -16,18 +16,15 @@
! specific language governing permissions and limitations
! under the License.
!-->
-
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>algebricks-rewriter</artifactId>
<name>algebricks-rewriter</name>
-
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>algebricks</artifactId>
<version>0.3.1-SNAPSHOT</version>
</parent>
-
<licenses>
<license>
<name>Apache License, Version 2.0</name>
@@ -36,11 +33,9 @@
<comments>A business-friendly OSS license</comments>
</license>
</licenses>
-
<properties>
<root.dir>${basedir}/../..</root.dir>
</properties>
-
<dependencies>
<dependency>
<groupId>org.apache.hyracks</groupId>
@@ -62,4 +57,4 @@
<artifactId>commons-lang3</artifactId>
</dependency>
</dependencies>
-</project>
+</project>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
index 1ca8bb3..963e123 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
@@ -46,6 +46,7 @@
public static final int JOB_REQUIREMENTS_EXCEED_CAPACITY = 9;
public static final int NO_SUCH_NODE = 10;
public static final int CLASS_LOADING_ISSUE = 11;
+ public static final int ILLEGAL_WRITE_AFTER_FLUSH_ATTEMPT = 12;
// Compilation error codes.
public static final int RULECOLLECTION_NOT_INSTANCE_OF_LIST = 10001;
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
index d17c9aa..6b74b54 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
@@ -20,15 +20,16 @@
# 0 --- 9999: runtime errors
# 10000 ---- 19999: compilation errors
-1=Unsupported operation %1$s in %2$s operator
-2=Error in processing tuple %1$s in a frame
-4=The file with absolute path %1$s is not within any of the current IO devices
-5=Phrase search in Full-text is not supported. An expression should include only one word
-6=Job queue is full with %1$s jobs
-7=Network address cannot be resolved -- %1$s
-8=Invalid internal input parameter
-9=Job requirement %1$s exceeds capacity %2$s
-10=Node %1$s does not exist
-11=Class loading issue: %1$s
+1 = Unsupported operation %1$s in %2$s operator
+2 = Error in processing tuple %1$s in a frame
+4 = The file with absolute path %1$s is not within any of the current IO devices
+5 = Phrase search in Full-text is not supported. An expression should include only one word
+6 = Job queue is full with %1$s jobs
+7 = Network address cannot be resolved -- %1$s
+8 = Invalid internal input parameter
+9 = Job requirement %1$s exceeds capacity %2$s
+10 = Node %1$s does not exist
+11 = Class loading issue: %1$s
+12 = Invalid attempt to write to a flushed append only metadata page
10000 = The given rule collection %1$s is not an instance of the List class.
diff --git a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/api/IValueReference.java b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/api/IValueReference.java
index e83d2ad..01567e1 100644
--- a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/api/IValueReference.java
+++ b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/api/IValueReference.java
@@ -19,9 +19,9 @@
package org.apache.hyracks.data.std.api;
public interface IValueReference {
- public byte[] getByteArray();
+ byte[] getByteArray();
- public int getStartOffset();
+ int getStartOffset();
- public int getLength();
+ int getLength();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/LongPointable.java b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/LongPointable.java
index 41af193..0f27e63 100644
--- a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/LongPointable.java
+++ b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/LongPointable.java
@@ -27,6 +27,7 @@
import org.apache.hyracks.data.std.api.IPointableFactory;
public final class LongPointable extends AbstractPointable implements IHashable, IComparable, INumeric {
+ public static final LongPointableFactory FACTORY = new LongPointableFactory();
public static final ITypeTraits TYPE_TRAITS = new ITypeTraits() {
private static final long serialVersionUID = 1L;
@@ -41,19 +42,28 @@
}
};
- public static final IPointableFactory FACTORY = new IPointableFactory() {
+ public static class LongPointableFactory implements IPointableFactory {
private static final long serialVersionUID = 1L;
+ private LongPointableFactory() {
+ }
+
@Override
- public IPointable createPointable() {
+ public LongPointable createPointable() {
return new LongPointable();
}
+ public LongPointable createPointable(long value) {
+ LongPointable pointable = new LongPointable();
+ pointable.setLong(value);
+ return pointable;
+ }
+
@Override
public ITypeTraits getTypeTraits() {
return TYPE_TRAITS;
}
- };
+ }
public static long getLong(byte[] bytes, int start) {
return (((long) (bytes[start] & 0xff)) << 56) + (((long) (bytes[start + 1] & 0xff)) << 48)
@@ -78,6 +88,11 @@
}
public void setLong(long value) {
+ if (bytes == null) {
+ start = 0;
+ length = TYPE_TRAITS.getFixedLength();
+ bytes = new byte[length];
+ }
setLong(bytes, start, value);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ArrayBackedValueStorage.java b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ArrayBackedValueStorage.java
index 33aa58b..e075f4e 100644
--- a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ArrayBackedValueStorage.java
+++ b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ArrayBackedValueStorage.java
@@ -20,6 +20,7 @@
import java.io.DataOutput;
import java.io.IOException;
+import java.util.Objects;
import org.apache.hyracks.data.std.api.IMutableValueStorage;
import org.apache.hyracks.data.std.api.IValueReference;
@@ -53,6 +54,7 @@
return data.getLength();
}
+ //TODO: don't swallow, but throw the exception
public void append(IValueReference value) {
try {
data.append(value);
@@ -69,4 +71,22 @@
public void setSize(int bytesRequired) {
data.setSize(bytesRequired);
}
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(data);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof ArrayBackedValueStorage)) {
+ return false;
+ }
+ ArrayBackedValueStorage other = (ArrayBackedValueStorage) obj;
+ return Objects.equals(data, other.data);
+ }
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/GrowableArray.java b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/GrowableArray.java
index 39c9095..994d286 100644
--- a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/GrowableArray.java
+++ b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/GrowableArray.java
@@ -71,4 +71,33 @@
public void setSize(int bytesRequired) {
baaos.setSize(bytesRequired);
}
+
+ @Override
+ public int hashCode() {
+ return 31 * baaos.getLength();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof GrowableArray)) {
+ return false;
+ }
+ GrowableArray other = (GrowableArray) obj;
+ int length = baaos.getLength();
+ if (other.baaos.getLength() != length) {
+ return false;
+ }
+ byte[] array1 = baaos.getByteArray();
+ byte[] array2 = other.baaos.getByteArray();
+ for (int i = 0; i < length; i++) {
+ if (array1[i] != array2[i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java
index b76e458..6571f7e 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobFailureTest.java
@@ -39,8 +39,8 @@
private void execTest() throws Exception {
JobSpecification spec = new JobSpecification();
- AbstractSingleActivityOperatorDescriptor sourceOpDesc = new ExceptionOnCreatePushRuntimeOperatorDescriptor(spec,
- 0, 1, new int[] { 4 }, true);
+ AbstractSingleActivityOperatorDescriptor sourceOpDesc =
+ new ExceptionOnCreatePushRuntimeOperatorDescriptor(spec, 0, 1, new int[] { 4 }, true);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sourceOpDesc, ASTERIX_IDS);
SinkOperatorDescriptor sinkOpDesc = new SinkOperatorDescriptor(spec, 1);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sinkOpDesc, ASTERIX_IDS);
@@ -53,7 +53,10 @@
e.printStackTrace();
throw e;
}
- Assert.assertTrue(ExceptionOnCreatePushRuntimeOperatorDescriptor.succeed());
+ Assert.assertTrue(
+ ExceptionOnCreatePushRuntimeOperatorDescriptor.stats()
+ + ExceptionOnCreatePushRuntimeOperatorDescriptor.succeed(),
+ ExceptionOnCreatePushRuntimeOperatorDescriptor.succeed());
// should also check the content of the different ncs
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/util/ExceptionOnCreatePushRuntimeOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/util/ExceptionOnCreatePushRuntimeOperatorDescriptor.java
index 14c644a..f814cd5 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/util/ExceptionOnCreatePushRuntimeOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/util/ExceptionOnCreatePushRuntimeOperatorDescriptor.java
@@ -136,4 +136,9 @@
}
return success;
}
+
+ public static String stats() {
+ return "Failure: CreatePushRuntime:" + createPushRuntime.get() + ", InitializeCounter:"
+ + initializeCounter.get() + ", OpenCloseCounter:" + openCloseCounter.get();
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedNettyOutputStream.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedNettyOutputStream.java
index 65cdd52..6509dcc 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedNettyOutputStream.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedNettyOutputStream.java
@@ -34,6 +34,7 @@
private final ChannelHandlerContext ctx;
private final ChunkedResponse response;
private ByteBuf buffer;
+ private boolean closed;
public ChunkedNettyOutputStream(ChannelHandlerContext ctx, int chunkSize, ChunkedResponse response) {
this.response = response;
@@ -73,13 +74,16 @@
@Override
public void close() throws IOException {
- if (response.isHeaderSent() || response.status() != HttpResponseStatus.OK) {
- flush();
- buffer.release();
- } else {
- response.fullReponse(buffer);
+ if (!closed) {
+ if (response.isHeaderSent() || response.status() != HttpResponseStatus.OK) {
+ flush();
+ buffer.release();
+ } else {
+ response.fullReponse(buffer);
+ }
+ super.close();
}
- super.close();
+ closed = true;
}
@Override
@@ -96,6 +100,7 @@
ByteBuf aBuffer = ctx.alloc().buffer(buffer.readableBytes());
aBuffer.writeBytes(buffer);
response.error(aBuffer);
+ buffer.clear();
}
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
index 82333a9..6f6722a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
@@ -22,7 +22,6 @@
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.storage.am.btree.exceptions.BTreeException;
import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import org.apache.hyracks.storage.am.btree.util.BTreeUtils;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
@@ -42,16 +41,11 @@
@Override
public ITreeIndex createIndexInstance() throws HyracksDataException {
AbstractTreeIndexOperatorDescriptor treeOpDesc = (AbstractTreeIndexOperatorDescriptor) opDesc;
- try {
- FileReference fileRef = IndexFileNameUtil.getIndexAbsoluteFileRef(treeOpDesc, ctx.getTaskAttemptId()
- .getTaskId().getPartition(), ctx.getIOManager());
- IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
- return BTreeUtils.createBTree(bufferCache, opDesc.getStorageManager()
- .getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc
- .getTreeIndexComparatorFactories(), BTreeLeafFrameType.REGULAR_NSM, fileRef,
- pageManagerFactory.createPageManager(bufferCache));
- } catch (BTreeException e) {
- throw new HyracksDataException(e);
- }
+ FileReference fileRef = IndexFileNameUtil.getIndexAbsoluteFileRef(treeOpDesc,
+ ctx.getTaskAttemptId().getTaskId().getPartition(), ctx.getIOManager());
+ IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
+ return BTreeUtils.createBTree(bufferCache, opDesc.getStorageManager().getFileMapProvider(ctx),
+ treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc.getTreeIndexComparatorFactories(),
+ BTreeLeafFrameType.REGULAR_NSM, fileRef, pageManagerFactory.createPageManager(bufferCache));
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
index 37e15b4..c7a2fee 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
@@ -21,9 +21,9 @@
import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.btree.exceptions.BTreeException;
import org.apache.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrameFactory;
import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
@@ -40,7 +40,7 @@
public class BTreeUtils {
public static BTree createBTree(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, BTreeLeafFrameType leafType,
- FileReference file, IPageManager freePageManager) throws BTreeException {
+ FileReference file, IPageManager freePageManager) throws HyracksDataException {
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
@@ -51,7 +51,7 @@
public static BTree createBTree(IBufferCache bufferCache, IPageManager freePageManager,
IFileMapProvider fileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
- BTreeLeafFrameType leafType, FileReference file) throws BTreeException {
+ BTreeLeafFrameType leafType, FileReference file) throws HyracksDataException {
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
@@ -74,7 +74,7 @@
}
public static ITreeIndexFrameFactory getLeafFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory,
- BTreeLeafFrameType leafType) throws BTreeException {
+ BTreeLeafFrameType leafType) throws HyracksDataException {
switch (leafType) {
case REGULAR_NSM: {
return new BTreeNSMLeafFrameFactory(tupleWriterFactory);
@@ -83,7 +83,7 @@
return new BTreeFieldPrefixNSMLeafFrameFactory(tupleWriterFactory);
}
default: {
- throw new BTreeException("Unknown BTreeLeafFrameType: " + leafType.toString());
+ throw new HyracksDataException("Unknown BTreeLeafFrameType: " + leafType.toString());
}
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/AppendOnlyLinkedMetadataPageManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/AppendOnlyLinkedMetadataPageManager.java
index 32d2515..1ee48f6 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/AppendOnlyLinkedMetadataPageManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/AppendOnlyLinkedMetadataPageManager.java
@@ -18,6 +18,7 @@
*/
package org.apache.hyracks.storage.am.common.freepage;
+import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IPointable;
import org.apache.hyracks.data.std.api.IValueReference;
@@ -38,9 +39,8 @@
private int metadataPage = IBufferCache.INVALID_PAGEID;
private int fileId = -1;
private final ITreeIndexMetadataFrameFactory frameFactory;
- ICachedPage confiscatedPage;
- ICachedPage filterPage;
- boolean ready = false;
+ private ICachedPage confiscatedPage;
+ private boolean ready = false;
public AppendOnlyLinkedMetadataPageManager(IBufferCache bufferCache, ITreeIndexMetadataFrameFactory frameFactory) {
this.bufferCache = bufferCache;
@@ -299,6 +299,9 @@
@Override
public void put(ITreeIndexMetadataFrame frame, IValueReference key, IValueReference value)
throws HyracksDataException {
+ if (confiscatedPage == null) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_WRITE_AFTER_FLUSH_ATTEMPT);
+ }
confiscatedPage.acquireWriteLatch();
try {
frame.setPage(confiscatedPage);
@@ -309,8 +312,9 @@
}
private ICachedPage pinPage() throws HyracksDataException {
- return confiscatedPage == null ? bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, getMetadataPageId()),
- false) : confiscatedPage;
+ return confiscatedPage == null
+ ? bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, getMetadataPageId()), false)
+ : confiscatedPage;
}
private void unpinPage(ICachedPage page) throws HyracksDataException {
@@ -342,8 +346,7 @@
frame.setPage(page);
int inPageOffset = frame.getOffset(key);
return inPageOffset >= 0 ? ((long) pageId * bufferCache.getPageSizeWithHeader()) + frame.getOffset(key)
- + IBufferCache.RESERVED_HEADER_BYTES
- : -1L;
+ + IBufferCache.RESERVED_HEADER_BYTES : -1L;
} finally {
page.releaseReadLatch();
unpinPage(page);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
index 0d8f32e..d471956 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
@@ -72,7 +72,7 @@
.getTreeIndexComparatorFactories(), treeOpDesc.getTreeIndexBloomFilterKeyFields(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx.getJobletContext()
.getApplicationContext()), ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback(), getVersion(), durable,
+ ioOpCallbackFactory.createIoOpCallback(), getVersion(), durable,
(IMetadataPageManagerFactory) opDesc.getPageManagerFactory());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
index b4d2797..1e60690 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
@@ -83,7 +83,7 @@
opDesc.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
treeOpDesc.getTreeIndexComparatorFactories(), bloomFilterFalsePositiveRate, mergePolicy,
opTrackerFactory.getOperationTracker(ctx.getJobletContext().getApplicationContext()), ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback(), buddyBtreeFields, version, durable,
+ ioOpCallbackFactory.createIoOpCallback(), buddyBtreeFields, version, durable,
(IMetadataPageManagerFactory) opDesc.getPageManagerFactory());
}
public int getTargetVersion() {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
index 5f8c951..0346624 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
@@ -78,7 +78,7 @@
treeOpDesc.getTreeIndexComparatorFactories(), treeOpDesc.getTreeIndexBloomFilterKeyFields(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx.getJobletContext()
.getApplicationContext()), ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback(), needKeyDupCheck, filterTypeTraits, filterCmpFactories,
+ ioOpCallbackFactory.createIoOpCallback(), needKeyDupCheck, filterTypeTraits, filterCmpFactories,
btreeFields, filterFields, durable, (IMetadataPageManagerFactory) opDesc.getPageManagerFactory());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index c79205e..cc4bfb9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -49,11 +49,12 @@
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeRefrencingTupleWriterFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -81,7 +82,7 @@
private final LSMBTreeDiskComponentFactory transactionComponentFactory;
// A second disk component list that will be used when a transaction is
// committed and will be seen by subsequent accessors
- private final List<ILSMComponent> secondDiskComponents;
+ private final List<ILSMDiskComponent> secondDiskComponents;
// A pointer that points to the current most recent list (either
// diskComponents = 0, or secondDiskComponents = 1). It starts with -1 to
// indicate first time activation
@@ -91,45 +92,30 @@
//TODO remove BloomFilter from external dataset's secondary LSMBTree index
public ExternalBTree(IIOManager ioManager, ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory insertLeafFrameFactory,
- ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager,
- TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory,
- BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
- IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, TreeIndexFactory<BTree> transactionBTreeFactory, int version,
- boolean durable) {
- super(ioManager, insertLeafFrameFactory, deleteLeafFrameFactory, fileManager,
- diskBTreeFactory,
- bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, fieldCount,
- cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, false, durable);
- this.transactionComponentFactory = new LSMBTreeDiskComponentFactory(transactionBTreeFactory, bloomFilterFactory,
- null);
+ ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory,
+ ILSMIndexFileManager fileManager, TreeIndexFactory<BTree> diskBTreeFactory,
+ TreeIndexFactory<BTree> bulkLoadBTreeFactory, BloomFilterFactory bloomFilterFactory,
+ double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount,
+ IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ TreeIndexFactory<BTree> transactionBTreeFactory, int version, boolean durable) {
+ super(ioManager, insertLeafFrameFactory, deleteLeafFrameFactory, fileManager, diskBTreeFactory,
+ bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider,
+ fieldCount, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, false, durable);
+ this.transactionComponentFactory =
+ new LSMBTreeDiskComponentFactory(transactionBTreeFactory, bloomFilterFactory, null);
this.secondDiskComponents = new LinkedList<>();
this.interiorFrameFactory = interiorFrameFactory;
this.version = version;
}
- // This method is used to create a target for a bulk modify operation. This
- // component must then be either committed or deleted
- private ILSMComponent createTransactionTarget() throws HyracksDataException, IndexException {
- LSMComponentFileReferences componentFileRefs;
- try {
- componentFileRefs = fileManager.getNewTransactionFileReference();
- } catch (IOException e) {
- throw new HyracksDataException("Failed to create transaction components", e);
- }
- return createDiskComponent(transactionComponentFactory, componentFileRefs.getInsertIndexFileReference(),
- componentFileRefs.getBloomFilterFileReference(), true);
- }
-
// The subsume merged components is overridden to account for:
// Maintaining two versions of the index
@Override
- public void subsumeMergedComponents(ILSMComponent newComponent, List<ILSMComponent> mergedComponents)
+ public void subsumeMergedComponents(ILSMDiskComponent newComponent, List<ILSMComponent> mergedComponents)
throws HyracksDataException {
- List<ILSMComponent> newerList;
- List<ILSMComponent> olderList;
+ List<ILSMDiskComponent> newerList;
+ List<ILSMDiskComponent> olderList;
if (version == 0) {
newerList = diskComponents;
olderList = secondDiskComponents;
@@ -155,7 +141,7 @@
// is needed.
// It only needs to return the newer list
@Override
- public List<ILSMComponent> getImmutableComponents() {
+ public List<ILSMDiskComponent> getImmutableComponents() {
if (version == 0) {
return diskComponents;
} else {
@@ -204,9 +190,9 @@
BTree lastBTree = ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1)).getBTree();
FileReference firstFile = firstBTree.getFileReference();
FileReference lastFile = lastBTree.getFileReference();
- LSMComponentFileReferences relMergeFileRefs = fileManager
- .getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
- ILSMIndexAccessorInternal accessor = new LSMBTreeAccessor(lsmHarness, opCtx);
+ LSMComponentFileReferences relMergeFileRefs =
+ fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
+ ILSMIndexAccessor accessor = new LSMBTreeAccessor(lsmHarness, opCtx);
ioScheduler.scheduleOperation(new LSMBTreeMergeOperation(accessor, mergingComponents, cursor,
relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(),
callback, fileManager.getBaseDir()));
@@ -222,7 +208,7 @@
// This function in an instance of this index is only used after a bulk load
// is successful
@Override
- public void addComponent(ILSMComponent c) throws HyracksDataException {
+ public void addDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
if (version == 0) {
diskComponents.add(0, c);
} else if (version == 1) {
@@ -232,10 +218,10 @@
// This function is used when a new component is to be committed.
@Override
- public void commitTransactionDiskComponent(ILSMComponent newComponent) throws HyracksDataException {
+ public void commitTransactionDiskComponent(ILSMDiskComponent newComponent) throws HyracksDataException {
// determine which list is the new one and flip the pointer
- List<ILSMComponent> newerList;
- List<ILSMComponent> olderList;
+ List<ILSMDiskComponent> newerList;
+ List<ILSMDiskComponent> olderList;
if (version == 0) {
newerList = diskComponents;
olderList = secondDiskComponents;
@@ -409,7 +395,8 @@
// Not supported
@Override
- public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
+ public void modify(IIndexOperationContext ictx, ITupleReference tuple)
+ throws HyracksDataException, IndexException {
throw new UnsupportedOperationException("tuple modify not supported in LSM-Disk-Only-BTree");
}
@@ -422,7 +409,7 @@
// Not supported
@Override
- public ILSMComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
throw new UnsupportedOperationException("flush not supported in LSM-Disk-Only-BTree");
}
@@ -430,7 +417,7 @@
@Override
public void getOperationalComponents(ILSMIndexOperationContext ctx) {
List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
- List<ILSMComponent> immutableComponents;
+ List<ILSMDiskComponent> immutableComponents;
// Identify current list in case of a merge
if (version == 0) {
immutableComponents = diskComponents;
@@ -486,7 +473,7 @@
// The bulk loader used for both initial loading and transaction
// modifications
public class LSMTwoPCBTreeBulkLoader implements IIndexBulkLoader, ITwoPCIndexBulkLoader {
- private final ILSMComponent component;
+ private final ILSMDiskComponent component;
private final BTreeBulkLoader bulkLoader;
private final IIndexBulkLoader builder;
private boolean cleanedUpArtifacts = false;
@@ -516,14 +503,14 @@
}
}
- frameTupleWriterFactory = ((LSMBTreeDiskComponent) component).getBTree().getLeafFrameFactory()
- .getTupleWriterFactory();
+ frameTupleWriterFactory =
+ ((LSMBTreeDiskComponent) component).getBTree().getLeafFrameFactory().getTupleWriterFactory();
bulkLoader = (BTreeBulkLoader) ((LSMBTreeDiskComponent) component).getBTree().createBulkLoader(fillFactor,
verifyInput, numElementsHint, false);
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
- BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
- bloomFilterFalsePositiveRate);
+ BloomFilterSpecification bloomFilterSpec =
+ BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate);
builder = ((LSMBTreeDiskComponent) component).getBloomFilter().createBuilder(numElementsHint,
bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
}
@@ -619,6 +606,19 @@
// Do nothing
}
}
+
+ // This method is used to create a target for a bulk modify operation. This
+ // component must then be either committed or deleted
+ private ILSMDiskComponent createTransactionTarget() throws HyracksDataException, IndexException {
+ LSMComponentFileReferences componentFileRefs;
+ try {
+ componentFileRefs = fileManager.getNewTransactionFileReference();
+ } catch (IOException e) {
+ throw new HyracksDataException("Failed to create transaction components", e);
+ }
+ return createDiskComponent(transactionComponentFactory, componentFileRefs.getInsertIndexFileReference(),
+ componentFileRefs.getBloomFilterFileReference(), true);
+ }
}
@Override
@@ -628,13 +628,13 @@
// The accessor for disk only indexes don't use modification callback and always carry the target index version with them
@Override
- public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
+ public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
return new LSMBTreeAccessor(lsmHarness, createOpContext(searchCallback, version));
}
@Override
- public ILSMIndexAccessorInternal createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion)
+ public ILSMIndexAccessor createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion)
throws HyracksDataException {
return new LSMBTreeAccessor(lsmHarness, createOpContext(searchCallback, targetIndexVersion));
}
@@ -670,12 +670,12 @@
}
@Override
- public List<ILSMComponent> getFirstComponentList() {
+ public List<ILSMDiskComponent> getFirstComponentList() {
return diskComponents;
}
@Override
- public List<ILSMComponent> getSecondComponentList() {
+ public List<ILSMDiskComponent> getSecondComponentList() {
return secondDiskComponents;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeOpContext.java
index 29fedef..5f867db 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeOpContext.java
@@ -29,6 +29,7 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
@@ -43,8 +44,8 @@
public final MultiComparator bloomFilterCmp;
public final ISearchOperationCallback searchCallback;
private final List<ILSMComponent> componentHolder;
- private final List<ILSMComponent> componentsToBeMerged;
- private final List<ILSMComponent> componentsToBeReplicated;
+ private final List<ILSMDiskComponent> componentsToBeMerged;
+ private final List<ILSMDiskComponent> componentsToBeReplicated;
private final int targetIndexVersion;
public ISearchPredicate searchPredicate;
public LSMBTreeCursorInitialState searchInitialState;
@@ -69,9 +70,9 @@
if (deleteLeafFrame != null && this.cmp != null) {
deleteLeafFrame.setMultiComparator(cmp);
}
- this.componentHolder = new LinkedList<ILSMComponent>();
- this.componentsToBeMerged = new LinkedList<ILSMComponent>();
- this.componentsToBeReplicated = new LinkedList<ILSMComponent>();
+ this.componentHolder = new LinkedList<>();
+ this.componentsToBeMerged = new LinkedList<>();
+ this.componentsToBeReplicated = new LinkedList<>();
this.searchCallback = searchCallback;
this.targetIndexVersion = targetIndexVersion;
searchInitialState = new LSMBTreeCursorInitialState(insertLeafFrameFactory, cmp, bloomFilterCmp, lsmHarness,
@@ -119,7 +120,7 @@
}
@Override
- public List<ILSMComponent> getComponentsToBeMerged() {
+ public List<ILSMDiskComponent> getComponentsToBeMerged() {
return componentsToBeMerged;
}
@@ -139,7 +140,7 @@
}
@Override
- public List<ILSMComponent> getComponentsToBeReplicated() {
+ public List<ILSMDiskComponent> getComponentsToBeReplicated() {
return componentsToBeReplicated;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index 8e29ef2..dc28db4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -52,13 +52,14 @@
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -92,7 +93,7 @@
// A second disk component list that will be used when a transaction is
// committed and will be seen by subsequent accessors
- private final List<ILSMComponent> secondDiskComponents;
+ private final List<ILSMDiskComponent> secondDiskComponents;
private int version = -1;
public ExternalBTreeWithBuddy(IIOManager ioManager, ITreeIndexFrameFactory btreeInteriorFrameFactory,
@@ -105,18 +106,17 @@
IBinaryComparatorFactory[] btreeCmpFactories, IBinaryComparatorFactory[] buddyBtreeCmpFactories,
int[] buddyBTreeFields, int version, boolean durable) {
super(ioManager, diskBufferCache, fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy,
- opTracker,
- ioScheduler, ioOpCallback, durable);
+ opTracker, ioScheduler, ioOpCallback, durable);
this.btreeCmpFactories = btreeCmpFactories;
this.buddyBtreeCmpFactories = buddyBtreeCmpFactories;
this.buddyBTreeFields = buddyBTreeFields;
this.btreeInteriorFrameFactory = btreeInteriorFrameFactory;
this.btreeLeafFrameFactory = btreeLeafFrameFactory;
this.buddyBtreeLeafFrameFactory = buddyBtreeLeafFrameFactory;
- this.componentFactory = new LSMBTreeWithBuddyDiskComponentFactory(copyBtreeFactory, buddyBtreeFactory,
- bloomFilterFactory);
- this.bulkComponentFactory = new LSMBTreeWithBuddyDiskComponentFactory(bulkLoadBTreeFactory, buddyBtreeFactory,
- bloomFilterFactory);
+ this.componentFactory =
+ new LSMBTreeWithBuddyDiskComponentFactory(copyBtreeFactory, buddyBtreeFactory, bloomFilterFactory);
+ this.bulkComponentFactory =
+ new LSMBTreeWithBuddyDiskComponentFactory(bulkLoadBTreeFactory, buddyBtreeFactory, bloomFilterFactory);
this.secondDiskComponents = new LinkedList<>();
this.version = version;
}
@@ -253,7 +253,7 @@
}
@Override
- public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
+ public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) throws HyracksDataException {
return new LSMBTreeWithBuddyAccessor(lsmHarness, createOpContext(searchCallback, version));
}
@@ -271,10 +271,10 @@
// The subsume merged components is overridden to account for:
// Maintaining two versions of the index
@Override
- public void subsumeMergedComponents(ILSMComponent newComponent, List<ILSMComponent> mergedComponents)
+ public void subsumeMergedComponents(ILSMDiskComponent newComponent, List<ILSMComponent> mergedComponents)
throws HyracksDataException {
- List<ILSMComponent> newerList;
- List<ILSMComponent> olderList;
+ List<ILSMDiskComponent> newerList;
+ List<ILSMDiskComponent> olderList;
if (version == 0) {
newerList = diskComponents;
olderList = secondDiskComponents;
@@ -320,7 +320,8 @@
}
@Override
- public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
+ public void modify(IIndexOperationContext ictx, ITupleReference tuple)
+ throws HyracksDataException, IndexException {
throw new UnsupportedOperationException("tuple modify not supported in LSM-Disk-Only-BTree");
}
@@ -340,7 +341,7 @@
}
@Override
- public ILSMComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
throw new UnsupportedOperationException("flush not supported in LSM-Disk-Only-BTree");
}
@@ -351,8 +352,8 @@
.getBTree();
FileReference firstFile = firstTree.getFileReference();
FileReference lastFile = lastTree.getFileReference();
- LSMComponentFileReferences fileRefs = fileManager.getRelMergeFileReference(firstFile.getFile().getName(),
- lastFile.getFile().getName());
+ LSMComponentFileReferences fileRefs =
+ fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
return fileRefs;
}
@@ -364,7 +365,7 @@
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
ITreeIndexCursor cursor = new LSMBTreeWithBuddySortedCursor(bctx, buddyBTreeFields);
LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
- ILSMIndexAccessorInternal accessor = new LSMBTreeWithBuddyAccessor(lsmHarness, bctx);
+ ILSMIndexAccessor accessor = new LSMBTreeWithBuddyAccessor(lsmHarness, bctx);
// Since we have two lists of components, to tell whether we need to
// keep deleted tuples, we need to know
@@ -384,14 +385,15 @@
}
// This method creates the appropriate opContext for the targeted version
- public ExternalBTreeWithBuddyOpContext createOpContext(ISearchOperationCallback searchCallback, int targetVersion) {
+ public ExternalBTreeWithBuddyOpContext createOpContext(ISearchOperationCallback searchCallback,
+ int targetVersion) {
return new ExternalBTreeWithBuddyOpContext(btreeCmpFactories, buddyBtreeCmpFactories, searchCallback,
targetVersion, lsmHarness, btreeInteriorFrameFactory, btreeLeafFrameFactory,
buddyBtreeLeafFrameFactory);
}
@Override
- public ILSMComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
LSMBTreeWithBuddyMergeOperation mergeOp = (LSMBTreeWithBuddyMergeOperation) operation;
ITreeIndexCursor cursor = mergeOp.getCursor();
ISearchPredicate btreeSearchPred = new RangePredicate(null, null, true, true, null, null);
@@ -399,9 +401,9 @@
opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
search(opCtx, cursor, btreeSearchPred);
- LSMBTreeWithBuddyDiskComponent mergedComponent = createDiskComponent(componentFactory,
- mergeOp.getBTreeMergeTarget(), mergeOp.getBuddyBTreeMergeTarget(), mergeOp.getBloomFilterMergeTarget(),
- true);
+ LSMBTreeWithBuddyDiskComponent mergedComponent =
+ createDiskComponent(componentFactory, mergeOp.getBTreeMergeTarget(),
+ mergeOp.getBuddyBTreeMergeTarget(), mergeOp.getBloomFilterMergeTarget(), true);
// In case we must keep the deleted-keys BuddyBTrees, then they must be
// merged *before* merging the b-trees so that
@@ -419,13 +421,13 @@
long numElements = 0L;
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
- numElements += ((LSMBTreeWithBuddyDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter()
- .getNumElements();
+ numElements += ((LSMBTreeWithBuddyDiskComponent) mergeOp.getMergingComponents().get(i))
+ .getBloomFilter().getNumElements();
}
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
- BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
- bloomFilterFalsePositiveRate);
+ BloomFilterSpecification bloomFilterSpec =
+ BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate);
IIndexBulkLoader builder = mergedComponent.getBloomFilter().createBuilder(numElements,
bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
@@ -460,7 +462,7 @@
@Override
public void getOperationalComponents(ILSMIndexOperationContext ctx) {
List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
- List<ILSMComponent> immutableComponents;
+ List<ILSMDiskComponent> immutableComponents;
// Identify current list in case of a merge
if (version == 0) {
immutableComponents = diskComponents;
@@ -498,7 +500,7 @@
}
@Override
- public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException {
+ public void markAsValid(ILSMDiskComponent lsmComponent) throws HyracksDataException {
LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) lsmComponent;
// Flush the bloom filter first.
markAsValidInternal(component.getBTree().getBufferCache(), component.getBloomFilter());
@@ -509,11 +511,11 @@
// This function is used when a new component is to be committed -- is
// called by the harness.
@Override
- public void commitTransactionDiskComponent(ILSMComponent newComponent) throws HyracksDataException {
+ public void commitTransactionDiskComponent(ILSMDiskComponent newComponent) throws HyracksDataException {
// determine which list is the new one and flip the pointer
- List<ILSMComponent> newerList;
- List<ILSMComponent> olderList;
+ List<ILSMDiskComponent> newerList;
+ List<ILSMDiskComponent> olderList;
if (version == 0) {
newerList = diskComponents;
olderList = secondDiskComponents;
@@ -609,12 +611,12 @@
return btreeCmpFactories;
}
- private LSMBTreeWithBuddyDiskComponent createDiskComponent(ILSMComponentFactory factory,
+ private LSMBTreeWithBuddyDiskComponent createDiskComponent(ILSMDiskComponentFactory factory,
FileReference insertFileRef, FileReference deleteFileRef, FileReference bloomFilterFileRef,
boolean createComponent) throws HyracksDataException, IndexException {
// Create new instance.
- LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) factory.createLSMComponentInstance(
- new LSMComponentFileReferences(insertFileRef, deleteFileRef, bloomFilterFileRef));
+ LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) factory
+ .createComponent(new LSMComponentFileReferences(insertFileRef, deleteFileRef, bloomFilterFileRef));
if (createComponent) {
component.getBTree().create();
component.getBuddyBTree().create();
@@ -654,7 +656,7 @@
// The bulk loader used for both initial loading and transaction
// modifications
public class LSMTwoPCBTreeWithBuddyBulkLoader implements IIndexBulkLoader, ITwoPCIndexBulkLoader {
- private final ILSMComponent component;
+ private final ILSMDiskComponent component;
private final BTreeBulkLoader btreeBulkLoader;
private final BTreeBulkLoader buddyBtreeBulkLoader;
private final IIndexBulkLoader builder;
@@ -690,8 +692,8 @@
buddyBtreeBulkLoader = (BTreeBulkLoader) ((LSMBTreeWithBuddyDiskComponent) component).getBuddyBTree()
.createBulkLoader(fillFactor, verifyInput, numElementsHint, false);
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
- BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
- bloomFilterFalsePositiveRate);
+ BloomFilterSpecification bloomFilterSpec =
+ BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate);
builder = ((LSMBTreeWithBuddyDiskComponent) component).getBloomFilter().createBuilder(numElementsHint,
bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
}
@@ -783,29 +785,31 @@
} catch (Exception e) {
}
}
+
+ // This method is used to create a target for a bulk modify operation. This
+ // component must then eventually be either committed or deleted
+ private ILSMDiskComponent createTransactionTarget() throws HyracksDataException, IndexException {
+ LSMComponentFileReferences componentFileRefs;
+ try {
+ componentFileRefs = fileManager.getNewTransactionFileReference();
+ } catch (IOException e) {
+ throw new HyracksDataException("Failed to create transaction components", e);
+ }
+ return createDiskComponent(bulkComponentFactory, componentFileRefs.getInsertIndexFileReference(),
+ componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(),
+ true);
+ }
}
- protected ILSMComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
+ protected ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
return createDiskComponent(bulkComponentFactory, componentFileRefs.getInsertIndexFileReference(),
- componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true);
- }
-
- // This method is used to create a target for a bulk modify operation. This
- // component must then eventually be either committed or deleted
- private ILSMComponent createTransactionTarget() throws HyracksDataException, IndexException {
- LSMComponentFileReferences componentFileRefs;
- try {
- componentFileRefs = fileManager.getNewTransactionFileReference();
- } catch (IOException e) {
- throw new HyracksDataException("Failed to create transaction components", e);
- }
- return createDiskComponent(bulkComponentFactory, componentFileRefs.getInsertIndexFileReference(),
- componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true);
+ componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(),
+ true);
}
@Override
- public ILSMIndexAccessorInternal createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion)
+ public ILSMIndexAccessor createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion)
throws HyracksDataException {
return new LSMBTreeWithBuddyAccessor(lsmHarness, createOpContext(searchCallback, targetIndexVersion));
}
@@ -814,7 +818,7 @@
// is successful
// it will therefore add the component to the first list and enter it.
@Override
- public void addComponent(ILSMComponent c) throws HyracksDataException {
+ public void addDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
if (version == 0) {
diskComponents.add(0, c);
} else if (version == 1) {
@@ -828,12 +832,12 @@
}
@Override
- public List<ILSMComponent> getFirstComponentList() {
+ public List<ILSMDiskComponent> getFirstComponentList() {
return diskComponents;
}
@Override
- public List<ILSMComponent> getSecondComponentList() {
+ public List<ILSMDiskComponent> getSecondComponentList() {
return secondDiskComponents;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddyOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddyOpContext.java
index c44f529..051b655 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddyOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddyOpContext.java
@@ -29,6 +29,7 @@
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
@@ -38,8 +39,8 @@
private MultiComparator bTreeCmp;
private MultiComparator buddyBTreeCmp;
public final List<ILSMComponent> componentHolder;
- private final List<ILSMComponent> componentsToBeMerged;
- private final List<ILSMComponent> componentsToBeReplicated;
+ private final List<ILSMDiskComponent> componentsToBeMerged;
+ private final List<ILSMDiskComponent> componentsToBeReplicated;
public final ISearchOperationCallback searchCallback;
private final int targetIndexVersion;
public ISearchPredicate searchPredicate;
@@ -49,9 +50,9 @@
IBinaryComparatorFactory[] buddyBtreeCmpFactories, ISearchOperationCallback searchCallback,
int targetIndexVersion, ILSMHarness lsmHarness, ITreeIndexFrameFactory btreeInteriorFrameFactory,
ITreeIndexFrameFactory btreeLeafFrameFactory, ITreeIndexFrameFactory buddyBtreeLeafFrameFactory) {
- this.componentHolder = new LinkedList<ILSMComponent>();
- this.componentsToBeMerged = new LinkedList<ILSMComponent>();
- this.componentsToBeReplicated = new LinkedList<ILSMComponent>();
+ this.componentHolder = new LinkedList<>();
+ this.componentsToBeMerged = new LinkedList<>();
+ this.componentsToBeReplicated = new LinkedList<>();
this.searchCallback = searchCallback;
this.targetIndexVersion = targetIndexVersion;
this.bTreeCmp = MultiComparator.create(btreeCmpFactories);
@@ -110,7 +111,7 @@
}
@Override
- public List<ILSMComponent> getComponentsToBeMerged() {
+ public List<ILSMDiskComponent> getComponentsToBeMerged() {
return componentsToBeMerged;
}
@@ -129,7 +130,7 @@
}
@Override
- public List<ILSMComponent> getComponentsToBeReplicated() {
+ public List<ILSMDiskComponent> getComponentsToBeReplicated() {
return componentsToBeReplicated;
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index b77cc15..53b8405 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -57,6 +57,7 @@
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
@@ -65,7 +66,6 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -102,16 +102,15 @@
private final boolean hasBloomFilter;
public LSMBTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory,
- ILSMIndexFileManager fileManager, TreeIndexFactory<BTree> diskBTreeFactory,
- TreeIndexFactory<BTree> bulkLoadBTreeFactory, BloomFilterFactory bloomFilterFactory,
- ILSMComponentFilterFactory filterFactory, ILSMComponentFilterFrameFactory filterFrameFactory,
- LSMComponentFilterManager filterManager, double bloomFilterFalsePositiveRate,
- IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, int[] btreeFields, int[] filterFields,
- boolean durable) throws HyracksDataException {
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory,
+ ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager,
+ TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory,
+ BloomFilterFactory bloomFilterFactory, ILSMComponentFilterFactory filterFactory,
+ ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
+ double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount,
+ IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck,
+ int[] btreeFields, int[] filterFields, boolean durable) throws HyracksDataException {
super(ioManager, virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
filterManager, filterFields, durable);
@@ -126,21 +125,20 @@
insertLeafFrameFactory, cmpFactories, fieldCount,
ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_" + i)),
virtualBufferCache, i == 0 ? true : false,
- filterFactory == null ? null : filterFactory.createLSMComponentFilter());
+ filterFactory == null ? null : filterFactory.createFilter());
memoryComponents.add(mutableComponent);
++i;
}
componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, filterFactory);
- bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory,
- filterFactory);
+ bulkLoadComponentFactory =
+ new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory, filterFactory);
this.needKeyDupCheck = needKeyDupCheck;
this.btreeFields = btreeFields;
this.hasBloomFilter = needKeyDupCheck;
}
// Without memory components
- public LSMBTree(IIOManager ioManager,
- ITreeIndexFrameFactory insertLeafFrameFactory,
+ public LSMBTree(IIOManager ioManager, ITreeIndexFrameFactory insertLeafFrameFactory,
ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager,
TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory,
BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
@@ -148,8 +146,7 @@
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, boolean durable) {
super(ioManager, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
- bloomFilterFalsePositiveRate,
- mergePolicy, opTracker, ioScheduler, ioOpCallback, durable);
+ bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, durable);
this.insertLeafFrameFactory = insertLeafFrameFactory;
this.deleteLeafFrameFactory = deleteLeafFrameFactory;
this.cmpFactories = cmpFactories;
@@ -177,7 +174,7 @@
if (isActivated) {
throw new HyracksDataException("Failed to activate the index since it is already activated.");
}
- List<ILSMComponent> immutableComponents = diskComponents;
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
immutableComponents.clear();
List<LSMComponentFileReferences> validFileReferences;
try {
@@ -188,9 +185,9 @@
for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
LSMBTreeDiskComponent component;
try {
- component = createDiskComponent(componentFactory, lsmComonentFileReference
- .getInsertIndexFileReference(),
- lsmComonentFileReference.getBloomFilterFileReference(), false);
+ component =
+ createDiskComponent(componentFactory, lsmComonentFileReference.getInsertIndexFileReference(),
+ lsmComonentFileReference.getBloomFilterFileReference(), false);
} catch (IndexException e) {
throw new HyracksDataException(e);
}
@@ -207,7 +204,8 @@
if (flushOnExit) {
BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
- ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ ILSMIndexAccessor accessor =
+ createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
accessor.scheduleFlush(cb);
try {
cb.waitForIO();
@@ -216,8 +214,8 @@
}
}
- List<ILSMComponent> immutableComponents = diskComponents;
- for (ILSMComponent c : immutableComponents) {
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
+ for (ILSMDiskComponent c : immutableComponents) {
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
component.getBTree().deactivateCloseHandle();
if (hasBloomFilter) {
@@ -239,8 +237,8 @@
throw new HyracksDataException("Failed to destroy the index since it is activated.");
}
- List<ILSMComponent> immutableComponents = diskComponents;
- for (ILSMComponent c : immutableComponents) {
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
+ for (ILSMDiskComponent c : immutableComponents) {
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
component.getBTree().destroy();
if (hasBloomFilter) {
@@ -261,7 +259,7 @@
}
clearMemoryComponents();
- List<ILSMComponent> immutableComponents = diskComponents;
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
for (ILSMComponent c : immutableComponents) {
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
if (hasBloomFilter) {
@@ -278,7 +276,7 @@
@Override
public void getOperationalComponents(ILSMIndexOperationContext ctx) throws HyracksDataException {
- List<ILSMComponent> immutableComponents = diskComponents;
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
int cmc = currentMutableComponentId.get();
ctx.setCurrentMutableComponentId(cmc);
@@ -328,7 +326,8 @@
}
@Override
- public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
+ public void modify(IIndexOperationContext ictx, ITupleReference tuple)
+ throws HyracksDataException, IndexException {
LSMBTreeOpContext ctx = (LSMBTreeOpContext) ictx;
ITupleReference indexTuple;
@@ -425,14 +424,14 @@
assert ctx.getComponentHolder().size() == 1;
opCtx.setOperation(IndexOperation.FLUSH);
opCtx.getComponentHolder().add(flushingComponent);
- ILSMIndexAccessorInternal flushAccessor = new LSMBTreeAccessor(lsmHarness, opCtx);
+ ILSMIndexAccessor flushAccessor = new LSMBTreeAccessor(lsmHarness, opCtx);
ioScheduler.scheduleOperation(new LSMBTreeFlushOperation(flushAccessor, flushingComponent,
componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getBloomFilterFileReference(),
- callback, fileManager.getBaseDir(), flushingComponent.getMostRecentMarkerLSN()));
+ callback, fileManager.getBaseDir()));
}
@Override
- public ILSMComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
LSMBTreeFlushOperation flushOp = (LSMBTreeFlushOperation) operation;
LSMBTreeMemoryComponent flushingComponent = (LSMBTreeMemoryComponent) flushOp.getFlushingComponent();
IIndexAccessor accessor = flushingComponent.getBTree().createAccessor(NoOpOperationCallback.INSTANCE,
@@ -490,10 +489,17 @@
List<ITupleReference> filterTuples = new ArrayList<>();
filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
- filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilterInfo(component.getLSMComponentFilter(), component.getBTree());
+ filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
+ filterManager.writeFilter(component.getLSMComponentFilter(), component.getBTree());
}
- component.setMostRecentMarkerLSN(flushOp.getPrevMarkerLSN());
+ // Write metadata from memory component to disk
+ // Q. what about the merge operation? how do we resolve conflicts
+ // A. Through providing an appropriate ILSMIOOperationCallback
+ // Must not reset the metadata before the flush is completed
+ // Use the copy of the metadata in the opContext
+ // TODO This code should be in the callback and not in the index
+ flushingComponent.getMetadata().copy(component.getMetadata());
+
bulkLoader.end();
return component;
}
@@ -514,16 +520,16 @@
BTree lastBTree = ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1)).getBTree();
FileReference firstFile = firstBTree.getFileReference();
FileReference lastFile = lastBTree.getFileReference();
- LSMComponentFileReferences relMergeFileRefs = fileManager.getRelMergeFileReference(firstFile.getFile()
- .getName(), lastFile.getFile().getName());
- ILSMIndexAccessorInternal accessor = new LSMBTreeAccessor(lsmHarness, opCtx);
+ LSMComponentFileReferences relMergeFileRefs =
+ fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
+ ILSMIndexAccessor accessor = new LSMBTreeAccessor(lsmHarness, opCtx);
ioScheduler.scheduleOperation(new LSMBTreeMergeOperation(accessor, mergingComponents, cursor,
relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(),
callback, fileManager.getBaseDir()));
}
@Override
- public ILSMComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
LSMBTreeMergeOperation mergeOp = (LSMBTreeMergeOperation) operation;
ITreeIndexCursor cursor = mergeOp.getCursor();
RangePredicate rangePred = new RangePredicate(null, null, true, true, null, null);
@@ -572,14 +578,10 @@
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
}
- filterManager.updateFilterInfo(mergedComponent.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilterInfo(mergedComponent.getLSMComponentFilter(), mergedComponent.getBTree());
+ filterManager.updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
+ filterManager.writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getBTree());
}
-
- mergedComponent
- .setMostRecentMarkerLSN(mergedComponents.get(mergedComponents.size() - 1).getMostRecentMarkerLSN());
bulkLoader.end();
-
return mergedComponent;
}
@@ -587,8 +589,8 @@
FileReference btreeFileRef, FileReference bloomFilterFileRef, boolean createComponent)
throws HyracksDataException, IndexException {
// Create new BTree instance.
- LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) factory
- .createLSMComponentInstance(new LSMComponentFileReferences(btreeFileRef, null, bloomFilterFileRef));
+ LSMBTreeDiskComponent component =
+ factory.createComponent(new LSMComponentFileReferences(btreeFileRef, null, bloomFilterFileRef));
// BTree will be closed during cleanup of merge().
if (createComponent) {
component.getBTree().create();
@@ -601,10 +603,7 @@
component.getBloomFilter().activate();
}
if (component.getLSMComponentFilter() != null && !createComponent) {
- filterManager.readFilterInfo(component.getLSMComponentFilter(), component.getBTree());
- }
- if (!createComponent) {
- component.readMostRecentMarkerLSN(component.getBTree());
+ filterManager.readFilter(component.getLSMComponentFilter(), component.getBTree());
}
return component;
}
@@ -619,14 +618,14 @@
}
}
- protected ILSMComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
+ protected ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
return createDiskComponent(bulkLoadComponentFactory, componentFileRefs.getInsertIndexFileReference(),
componentFileRefs.getBloomFilterFileReference(), true);
}
@Override
- public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException {
+ public void markAsValid(ILSMDiskComponent lsmComponent) throws HyracksDataException {
// The order of forcing the dirty page to be flushed is critical. The
// bloom filter must be always done first.
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) lsmComponent;
@@ -637,7 +636,7 @@
}
public class LSMBTreeBulkLoader implements IIndexBulkLoader {
- private final ILSMComponent component;
+ private final ILSMDiskComponent component;
private final BTreeBulkLoader bulkLoader;
private final IIndexBulkLoader builder;
private boolean cleanedUpArtifacts = false;
@@ -662,8 +661,8 @@
if (hasBloomFilter) {
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
- BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
- bloomFilterFalsePositiveRate);
+ BloomFilterSpecification bloomFilterSpec =
+ BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate);
builder = ((LSMBTreeDiskComponent) component).getBloomFilter().createBuilder(numElementsHint,
bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
} else {
@@ -735,10 +734,9 @@
}
if (component.getLSMComponentFilter() != null) {
- filterManager.writeFilterInfo(component.getLSMComponentFilter(),
+ filterManager.writeFilter(component.getLSMComponentFilter(),
((LSMBTreeDiskComponent) component).getBTree());
}
- component.setMostRecentMarkerLSN(-1L);
bulkLoader.end();
if (isEmptyComponent) {
@@ -774,7 +772,7 @@
}
@Override
- public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
+ public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
return new LSMBTreeAccessor(lsmHarness, createOpContext(modificationCallback, searchCallback));
}
@@ -807,36 +805,36 @@
@Override
public ITreeIndexFrameFactory getInteriorFrameFactory() {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(
- currentMutableComponentId.get());
+ LSMBTreeMemoryComponent mutableComponent =
+ (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
return mutableComponent.getBTree().getInteriorFrameFactory();
}
@Override
public int getFieldCount() {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(
- currentMutableComponentId.get());
+ LSMBTreeMemoryComponent mutableComponent =
+ (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
return mutableComponent.getBTree().getFieldCount();
}
@Override
public int getFileId() {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(
- currentMutableComponentId.get());
+ LSMBTreeMemoryComponent mutableComponent =
+ (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
return mutableComponent.getBTree().getFileId();
}
@Override
public IPageManager getPageManager() {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(
- currentMutableComponentId.get());
+ LSMBTreeMemoryComponent mutableComponent =
+ (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
return mutableComponent.getBTree().getPageManager();
}
@Override
public ITreeIndexFrameFactory getLeafFrameFactory() {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(
- currentMutableComponentId.get());
+ LSMBTreeMemoryComponent mutableComponent =
+ (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
return mutableComponent.getBTree().getLeafFrameFactory();
}
@@ -853,16 +851,16 @@
@Override
public int getRootPageId() {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(
- currentMutableComponentId.get());
+ LSMBTreeMemoryComponent mutableComponent =
+ (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
return mutableComponent.getBTree().getRootPageId();
}
@Override
public void validate() throws HyracksDataException {
validateMemoryComponents();
- List<ILSMComponent> immutableComponents = diskComponents;
- for (ILSMComponent c : immutableComponents) {
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
+ for (ILSMDiskComponent c : immutableComponents) {
BTree btree = ((LSMBTreeDiskComponent) c).getBTree();
btree.validate();
}
@@ -897,21 +895,11 @@
if (memoryComponentsAllocated) {
return;
}
- long markerLSN = -1L;
- if (!diskComponents.isEmpty()) {
- markerLSN = diskComponents.get(diskComponents.size() - 1).getMostRecentMarkerLSN();
- } else {
- // Needed in case a marker was added before any record
- if (memoryComponents != null && !memoryComponents.isEmpty()) {
- markerLSN = memoryComponents.get(0).getMostRecentMarkerLSN();
- }
- }
for (ILSMComponent c : memoryComponents) {
LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).open();
mutableComponent.getBTree().create();
mutableComponent.getBTree().activate();
- mutableComponent.setMostRecentMarkerLSN(markerLSN);
}
memoryComponentsAllocated = true;
}
@@ -961,14 +949,4 @@
memoryComponentsAllocated = false;
}
}
-
- public synchronized long getMostRecentMarkerLSN() throws HyracksDataException {
- if (!isPrimaryIndex()) {
- throw new HyracksDataException("Markers are only supported for primary indexes");
- }
- LSMBTreeOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- opCtx.setOperation(IndexOperation.SEARCH);
- getOperationalComponents(opCtx);
- return !opCtx.getComponentHolder().isEmpty() ? opCtx.getComponentHolder().get(0).getMostRecentMarkerLSN() : -1L;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
index 244f7a9..eb8ec92 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
@@ -23,14 +23,14 @@
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
-import org.apache.hyracks.storage.am.lsm.common.impls.AbstractDiskLSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponent;
-public class LSMBTreeDiskComponent extends AbstractDiskLSMComponent {
+public class LSMBTreeDiskComponent extends AbstractLSMDiskComponent {
private final BTree btree;
private final BloomFilter bloomFilter;
public LSMBTreeDiskComponent(BTree btree, BloomFilter bloomFilter, ILSMComponentFilter filter) {
- super(filter);
+ super((IMetadataPageManager) btree.getPageManager(), filter);
this.btree = btree;
this.bloomFilter = bloomFilter;
}
@@ -63,10 +63,4 @@
public int getFileReferenceCount() {
return btree.getBufferCache().getFileReferenceCount(btree.getFileId());
}
-
- public void readMostRecentMarkerLSN(BTree treeIndex) throws HyracksDataException {
- IMetadataPageManager treeMetaManager = (IMetadataPageManager) treeIndex.getPageManager();
- treeMetaManager.get(treeMetaManager.createMetadataFrame(), MARKER_LSN_KEY, pointable);
- mostRecentMarkerLSN = pointable.getByteArray() == null ? -1L : pointable.longValue();
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java
index c46119e..a8845cc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java
@@ -22,15 +22,12 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import org.apache.hyracks.storage.am.btree.impls.BTree;
-import org.apache.hyracks.storage.am.common.api.IndexException;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
-import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-public class LSMBTreeDiskComponentFactory implements ILSMComponentFactory {
+public class LSMBTreeDiskComponentFactory implements ILSMDiskComponentFactory {
private final TreeIndexFactory<BTree> btreeFactory;
private final BloomFilterFactory bloomFilterFactory;
private final ILSMComponentFilterFactory filterFactory;
@@ -43,17 +40,11 @@
}
@Override
- public ILSMComponent createLSMComponentInstance(LSMComponentFileReferences cfr)
- throws IndexException, HyracksDataException {
+ public LSMBTreeDiskComponent createComponent(LSMComponentFileReferences cfr) throws HyracksDataException {
return new LSMBTreeDiskComponent(btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()),
bloomFilterFactory == null ? null
: bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()),
- filterFactory == null ? null : filterFactory.createLSMComponentFilter());
- }
-
- @Override
- public IBufferCache getBufferCache() {
- return btreeFactory.getBufferCache();
+ filterFactory == null ? null : filterFactory.createFilter());
}
public int[] getBloomFilterKeyFields() {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java
index f82a1b0..a2ed40b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java
@@ -29,28 +29,26 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
public class LSMBTreeFlushOperation implements ILSMIOOperation, Comparable<LSMBTreeFlushOperation> {
- private final ILSMIndexAccessorInternal accessor;
+ private final ILSMIndexAccessor accessor;
private final ILSMComponent flushingComponent;
private final FileReference btreeFlushTarget;
private final FileReference bloomFilterFlushTarget;
private final ILSMIOOperationCallback callback;
private final String indexIdentifier;
- private final long prevMarkerLSN;
- public LSMBTreeFlushOperation(ILSMIndexAccessorInternal accessor, ILSMComponent flushingComponent,
+ public LSMBTreeFlushOperation(ILSMIndexAccessor accessor, ILSMComponent flushingComponent,
FileReference btreeFlushTarget, FileReference bloomFilterFlushTarget, ILSMIOOperationCallback callback,
- String indexIdentifier, long prevMarkerLSN) {
+ String indexIdentifier) {
this.accessor = accessor;
this.flushingComponent = flushingComponent;
this.btreeFlushTarget = btreeFlushTarget;
this.bloomFilterFlushTarget = bloomFilterFlushTarget;
this.callback = callback;
this.indexIdentifier = indexIdentifier;
- this.prevMarkerLSN = prevMarkerLSN;
}
@Override
@@ -60,7 +58,7 @@
@Override
public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ Set<IODeviceHandle> devs = new HashSet<>();
devs.add(btreeFlushTarget.getDeviceHandle());
if (bloomFilterFlushTarget != null) {
devs.add(bloomFilterFlushTarget.getDeviceHandle());
@@ -87,7 +85,7 @@
return bloomFilterFlushTarget;
}
- public ILSMIndexAccessorInternal getAccessor() {
+ public ILSMIndexAccessor getAccessor() {
return accessor;
}
@@ -109,8 +107,4 @@
public int compareTo(LSMBTreeFlushOperation o) {
return btreeFlushTarget.getFile().getName().compareTo(o.getBTreeFlushTarget().getFile().getName());
}
-
- public long getPrevMarkerLSN() {
- return prevMarkerLSN;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java
index 1a103f7..32b93bb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java
@@ -23,28 +23,24 @@
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
-import org.apache.hyracks.storage.am.lsm.common.impls.AbstractMemoryLSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMMemoryComponent;
-public class LSMBTreeMemoryComponent extends AbstractMemoryLSMComponent {
+public class LSMBTreeMemoryComponent extends AbstractLSMMemoryComponent {
private final BTree btree;
- public LSMBTreeMemoryComponent(BTree btree, IVirtualBufferCache vbc, boolean isActive, ILSMComponentFilter filter,
- long mostRecentMarkerLSN) {
- super(vbc, isActive, filter, mostRecentMarkerLSN);
+ public LSMBTreeMemoryComponent(BTree btree, IVirtualBufferCache vbc, boolean isActive,
+ ILSMComponentFilter filter) {
+ super(vbc, isActive, filter);
this.btree = btree;
}
- public LSMBTreeMemoryComponent(BTree btree, IVirtualBufferCache vbc, boolean isActive, ILSMComponentFilter filter) {
- this(btree, vbc, isActive, filter, -1L);
- }
-
public BTree getBTree() {
return btree;
}
@Override
- protected void reset() throws HyracksDataException {
+ public void reset() throws HyracksDataException {
super.reset();
btree.deactivate();
btree.destroy();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java
index 9f659e4..505a5f1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java
@@ -31,11 +31,11 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
public class LSMBTreeMergeOperation implements ILSMIOOperation {
- private final ILSMIndexAccessorInternal accessor;
+ private final ILSMIndexAccessor accessor;
private final List<ILSMComponent> mergingComponents;
private final ITreeIndexCursor cursor;
private final FileReference btreeMergeTarget;
@@ -43,7 +43,7 @@
private final ILSMIOOperationCallback callback;
private final String indexIdentifier;
- public LSMBTreeMergeOperation(ILSMIndexAccessorInternal accessor, List<ILSMComponent> mergingComponents,
+ public LSMBTreeMergeOperation(ILSMIndexAccessor accessor, List<ILSMComponent> mergingComponents,
ITreeIndexCursor cursor, FileReference btreeMergeTarget, FileReference bloomFilterMergeTarget,
ILSMIOOperationCallback callback, String indexIdentifier) {
this.accessor = accessor;
@@ -57,7 +57,7 @@
@Override
public Set<IODeviceHandle> getReadDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ Set<IODeviceHandle> devs = new HashSet<>();
for (ILSMComponent o : mergingComponents) {
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) o;
devs.add(component.getBTree().getFileReference().getDeviceHandle());
@@ -70,7 +70,7 @@
@Override
public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ Set<IODeviceHandle> devs = new HashSet<>();
devs.add(btreeMergeTarget.getDeviceHandle());
if (bloomFilterMergeTarget != null) {
devs.add(bloomFilterMergeTarget.getDeviceHandle());
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
index fc8f9b2..6dafe43 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
@@ -36,8 +36,10 @@
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
public final class LSMBTreeOpContext extends AbstractLSMIndexOperationContext {
@@ -57,8 +59,8 @@
public IModificationOperationCallback modificationCallback;
public ISearchOperationCallback searchCallback;
private final List<ILSMComponent> componentHolder;
- private final List<ILSMComponent> componentsToBeMerged;
- private final List<ILSMComponent> componentsToBeReplicated;
+ private final List<ILSMDiskComponent> componentsToBeMerged;
+ private final List<ILSMDiskComponent> componentsToBeReplicated;
public final PermutingTupleReference indexTuple;
public final MultiComparator filterCmp;
public final PermutingTupleReference filterTuple;
@@ -67,10 +69,10 @@
public LSMBTreeCursorInitialState searchInitialState;
public LSMBTreePointSearchCursor insertSearchCursor;
- public LSMBTreeOpContext(List<ILSMComponent> mutableComponents, ITreeIndexFrameFactory insertLeafFrameFactory,
+ public LSMBTreeOpContext(List<ILSMMemoryComponent> mutableComponents, ITreeIndexFrameFactory insertLeafFrameFactory,
ITreeIndexFrameFactory deleteLeafFrameFactory, IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback, int numBloomFilterKeyFields, int[] btreeFields, int[] filterFields,
- ILSMHarness lsmHarness) {
+ ISearchOperationCallback searchCallback, int numBloomFilterKeyFields, int[] btreeFields,
+ int[] filterFields, ILSMHarness lsmHarness) {
LSMBTreeMemoryComponent c = (LSMBTreeMemoryComponent) mutableComponents.get(0);
IBinaryComparatorFactory cmpFactories[] = c.getBTree().getComparatorFactories();
if (cmpFactories[0] != null) {
@@ -103,9 +105,9 @@
if (deleteLeafFrame != null && this.cmp != null) {
deleteLeafFrame.setMultiComparator(cmp);
}
- this.componentHolder = new LinkedList<ILSMComponent>();
- this.componentsToBeMerged = new LinkedList<ILSMComponent>();
- this.componentsToBeReplicated = new LinkedList<ILSMComponent>();
+ this.componentHolder = new LinkedList<>();
+ this.componentsToBeMerged = new LinkedList<>();
+ this.componentsToBeReplicated = new LinkedList<>();
this.modificationCallback = modificationCallback;
this.searchCallback = searchCallback;
@@ -199,7 +201,7 @@
}
@Override
- public List<ILSMComponent> getComponentsToBeMerged() {
+ public List<ILSMDiskComponent> getComponentsToBeMerged() {
return componentsToBeMerged;
}
@@ -214,7 +216,7 @@
}
@Override
- public List<ILSMComponent> getComponentsToBeReplicated() {
+ public List<ILSMDiskComponent> getComponentsToBeReplicated() {
return componentsToBeReplicated;
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java
index 3845740..57b9092 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java
@@ -21,22 +21,26 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import org.apache.hyracks.storage.am.btree.impls.BTree;
-import org.apache.hyracks.storage.am.lsm.common.impls.AbstractDiskLSMComponent;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponent;
-public class LSMBTreeWithBuddyDiskComponent extends AbstractDiskLSMComponent {
+public class LSMBTreeWithBuddyDiskComponent extends AbstractLSMDiskComponent {
private final BTree btree;
private final BTree buddyBtree;
private final BloomFilter bloomFilter;
- public LSMBTreeWithBuddyDiskComponent(BTree btree, BTree buddyBtree, BloomFilter bloomFilter) {
+ public LSMBTreeWithBuddyDiskComponent(BTree btree, BTree buddyBtree, BloomFilter bloomFilter,
+ ILSMComponentFilter filter) {
+ super((IMetadataPageManager) btree.getPageManager(), filter);
this.btree = btree;
this.buddyBtree = buddyBtree;
this.bloomFilter = bloomFilter;
}
@Override
- protected void destroy() throws HyracksDataException {
+ public void destroy() throws HyracksDataException {
btree.deactivate();
btree.destroy();
buddyBtree.deactivate();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentFactory.java
index bddea2a..4223954 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentFactory.java
@@ -21,45 +21,27 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import org.apache.hyracks.storage.am.btree.impls.BTree;
-import org.apache.hyracks.storage.am.common.api.IndexException;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
-import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-public class LSMBTreeWithBuddyDiskComponentFactory implements
- ILSMComponentFactory {
+public class LSMBTreeWithBuddyDiskComponentFactory implements ILSMDiskComponentFactory {
private final TreeIndexFactory<BTree> btreeFactory;
private final TreeIndexFactory<BTree> buddyBtreeFactory;
private final BloomFilterFactory bloomFilterFactory;
- public LSMBTreeWithBuddyDiskComponentFactory(
- TreeIndexFactory<BTree> btreeFactory,
- TreeIndexFactory<BTree> buddyBtreeFactory,
- BloomFilterFactory bloomFilterFactory) {
+ public LSMBTreeWithBuddyDiskComponentFactory(TreeIndexFactory<BTree> btreeFactory,
+ TreeIndexFactory<BTree> buddyBtreeFactory, BloomFilterFactory bloomFilterFactory) {
this.btreeFactory = btreeFactory;
this.buddyBtreeFactory = buddyBtreeFactory;
this.bloomFilterFactory = bloomFilterFactory;
}
@Override
- public ILSMComponent createLSMComponentInstance(
- LSMComponentFileReferences cfr) throws IndexException,
- HyracksDataException {
- return new LSMBTreeWithBuddyDiskComponent(
- btreeFactory.createIndexInstance(cfr
- .getInsertIndexFileReference()),
- buddyBtreeFactory.createIndexInstance(cfr
- .getDeleteIndexFileReference()),
- bloomFilterFactory.createBloomFiltertInstance(cfr
- .getBloomFilterFileReference()));
+ public LSMBTreeWithBuddyDiskComponent createComponent(LSMComponentFileReferences cfr) throws HyracksDataException {
+ return new LSMBTreeWithBuddyDiskComponent(btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()),
+ buddyBtreeFactory.createIndexInstance(cfr.getDeleteIndexFileReference()),
+ bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()), null);
}
-
- @Override
- public IBufferCache getBufferCache() {
- return btreeFactory.getBufferCache();
- }
-
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMemoryComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMemoryComponent.java
index a45cbf1..f58c737 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMemoryComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMemoryComponent.java
@@ -20,21 +20,21 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.btree.impls.BTree;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
-import org.apache.hyracks.storage.am.lsm.common.impls.AbstractMemoryLSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMMemoryComponent;
/*
* This class is also not needed at the moment but is implemented anyway
*/
-public class LSMBTreeWithBuddyMemoryComponent extends
- AbstractMemoryLSMComponent {
+public class LSMBTreeWithBuddyMemoryComponent extends AbstractLSMMemoryComponent {
private final BTree btree;
private final BTree buddyBtree;
- public LSMBTreeWithBuddyMemoryComponent(BTree btree, BTree buddyBtree,
- IVirtualBufferCache vbc, boolean isActive) {
- super(vbc, isActive);
+ public LSMBTreeWithBuddyMemoryComponent(BTree btree, BTree buddyBtree, IVirtualBufferCache vbc, boolean isActive,
+ ILSMComponentFilter filter) {
+ super(vbc, isActive, filter);
this.btree = btree;
this.buddyBtree = buddyBtree;
}
@@ -48,7 +48,7 @@
}
@Override
- protected void reset() throws HyracksDataException {
+ public void reset() throws HyracksDataException {
super.reset();
btree.deactivate();
btree.destroy();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java
index d70bdd5..2810b62 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java
@@ -30,11 +30,11 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
public class LSMBTreeWithBuddyMergeOperation implements ILSMIOOperation {
- private final ILSMIndexAccessorInternal accessor;
+ private final ILSMIndexAccessor accessor;
private final List<ILSMComponent> mergingComponents;
private final ITreeIndexCursor cursor;
private final FileReference btreeMergeTarget;
@@ -44,12 +44,10 @@
private final String indexIdentifier;
private final boolean keepDeletedTuples;
- public LSMBTreeWithBuddyMergeOperation(ILSMIndexAccessorInternal accessor,
- List<ILSMComponent> mergingComponents, ITreeIndexCursor cursor,
- FileReference btreeMergeTarget,
- FileReference buddyBtreeMergeTarget,
- FileReference bloomFilterMergeTarget,
- ILSMIOOperationCallback callback, String indexIdentifier, boolean keepDeletedTuples) {
+ public LSMBTreeWithBuddyMergeOperation(ILSMIndexAccessor accessor, List<ILSMComponent> mergingComponents,
+ ITreeIndexCursor cursor, FileReference btreeMergeTarget, FileReference buddyBtreeMergeTarget,
+ FileReference bloomFilterMergeTarget, ILSMIOOperationCallback callback, String indexIdentifier,
+ boolean keepDeletedTuples) {
this.accessor = accessor;
this.mergingComponents = mergingComponents;
this.cursor = cursor;
@@ -63,15 +61,13 @@
@Override
public Set<IODeviceHandle> getReadDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ Set<IODeviceHandle> devs = new HashSet<>();
for (ILSMComponent o : mergingComponents) {
LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) o;
devs.add(component.getBTree().getFileReference().getDeviceHandle());
- devs.add(component.getBuddyBTree().getFileReference()
- .getDeviceHandle());
- devs.add(component.getBloomFilter().getFileReference()
- .getDeviceHandle());
+ devs.add(component.getBuddyBTree().getFileReference().getDeviceHandle());
+ devs.add(component.getBloomFilter().getFileReference().getDeviceHandle());
}
return devs;
@@ -79,7 +75,7 @@
@Override
public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ Set<IODeviceHandle> devs = new HashSet<>();
devs.add(btreeMergeTarget.getDeviceHandle());
devs.add(buddyBtreeMergeTarget.getDeviceHandle());
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
index 8ab7235..ebfb6bd 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
@@ -76,5 +76,9 @@
<artifactId>hyracks-data-std</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IComponentMetadata.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IComponentMetadata.java
new file mode 100644
index 0000000..5dee557
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IComponentMetadata.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.common.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+public interface IComponentMetadata {
+
+ /**
+ * Put the key value pair in this metadata, overwrite if it exists
+ *
+ * @param key
+ * @param value
+ * @throws HyracksDataException
+ * if the component is immutable
+ */
+ void put(IValueReference key, IValueReference value) throws HyracksDataException;
+
+ /**
+ * Get the value of the key from the metadata, 0 length value if not exists
+ *
+ * @param key
+ * @param value
+ * @throws HyracksDataException
+ */
+ void get(IValueReference key, IPointable value) throws HyracksDataException;
+
+ /**
+ * Get the value
+ *
+ * @param key
+ * @return
+ * @throws HyracksDataException
+ */
+ IValueReference get(IValueReference key) throws HyracksDataException;
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponent.java
index a888dd5..a446f4e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponent.java
@@ -20,34 +20,105 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
+/**
+ * An LSM index component. can be an in memory or on disk. Can be readable or unreadable, writable or unwritable
+ */
public interface ILSMComponent {
+ /**
+ * Memory or Disk
+ */
enum LSMComponentType {
+ /**
+ * Memory component
+ */
MEMORY,
+ /**
+ * Disk component
+ */
DISK
}
+ /**
+ * The state of a component
+ */
enum ComponentState {
+ /**
+ * The component is inactive (Unreadable and Unwritable). Can be activated
+ */
INACTIVE,
+ /**
+ * The component can be read from and can be written to
+ */
READABLE_WRITABLE,
+ /**
+ * Immutable component that can be read from but not written to
+ */
READABLE_UNWRITABLE,
+ /**
+ * A component that is being flushed. Can be read from but not written to
+ */
READABLE_UNWRITABLE_FLUSHING,
+ /**
+ * A component that has completed flushing but still has some readers inside
+ * This is equivalent to a DEACTIVATING state
+ */
UNREADABLE_UNWRITABLE,
+ /**
+ * An immutable component that is being merged
+ */
READABLE_MERGING
}
- public boolean threadEnter(LSMOperationType opType, boolean isMutableComponent) throws HyracksDataException;
+ /**
+ * Enter the component
+ *
+ * @param opType
+ * the operation over the whole LSM index
+ * @param isMutableComponent
+ * true if the thread intends to modify this component (write access), false otherwise
+ * @return
+ * true if the thread entered the component, false otherwise
+ * @throws HyracksDataException
+ * if the attempted operation is not allowed on the component
+ */
+ boolean threadEnter(LSMOperationType opType, boolean isMutableComponent) throws HyracksDataException;
- public void threadExit(LSMOperationType opType, boolean failedOperation, boolean isMutableComponent)
+ /**
+ * Exit the component
+ *
+ * @param opType
+ * the operation over the whole LSM index under which the thread is running
+ * @param failedOperation
+ * whether the operation failed
+ * @param isMutableComponent
+ * true if the thread intended to modify the component
+ * @throws HyracksDataException
+ */
+ void threadExit(LSMOperationType opType, boolean failedOperation, boolean isMutableComponent)
throws HyracksDataException;
- public LSMComponentType getType();
+ /**
+ * @return the component type (memory vs, disk)
+ */
+ LSMComponentType getType();
- public ComponentState getState();
+ /**
+ * @return the component state
+ */
+ ComponentState getState();
- public ILSMComponentFilter getLSMComponentFilter();
+ /**
+ * @return the metadata associated with the component
+ */
+ IComponentMetadata getMetadata();
- public void setMostRecentMarkerLSN(long lsn);
-
- public long getMostRecentMarkerLSN();
+ /**
+ * Note: the component filter is a special case of component metadata in the sense that it is
+ * strongly connected with the index and is used by the index logic when doing read and write operations
+ * hence, we're leaving this call here
+ *
+ * @return the component filter
+ */
+ ILSMComponentFilter getLSMComponentFilter();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilter.java
index 0bfa466..1bb20c5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilter.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilter.java
@@ -25,16 +25,15 @@
public interface ILSMComponentFilter {
- public void update(ITupleReference tuple, MultiComparator cmp) throws HyracksDataException;
+ void update(ITupleReference tuple, MultiComparator cmp) throws HyracksDataException;
- public boolean satisfy(ITupleReference minTuple, ITupleReference maxTuple, MultiComparator filterCmp)
- throws HyracksDataException;
+ boolean satisfy(ITupleReference min, ITupleReference max, MultiComparator cmp) throws HyracksDataException;
- public ITupleReference getMinTuple();
+ ITupleReference getMinTuple();
- public ITupleReference getMaxTuple();
+ ITupleReference getMaxTuple();
- public IBinaryComparatorFactory[] getFilterCmpFactories();
+ IBinaryComparatorFactory[] getFilterCmpFactories();
- public void reset();
+ void reset();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFactory.java
index 1fa2175..64562bd 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFactory.java
@@ -18,7 +18,8 @@
*/
package org.apache.hyracks.storage.am.lsm.common.api;
+@FunctionalInterface
public interface ILSMComponentFilterFactory {
- public ILSMComponentFilter createLSMComponentFilter();
+ ILSMComponentFilter createFilter();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFrameFactory.java
index 1c853c8..17eb51e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFrameFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFrameFactory.java
@@ -18,6 +18,7 @@
*/
package org.apache.hyracks.storage.am.lsm.common.api;
+@FunctionalInterface
public interface ILSMComponentFilterFrameFactory {
- public ILSMComponentFilterReference createFrame();
+ ILSMComponentFilterReference createFrame();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterManager.java
index 12cafef..f310b4e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterManager.java
@@ -26,12 +26,10 @@
public interface ILSMComponentFilterManager {
- public void updateFilterInfo(ILSMComponentFilter filter, List<ITupleReference> filterTuples)
- throws HyracksDataException;
+ void updateFilter(ILSMComponentFilter filter, List<ITupleReference> filterTuples) throws HyracksDataException;
- public boolean readFilterInfo(ILSMComponentFilter filter, ITreeIndex treeIndex) throws HyracksDataException;
+ boolean readFilter(ILSMComponentFilter filter, ITreeIndex index) throws HyracksDataException;
- public void writeFilterInfo(ILSMComponentFilter filter, ITreeIndex treeIndex)
- throws HyracksDataException;
+ void writeFilter(ILSMComponentFilter filter, ITreeIndex index) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessorInternal.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java
similarity index 63%
rename from hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessorInternal.java
rename to hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java
index 4df1180..335e84e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessorInternal.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java
@@ -19,24 +19,31 @@
package org.apache.hyracks.storage.am.lsm.common.api;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.common.api.IndexException;
-import org.apache.hyracks.storage.am.common.api.TreeIndexException;
+import org.apache.hyracks.storage.am.lsm.common.impls.DiskComponentMetadata;
-public interface ILSMIndexAccessorInternal extends ILSMIndexAccessor {
+public interface ILSMDiskComponent extends ILSMComponent {
+ @Override
+ default LSMComponentType getType() {
+ return LSMComponentType.DISK;
+ }
+
+ @Override
+ DiskComponentMetadata getMetadata();
/**
- * Force a flush of the in-memory component.
- *
- * @throws HyracksDataException
- * @throws TreeIndexException
+ * @return the on disk size of this component
*/
- public void flush(ILSMIOOperation operation) throws HyracksDataException, IndexException;
+ long getComponentSize();
/**
- * Merge all on-disk components.
+ * @return the reference count for the component
+ */
+ int getFileReferenceCount();
+
+ /**
+ * Delete the component from disk
*
* @throws HyracksDataException
- * @throws TreeIndexException
*/
- public void merge(ILSMIOOperation operation) throws HyracksDataException, IndexException;
+ void destroy() throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentFactory.java
similarity index 70%
rename from hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFactory.java
rename to hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentFactory.java
index 7906082..9daf30b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentFactory.java
@@ -19,13 +19,18 @@
package org.apache.hyracks.storage.am.lsm.common.api;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
-import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-public interface ILSMComponentFactory {
- public ILSMComponent createLSMComponentInstance(LSMComponentFileReferences cfr) throws IndexException,
- HyracksDataException;
+@FunctionalInterface
+public interface ILSMDiskComponentFactory {
- public IBufferCache getBufferCache();
+ /**
+ * Create a disk component from the file references
+ *
+ * @param cfr
+ * the disk file references which points to the different physical files of the index
+ * @return a disk component
+ * @throws HyracksDataException
+ */
+ ILSMDiskComponent createComponent(LSMComponentFileReferences cfr) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java
index 4c4ed28..7a2bc7c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java
@@ -28,38 +28,34 @@
public interface ILSMHarness {
- public void forceModify(ILSMIndexOperationContext ctx, ITupleReference tuple)
+ void forceModify(ILSMIndexOperationContext ctx, ITupleReference tuple) throws HyracksDataException, IndexException;
+
+ boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple)
throws HyracksDataException, IndexException;
- public boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple)
+ void search(ILSMIndexOperationContext ctx, IIndexCursor cursor, ISearchPredicate pred)
throws HyracksDataException, IndexException;
- public void search(ILSMIndexOperationContext ctx, IIndexCursor cursor, ISearchPredicate pred)
+ void endSearch(ILSMIndexOperationContext ctx) throws HyracksDataException;
+
+ void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException, IndexException;
- public void endSearch(ILSMIndexOperationContext ctx) throws HyracksDataException;
-
- public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ void scheduleFullMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException, IndexException;
- public void scheduleFullMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
- throws HyracksDataException, IndexException;
+ void merge(ILSMIndexOperationContext ctx, ILSMIOOperation operation) throws HyracksDataException, IndexException;
- public void merge(ILSMIndexOperationContext ctx, ILSMIOOperation operation)
- throws HyracksDataException, IndexException;
+ void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback) throws HyracksDataException;
- public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
- throws HyracksDataException;
+ void flush(ILSMIndexOperationContext ctx, ILSMIOOperation operation) throws HyracksDataException, IndexException;
- public void flush(ILSMIndexOperationContext ctx, ILSMIOOperation operation)
- throws HyracksDataException, IndexException;
+ void addBulkLoadedComponent(ILSMDiskComponent index) throws HyracksDataException, IndexException;
- public void addBulkLoadedComponent(ILSMComponent index) throws HyracksDataException, IndexException;
+ ILSMOperationTracker getOperationTracker();
- public ILSMOperationTracker getOperationTracker();
-
- public void scheduleReplication(ILSMIndexOperationContext ctx, List<ILSMComponent> lsmComponents, boolean bulkload,
+ void scheduleReplication(ILSMIndexOperationContext ctx, List<ILSMDiskComponent> diskComponents, boolean bulkload,
LSMOperationType opType) throws HyracksDataException;
- public void endReplication(ILSMIndexOperationContext ctx) throws HyracksDataException;
+ void endReplication(ILSMIndexOperationContext ctx) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
index 0743e4a..d768122 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
@@ -27,20 +27,21 @@
public interface ILSMIOOperation extends Callable<Boolean> {
- public enum LSMIOOpertionType {
+ enum LSMIOOpertionType {
FLUSH,
MERGE
}
- public Set<IODeviceHandle> getReadDevices();
+ Set<IODeviceHandle> getReadDevices();
- public Set<IODeviceHandle> getWriteDevices();
+ Set<IODeviceHandle> getWriteDevices();
- public Boolean call() throws HyracksDataException, IndexException;
+ @Override
+ Boolean call() throws HyracksDataException, IndexException;
- public ILSMIOOperationCallback getCallback();
+ ILSMIOOperationCallback getCallback();
- public String getIndexUniqueIdentifier();
+ String getIndexUniqueIdentifier();
- public LSMIOOpertionType getIOOpertionType();
+ LSMIOOpertionType getIOOpertionType();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallback.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallback.java
index 6315124..7cfd268 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallback.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallback.java
@@ -25,20 +25,23 @@
public interface ILSMIOOperationCallback {
/**
- * This method is called on an IO operation sometime before the operation is executed.
+ * This method is called on an IO operation before the operation starts.
* (i.e. IO operations could be flush or merge operations.)
*/
- public void beforeOperation(LSMOperationType opType) throws HyracksDataException;
+ void beforeOperation(LSMOperationType opType) throws HyracksDataException;
/**
* This method is called on an IO operation sometime after the operation was completed.
* (i.e. IO operations could be flush or merge operations.)
*
+ * Copying content of metadata page from memory component to disk component should be done in this call
+ * Merging content of metadata pages from disk components to new disk component should be done in this call
+ *
* @param oldComponents
* @param newComponent
* @throws HyracksDataException
*/
- public void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents, ILSMComponent newComponent)
+ void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents, ILSMDiskComponent newComponent)
throws HyracksDataException;
/**
@@ -49,7 +52,7 @@
* @param newComponent
* @throws HyracksDataException
*/
- public void afterFinalize(LSMOperationType opType, ILSMComponent newComponent) throws HyracksDataException;
+ void afterFinalize(LSMOperationType opType, ILSMDiskComponent newComponent) throws HyracksDataException;
- public void setNumOfMutableComponents(int count);
+ void setNumOfMutableComponents(int count);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallbackFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallbackFactory.java
index c91fa8b..ef76415 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallbackFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallbackFactory.java
@@ -22,5 +22,5 @@
@FunctionalInterface
public interface ILSMIOOperationCallbackFactory extends Serializable {
- ILSMIOOperationCallback createIOOperationCallback();
+ ILSMIOOperationCallback createIoOpCallback();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallbackProvider.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallbackProvider.java
index 684be6d..11ff0bf 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallbackProvider.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallbackProvider.java
@@ -18,6 +18,7 @@
*/
package org.apache.hyracks.storage.am.lsm.common.api;
+@FunctionalInterface
public interface ILSMIOOperationCallbackProvider {
- public ILSMIOOperationCallback getIOOperationCallback(ILSMIndex index);
+ ILSMIOOperationCallback getIOOperationCallback(ILSMIndex index);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
index 11b933d..234006f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
@@ -22,9 +22,15 @@
import java.util.List;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.IIndex;
+import org.apache.hyracks.storage.am.common.api.IIndexCursor;
+import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
+import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
+import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMHarness;
/**
@@ -37,24 +43,94 @@
*/
public interface ILSMIndex extends IIndex {
- public void deactivate(boolean flushOnExit) throws HyracksDataException;
+ void deactivate(boolean flushOnExit) throws HyracksDataException;
@Override
- public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
+ ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) throws HyracksDataException;
- public ILSMOperationTracker getOperationTracker();
+ ILSMOperationTracker getOperationTracker();
- public ILSMIOOperationScheduler getIOScheduler();
+ ILSMIOOperationScheduler getIOScheduler();
- public ILSMIOOperationCallback getIOOperationCallback();
+ ILSMIOOperationCallback getIOOperationCallback();
- public List<ILSMComponent> getImmutableComponents();
+ /**
+ * components with lower indexes are newer than components with higher index
+ */
+ List<ILSMDiskComponent> getImmutableComponents();
- public boolean isPrimaryIndex();
+ boolean isPrimaryIndex();
+
+ void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException;
+
+ void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred)
+ throws HyracksDataException, IndexException;
+
+ void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback) throws HyracksDataException;
+
+ ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException;
+
+ void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
+ throws HyracksDataException, IndexException;
+
+ ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException;
+
+ void addDiskComponent(ILSMDiskComponent index) throws HyracksDataException;
+
+ void subsumeMergedComponents(ILSMDiskComponent newComponent, List<ILSMComponent> mergedComponents)
+ throws HyracksDataException;
+
+ void changeMutableComponent();
+
+ void changeFlushStatusForCurrentMutableCompoent(boolean needsFlush);
+
+ boolean hasFlushRequestForCurrentMutableComponent();
+
+ /**
+ * Populates the context's component holder with a snapshot of the components involved in the operation.
+ *
+ * @param ctx
+ * - the operation's context
+ * @throws HyracksDataException
+ */
+ void getOperationalComponents(ILSMIndexOperationContext ctx) throws HyracksDataException;
+
+ List<ILSMDiskComponent> getInactiveDiskComponents();
+
+ void addInactiveDiskComponent(ILSMDiskComponent diskComponent);
+
+ /**
+ * Persist the LSM component
+ *
+ * @param lsmComponent
+ * , the component to be persistent
+ * @throws HyracksDataException
+ */
+ void markAsValid(ILSMDiskComponent lsmComponent) throws HyracksDataException;
+
+ boolean isCurrentMutableComponentEmpty() throws HyracksDataException;
+
+ void scheduleReplication(ILSMIndexOperationContext ctx, List<ILSMDiskComponent> diskComponents, boolean bulkload,
+ ReplicationOperation operation, LSMOperationType opType) throws HyracksDataException;
+
+ boolean isMemoryComponentsAllocated();
+
+ /**
+ * Allocates the memory components of an LSM index in the buffer cache.
+ *
+ * @throws HyracksDataException
+ */
+ void allocateMemoryComponents() throws HyracksDataException;
+
+ ILSMMemoryComponent getCurrentMemoryComponent();
+
+ int getCurrentMemoryComponentIndex();
+
+ List<ILSMMemoryComponent> getMemoryComponents();
/**
* @return true if the index is durable. Otherwise false.
*/
- public boolean isDurable();
+ boolean isDurable();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
index 439a020..fecc674 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.IIndexAccessor;
import org.apache.hyracks.storage.am.common.api.IndexException;
+import org.apache.hyracks.storage.am.common.api.TreeIndexException;
/**
* Client handle for performing operations
@@ -33,12 +34,12 @@
* concurrent operations).
*/
public interface ILSMIndexAccessor extends IIndexAccessor {
- public void scheduleFlush(ILSMIOOperationCallback callback) throws HyracksDataException;
+ void scheduleFlush(ILSMIOOperationCallback callback) throws HyracksDataException;
- public void scheduleMerge(ILSMIOOperationCallback callback, List<ILSMComponent> components)
+ void scheduleMerge(ILSMIOOperationCallback callback, List<ILSMDiskComponent> components)
throws HyracksDataException, IndexException;
- public void scheduleFullMerge(ILSMIOOperationCallback callback) throws HyracksDataException, IndexException;
+ void scheduleFullMerge(ILSMIOOperationCallback callback) throws HyracksDataException, IndexException;
/**
* Deletes the tuple from the memory component only.
@@ -46,7 +47,7 @@
* @throws HyracksDataException
* @throws IndexException
*/
- public void physicalDelete(ITupleReference tuple) throws HyracksDataException, IndexException;
+ void physicalDelete(ITupleReference tuple) throws HyracksDataException, IndexException;
/**
* Attempts to insert the given tuple.
@@ -62,7 +63,7 @@
* If an index-specific constraint is violated, e.g., the key
* already exists.
*/
- public boolean tryInsert(ITupleReference tuple) throws HyracksDataException, IndexException;
+ boolean tryInsert(ITupleReference tuple) throws HyracksDataException, IndexException;
/**
* Attempts to delete the given tuple.
@@ -77,7 +78,7 @@
* @throws IndexException
* If there is no matching tuple in the index.
*/
- public boolean tryDelete(ITupleReference tuple) throws HyracksDataException, IndexException;
+ boolean tryDelete(ITupleReference tuple) throws HyracksDataException, IndexException;
/**
* Attempts to update the given tuple.
@@ -93,7 +94,7 @@
* @throws IndexException
* If there is no matching tuple in the index.
*/
- public boolean tryUpdate(ITupleReference tuple) throws HyracksDataException, IndexException;
+ boolean tryUpdate(ITupleReference tuple) throws HyracksDataException, IndexException;
/**
* This operation is only supported by indexes with the notion of a unique key.
@@ -110,14 +111,30 @@
* @throws IndexException
* If there is no matching tuple in the index.
*/
- public boolean tryUpsert(ITupleReference tuple) throws HyracksDataException, IndexException;
+ boolean tryUpsert(ITupleReference tuple) throws HyracksDataException, IndexException;
- public void forcePhysicalDelete(ITupleReference tuple) throws HyracksDataException, IndexException;
+ void forcePhysicalDelete(ITupleReference tuple) throws HyracksDataException, IndexException;
- public void forceInsert(ITupleReference tuple) throws HyracksDataException, IndexException;
+ void forceInsert(ITupleReference tuple) throws HyracksDataException, IndexException;
- public void forceDelete(ITupleReference tuple) throws HyracksDataException, IndexException;
+ void forceDelete(ITupleReference tuple) throws HyracksDataException, IndexException;
- public void scheduleReplication(List<ILSMComponent> lsmComponents, boolean bulkload, LSMOperationType opType)
+ void scheduleReplication(List<ILSMDiskComponent> diskComponents, boolean bulkload, LSMOperationType opType)
throws HyracksDataException;
+
+ /**
+ * Force a flush of the in-memory component.
+ *
+ * @throws HyracksDataException
+ * @throws TreeIndexException
+ */
+ void flush(ILSMIOOperation operation) throws HyracksDataException, IndexException;
+
+ /**
+ * Merge all on-disk components.
+ *
+ * @throws HyracksDataException
+ * @throws TreeIndexException
+ */
+ void merge(ILSMIOOperation operation) throws HyracksDataException, IndexException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
deleted file mode 100644
index e69c714..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.storage.am.lsm.common.api;
-
-import java.util.List;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation;
-import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.common.api.IIndexCursor;
-import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
-import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
-import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
-import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
-import org.apache.hyracks.storage.am.common.api.IndexException;
-
-public interface ILSMIndexInternal extends ILSMIndex {
- @Override
- public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback) throws HyracksDataException;
-
- public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException;
-
- public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred)
- throws HyracksDataException, IndexException;
-
- public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
- throws HyracksDataException;
-
- public ILSMComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException;
-
- public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
- throws HyracksDataException, IndexException;
-
- public ILSMComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException;
-
- public void addComponent(ILSMComponent index) throws HyracksDataException;
-
- public void subsumeMergedComponents(ILSMComponent newComponent, List<ILSMComponent> mergedComponents)
- throws HyracksDataException;
-
- public void changeMutableComponent();
-
- public void changeFlushStatusForCurrentMutableCompoent(boolean needsFlush);
-
- public boolean hasFlushRequestForCurrentMutableComponent();
-
- /**
- * Populates the context's component holder with a snapshot of the components involved in the operation.
- *
- * @param ctx
- * - the operation's context
- * @throws HyracksDataException
- */
- public void getOperationalComponents(ILSMIndexOperationContext ctx) throws HyracksDataException;
-
- public List<ILSMComponent> getInactiveDiskComponents();
-
- public void addInactiveDiskComponent(ILSMComponent diskComponent);
-
- /**
- * Persist the LSM component
- *
- * @param lsmComponent
- * , the component to be persistent
- * @throws HyracksDataException
- */
- public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException;
-
- public boolean isCurrentMutableComponentEmpty() throws HyracksDataException;
-
- public void scheduleReplication(ILSMIndexOperationContext ctx, List<ILSMComponent> lsmComponents, boolean bulkload,
- ReplicationOperation operation, LSMOperationType opType) throws HyracksDataException;
-
- /**
- * Allocates the memory components of an LSM index in the buffer cache.
- * @throws HyracksDataException
- */
- public void allocateMemoryComponents() throws HyracksDataException;
-
- public boolean isMemoryComponentsAllocated();
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java
index acf2233..9c5701a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java
@@ -26,26 +26,26 @@
import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
public interface ILSMIndexOperationContext extends IIndexOperationContext {
- public List<ILSMComponent> getComponentHolder();
+ List<ILSMComponent> getComponentHolder();
- public List<ILSMComponent> getComponentsToBeMerged();
+ List<ILSMDiskComponent> getComponentsToBeMerged();
- public ISearchOperationCallback getSearchOperationCallback();
+ ISearchOperationCallback getSearchOperationCallback();
- public IModificationOperationCallback getModificationCallback();
+ IModificationOperationCallback getModificationCallback();
- public void setCurrentMutableComponentId(int currentMutableComponentId);
+ void setCurrentMutableComponentId(int currentMutableComponentId);
- public void setSearchPredicate(ISearchPredicate searchPredicate);
+ void setSearchPredicate(ISearchPredicate searchPredicate);
- public ISearchPredicate getSearchPredicate();
+ ISearchPredicate getSearchPredicate();
- public List<ILSMComponent> getComponentsToBeReplicated();
+ List<ILSMDiskComponent> getComponentsToBeReplicated();
/**
* @return true if this operation entered the components. Otherwise false.
*/
- public boolean isAccessingComponents();
+ boolean isAccessingComponents();
- public void setAccessingComponents(boolean accessingComponents);
+ void setAccessingComponents(boolean accessingComponents);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMMemoryComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMMemoryComponent.java
new file mode 100644
index 0000000..18121e0
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMMemoryComponent.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.common.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.common.impls.MemoryComponentMetadata;
+
+public interface ILSMMemoryComponent extends ILSMComponent {
+ @Override
+ default LSMComponentType getType() {
+ return LSMComponentType.MEMORY;
+ }
+
+ @Override
+ MemoryComponentMetadata getMetadata();
+
+ /**
+ * @return true if the component can be entered for reading
+ */
+ boolean isReadable();
+
+ /**
+ * @return the number of writers inside the component
+ */
+ int getWriterCount();
+
+ /**
+ * Clear the component and its metadata page completely
+ *
+ * @throws HyracksDataException
+ */
+ void reset() throws HyracksDataException;
+
+ /**
+ * @return true if the memory budget has been exceeded
+ */
+ boolean isFull();
+
+ /**
+ * @return true if there are data in the memory component, false otherwise
+ */
+ boolean isModified();
+
+ /**
+ * Set the component as modified
+ */
+ void setModified();
+
+ /**
+ * request the component to be active
+ */
+ void activate();
+
+ /**
+ * Set the component state
+ *
+ * @param state
+ * the new state
+ */
+ void setState(ComponentState state);
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMMergePolicy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMMergePolicy.java
index c64fe63..c2bd45d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMMergePolicy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMMergePolicy.java
@@ -25,10 +25,9 @@
import org.apache.hyracks.storage.am.common.api.IndexException;
public interface ILSMMergePolicy {
- public void diskComponentAdded(ILSMIndex index, boolean fullMergeIsRequested)
- throws HyracksDataException, IndexException;
+ void diskComponentAdded(ILSMIndex index, boolean fullMergeIsRequested) throws HyracksDataException, IndexException;
- public void configure(Map<String, String> properties);
+ void configure(Map<String, String> properties);
/**
* This method is used for flush-operation flow control:
@@ -55,5 +54,5 @@
* @throws HyracksDataException
* @throws IndexException
*/
- public boolean isMergeLagging(ILSMIndex index) throws HyracksDataException, IndexException;
+ boolean isMergeLagging(ILSMIndex index) throws HyracksDataException, IndexException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMMergePolicyFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMMergePolicyFactory.java
index c047777..3e605bd 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMMergePolicyFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMMergePolicyFactory.java
@@ -30,11 +30,11 @@
// because we want to have an access to the IIndexLifecycleManager inside some of the merge policies. However,
// in order to get the IIndexLifecycleManager instance, we need to cast to IAsterixAppRuntimeContext which exist
// in asterix and cannot be seen in hyracks. Thus we pass IHyracksTaskContext and let the merge policy do the casting.
- public ILSMMergePolicy createMergePolicy(Map<String, String> configuration, IHyracksTaskContext ctx);
+ ILSMMergePolicy createMergePolicy(Map<String, String> configuration, IHyracksTaskContext ctx);
- public ILSMMergePolicy createMergePolicy(Map<String, String> configuration, IResourceLifecycleManager ilcm);
+ ILSMMergePolicy createMergePolicy(Map<String, String> configuration, IResourceLifecycleManager ilcm);
- public String getName();
+ String getName();
- public Set<String> getPropertiesNames();
+ Set<String> getPropertiesNames();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMOperationTracker.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMOperationTracker.java
index e523f52..d3f33af 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMOperationTracker.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMOperationTracker.java
@@ -27,8 +27,6 @@
* This interface exposes methods for tracking and setting the status of operations for the purpose
* of coordinating flushes/merges in {@link ILSMIndex}.
* Note that 'operation' below refers to {@link IIndexAccessor} methods.
- *
- * @author zheilbron
*/
public interface ILSMOperationTracker {
@@ -39,7 +37,7 @@
* then this method does not block and returns false.
* Otherwise, this method returns true, and the operation is considered 'active' in the index.
*/
- public void beforeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
+ void beforeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException;
/**
@@ -48,7 +46,7 @@
* After this method has been called, the operation is still considered 'active',
* until the issuer of the operation declares it completed by calling completeOperation().
*/
- public void afterOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
+ void afterOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException;
/**
@@ -56,6 +54,6 @@
* The use of this method indicates that the operation is no longer 'active'
* for the purpose of coordinating flushes/merges.
*/
- public void completeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
+ void completeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ITwoPCIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ITwoPCIndex.java
index 95e95e4..65e91b2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ITwoPCIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ITwoPCIndex.java
@@ -53,22 +53,24 @@
/**
* This function is used to add the committed disk component to the appropriate list and reflect the changes
*/
- public void commitTransactionDiskComponent(ILSMComponent newComponent) throws IndexException, HyracksDataException;
+ public void commitTransactionDiskComponent(ILSMDiskComponent newComponent)
+ throws IndexException, HyracksDataException;
/**
* This function is used to create a version specific accessor to search a specific version
*/
- public ILSMIndexAccessorInternal createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion) throws HyracksDataException;
+ public ILSMIndexAccessor createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion)
+ throws HyracksDataException;
/**
* This function is used to get the first components list
*/
- public List<ILSMComponent> getFirstComponentList();
+ public List<ILSMDiskComponent> getFirstComponentList();
/**
* This function is used to get teh second components list
*/
- public List<ILSMComponent> getSecondComponentList();
+ public List<ILSMDiskComponent> getSecondComponentList();
/**
* This function is used to get the current version id of the index
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCache.java
index 09cdb61..ff7bfb5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCache.java
@@ -23,11 +23,11 @@
import org.apache.hyracks.storage.common.file.IFileMapManager;
public interface IVirtualBufferCache extends IBufferCache {
- public void open() throws HyracksDataException;
+ void open() throws HyracksDataException;
- public boolean isFull();
+ boolean isFull();
- public void reset();
+ void reset();
- public IFileMapManager getFileMapProvider();
+ IFileMapManager getFileMapProvider();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java
index ca61075..c1a6887 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java
@@ -25,7 +25,8 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
+@FunctionalInterface
public interface IVirtualBufferCacheProvider extends Serializable {
- public List<IVirtualBufferCache> getVirtualBufferCaches(IHyracksTaskContext ctx,
- IFileSplitProvider fileSplitProvider) throws HyracksDataException;
+ List<IVirtualBufferCache> getVirtualBufferCaches(IHyracksTaskContext ctx, IFileSplitProvider fileSplitProvider)
+ throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMComponent.java
index cf50c00..d69a78c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMComponent.java
@@ -19,33 +19,20 @@
package org.apache.hyracks.storage.am.lsm.common.impls;
import org.apache.hyracks.data.std.primitive.LongPointable;
-import org.apache.hyracks.storage.am.common.freepage.MutableArrayValueReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
public abstract class AbstractLSMComponent implements ILSMComponent {
-
- public static final MutableArrayValueReference MARKER_LSN_KEY = new MutableArrayValueReference("Marker"
- .getBytes());
-
+ // Finals
+ protected final LongPointable pointable = LongPointable.FACTORY.createPointable();
+ protected final ILSMComponentFilter filter;
+ // Mutables
protected ComponentState state;
protected int readerCount;
- protected final ILSMComponentFilter filter;
- protected long mostRecentMarkerLSN;
- protected final LongPointable pointable = (LongPointable) LongPointable.FACTORY.createPointable();
-
- public AbstractLSMComponent(ILSMComponentFilter filter, long mostRecentMarkerLSN) {
- this.filter = filter;
- this.mostRecentMarkerLSN = mostRecentMarkerLSN;
- readerCount = 0;
- }
public AbstractLSMComponent(ILSMComponentFilter filter) {
- this(filter, -1L);
- }
-
- public AbstractLSMComponent() {
- this(null);
+ this.filter = filter;
+ readerCount = 0;
}
@Override
@@ -57,14 +44,4 @@
public ILSMComponentFilter getLSMComponentFilter() {
return filter;
}
-
- @Override
- public long getMostRecentMarkerLSN() {
- return mostRecentMarkerLSN;
- }
-
- @Override
- public void setMostRecentMarkerLSN(long lsn) {
- this.mostRecentMarkerLSN = lsn;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractDiskLSMComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java
similarity index 82%
rename from hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractDiskLSMComponent.java
rename to hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java
index 54ec029..508a6cc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractDiskLSMComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponent.java
@@ -19,18 +19,19 @@
package org.apache.hyracks.storage.am.lsm.common.impls;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
-public abstract class AbstractDiskLSMComponent extends AbstractLSMComponent {
+public abstract class AbstractLSMDiskComponent extends AbstractLSMComponent implements ILSMDiskComponent {
- public AbstractDiskLSMComponent(ILSMComponentFilter filter) {
+ private final DiskComponentMetadata metadata;
+
+ public AbstractLSMDiskComponent(IMetadataPageManager mdPageManager, ILSMComponentFilter filter) {
super(filter);
state = ComponentState.READABLE_UNWRITABLE;
- }
-
- public AbstractDiskLSMComponent() {
- this(null);
+ metadata = new DiskComponentMetadata(mdPageManager);
}
@Override
@@ -67,11 +68,12 @@
throws HyracksDataException {
switch (opType) {
case MERGE:
- // In case two merge operations were scheduled to merge an overlapping set of components, the second merge will fail and it must
- // reset those components back to their previous state.
+ // In case two merge operations were scheduled to merge an overlapping set of components,
+ // the second merge will fail and it must reset those components back to their previous state.
if (failedOperation) {
state = ComponentState.READABLE_UNWRITABLE;
}
+ // Fallthrough
case FORCE_MODIFICATION:
case MODIFICATION:
case REPLICATE:
@@ -91,19 +93,7 @@
}
@Override
- public LSMComponentType getType() {
- return LSMComponentType.DISK;
+ public DiskComponentMetadata getMetadata() {
+ return metadata;
}
-
- @Override
- public ComponentState getState() {
- return state;
- }
-
- protected abstract void destroy() throws HyracksDataException;
-
- public abstract long getComponentSize();
-
- public abstract int getFileReferenceCount();
-
-}
+}
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index eaca234..aa23093 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -34,30 +34,32 @@
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.ComponentState;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexInternal;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
-public abstract class AbstractLSMIndex implements ILSMIndexInternal {
+public abstract class AbstractLSMIndex implements ILSMIndex {
protected final ILSMHarness lsmHarness;
protected final IIOManager ioManager;
protected final ILSMIOOperationScheduler ioScheduler;
protected final ILSMIOOperationCallback ioOpCallback;
// In-memory components.
- protected final List<ILSMComponent> memoryComponents;
+ protected final List<ILSMMemoryComponent> memoryComponents;
protected final List<IVirtualBufferCache> virtualBufferCaches;
protected AtomicInteger currentMutableComponentId;
@@ -65,8 +67,9 @@
protected final IBufferCache diskBufferCache;
protected final ILSMIndexFileManager fileManager;
protected final IFileMapProvider diskFileMapProvider;
- protected final List<ILSMComponent> diskComponents;
- protected final List<ILSMComponent> inactiveDiskComponents;
+ // components with lower indexes are newer than components with higher index
+ protected final List<ILSMDiskComponent> diskComponents;
+ protected final List<ILSMDiskComponent> inactiveDiskComponents;
protected final double bloomFilterFalsePositiveRate;
protected final ILSMComponentFilterFrameFactory filterFrameFactory;
protected final LSMComponentFilterManager filterManager;
@@ -77,11 +80,11 @@
protected boolean memoryComponentsAllocated = false;
public AbstractLSMIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- IBufferCache diskBufferCache,
- ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFrameFactory filterFrameFactory,
- LSMComponentFilterManager filterManager, int[] filterFields, boolean durable) {
+ IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider,
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
+ int[] filterFields, boolean durable) {
this.ioManager = ioManager;
this.virtualBufferCaches = virtualBufferCaches;
this.diskBufferCache = diskBufferCache;
@@ -154,12 +157,12 @@
}
@Override
- public void addComponent(ILSMComponent c) throws HyracksDataException {
+ public void addDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
diskComponents.add(0, c);
}
@Override
- public void subsumeMergedComponents(ILSMComponent newComponent, List<ILSMComponent> mergedComponents)
+ public void subsumeMergedComponents(ILSMDiskComponent newComponent, List<ILSMComponent> mergedComponents)
throws HyracksDataException {
int swapIndex = diskComponents.indexOf(mergedComponents.get(0));
diskComponents.removeAll(mergedComponents);
@@ -169,11 +172,11 @@
@Override
public void changeMutableComponent() {
currentMutableComponentId.set((currentMutableComponentId.get() + 1) % memoryComponents.size());
- ((AbstractMemoryLSMComponent) memoryComponents.get(currentMutableComponentId.get())).setActive();
+ memoryComponents.get(currentMutableComponentId.get()).activate();
}
@Override
- public List<ILSMComponent> getImmutableComponents() {
+ public List<ILSMDiskComponent> getImmutableComponents() {
return diskComponents;
}
@@ -210,7 +213,7 @@
public boolean isEmptyIndex() {
boolean isModified = false;
for (ILSMComponent c : memoryComponents) {
- AbstractMemoryLSMComponent mutableComponent = (AbstractMemoryLSMComponent) c;
+ AbstractLSMMemoryComponent mutableComponent = (AbstractLSMMemoryComponent) c;
if (mutableComponent.isModified()) {
isModified = true;
break;
@@ -232,36 +235,36 @@
@Override
public boolean isCurrentMutableComponentEmpty() throws HyracksDataException {
//check if the current memory component has been modified
- return !((AbstractMemoryLSMComponent) memoryComponents.get(currentMutableComponentId.get())).isModified();
+ return !memoryComponents.get(currentMutableComponentId.get()).isModified();
}
public void setCurrentMutableComponentState(ComponentState componentState) {
- ((AbstractMemoryLSMComponent) memoryComponents.get(currentMutableComponentId.get())).setState(componentState);
+ memoryComponents.get(currentMutableComponentId.get()).setState(componentState);
}
public ComponentState getCurrentMutableComponentState() {
- return ((AbstractMemoryLSMComponent) memoryComponents.get(currentMutableComponentId.get())).getState();
+ return memoryComponents.get(currentMutableComponentId.get()).getState();
}
public int getCurrentMutableComponentWriterCount() {
- return ((AbstractMemoryLSMComponent) memoryComponents.get(currentMutableComponentId.get())).getWriterCount();
+ return memoryComponents.get(currentMutableComponentId.get()).getWriterCount();
}
@Override
- public List<ILSMComponent> getInactiveDiskComponents() {
+ public List<ILSMDiskComponent> getInactiveDiskComponents() {
return inactiveDiskComponents;
}
@Override
- public void addInactiveDiskComponent(ILSMComponent diskComponent) {
+ public void addInactiveDiskComponent(ILSMDiskComponent diskComponent) {
inactiveDiskComponents.add(diskComponent);
}
public abstract Set<String> getLSMComponentPhysicalFiles(ILSMComponent newComponent);
@Override
- public void scheduleReplication(ILSMIndexOperationContext ctx, List<ILSMComponent> lsmComponents, boolean bulkload,
- ReplicationOperation operation, LSMOperationType opType) throws HyracksDataException {
+ public void scheduleReplication(ILSMIndexOperationContext ctx, List<ILSMDiskComponent> lsmComponents,
+ boolean bulkload, ReplicationOperation operation, LSMOperationType opType) throws HyracksDataException {
//get set of files to be replicated for this component
Set<String> componentFiles = new HashSet<>();
@@ -278,8 +281,8 @@
}
//create replication job and submit it
- LSMIndexReplicationJob job = new LSMIndexReplicationJob(this, ctx, componentFiles, operation, executionType,
- opType);
+ LSMIndexReplicationJob job =
+ new LSMIndexReplicationJob(this, ctx, componentFiles, operation, executionType, opType);
try {
diskBufferCache.getIOReplicationManager().submitJob(job);
} catch (IOException e) {
@@ -300,7 +303,18 @@
return durable;
}
- public ILSMComponent getCurrentMemoryComponent() {
+ @Override
+ public ILSMMemoryComponent getCurrentMemoryComponent() {
return memoryComponents.get(currentMutableComponentId.get());
}
+
+ @Override
+ public int getCurrentMemoryComponentIndex() {
+ return currentMutableComponentId.get();
+ }
+
+ @Override
+ public List<ILSMMemoryComponent> getMemoryComponents() {
+ return memoryComponents;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractMemoryLSMComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java
similarity index 87%
rename from hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractMemoryLSMComponent.java
rename to hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java
index 500996f..bfe7fc0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractMemoryLSMComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMMemoryComponent.java
@@ -22,19 +22,20 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
-public abstract class AbstractMemoryLSMComponent extends AbstractLSMComponent {
+public abstract class AbstractLSMMemoryComponent extends AbstractLSMComponent implements ILSMMemoryComponent {
private final IVirtualBufferCache vbc;
private final AtomicBoolean isModified;
private int writerCount;
private boolean requestedToBeActive;
+ private final MemoryComponentMetadata metadata;
- public AbstractMemoryLSMComponent(IVirtualBufferCache vbc, boolean isActive, ILSMComponentFilter filter,
- long mostRecentMarkerLSN) {
- super(filter, mostRecentMarkerLSN);
+ public AbstractLSMMemoryComponent(IVirtualBufferCache vbc, boolean isActive, ILSMComponentFilter filter) {
+ super(filter);
this.vbc = vbc;
writerCount = 0;
if (isActive) {
@@ -43,14 +44,7 @@
state = ComponentState.INACTIVE;
}
isModified = new AtomicBoolean();
- }
-
- public AbstractMemoryLSMComponent(IVirtualBufferCache vbc, boolean isActive, ILSMComponentFilter filter) {
- this(vbc, isActive, filter, -1L);
- }
-
- public AbstractMemoryLSMComponent(IVirtualBufferCache vbc, boolean isActive) {
- this(vbc, isActive, null);
+ metadata = new MemoryComponentMetadata();
}
@Override
@@ -126,7 +120,8 @@
case MODIFICATION:
if (isMutableComponent) {
writerCount--;
- //A failed operation should not change the component state since it's better for the failed operation's effect to be no-op.
+ // A failed operation should not change the component state since it's better for
+ // the failed operation's effect to be no-op.
if (state == ComponentState.READABLE_WRITABLE && !failedOperation && isFull()) {
state = ComponentState.READABLE_UNWRITABLE;
}
@@ -164,6 +159,7 @@
}
}
+ @Override
public boolean isReadable() {
if (state == ComponentState.INACTIVE || state == ComponentState.UNREADABLE_UNWRITABLE) {
return false;
@@ -172,43 +168,43 @@
}
@Override
- public LSMComponentType getType() {
- return LSMComponentType.MEMORY;
- }
-
- @Override
- public ComponentState getState() {
- return state;
- }
-
public void setState(ComponentState state) {
this.state = state;
}
- public void setActive() {
+ @Override
+ public void activate() {
requestedToBeActive = true;
}
- public void setIsModified() {
+ @Override
+ public void setModified() {
isModified.set(true);
}
+ @Override
public boolean isModified() {
return isModified.get();
}
+ @Override
public boolean isFull() {
return vbc.isFull();
}
- protected void reset() throws HyracksDataException {
+ @Override
+ public void reset() throws HyracksDataException {
isModified.set(false);
- if (filter != null) {
- filter.reset();
- }
+ metadata.reset();
}
+ @Override
public int getWriterCount() {
return writerCount;
}
+
+ @Override
+ public MemoryComponentMetadata getMetadata() {
+ return metadata;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
index 6164c0a..562ed5a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
@@ -25,7 +25,6 @@
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
@@ -39,7 +38,7 @@
}
@Override
- public BTree createIndexInstance(FileReference file) throws IndexException {
+ public BTree createIndexInstance(FileReference file) {
return new BTree(bufferCache, fileMapProvider, freePageManagerFactory.createPageManager(bufferCache),
interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, file);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BlockingIOOperationCallbackWrapper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BlockingIOOperationCallbackWrapper.java
index a9a2129..8ae535b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BlockingIOOperationCallbackWrapper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BlockingIOOperationCallbackWrapper.java
@@ -22,6 +22,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
@@ -48,13 +49,14 @@
}
@Override
- public void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents, ILSMComponent newComponent)
- throws HyracksDataException {
+ public void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents,
+ ILSMDiskComponent newComponent) throws HyracksDataException {
wrappedCallback.afterOperation(opType, oldComponents, newComponent);
}
@Override
- public synchronized void afterFinalize(LSMOperationType opType, ILSMComponent newComponent) throws HyracksDataException {
+ public synchronized void afterFinalize(LSMOperationType opType, ILSMDiskComponent newComponent)
+ throws HyracksDataException {
wrappedCallback.afterFinalize(opType, newComponent);
notifyAll();
notified = true;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ConstantMergePolicy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ConstantMergePolicy.java
index 7340fdb..b3e1f6f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ConstantMergePolicy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ConstantMergePolicy.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.ComponentState;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
@@ -37,19 +38,19 @@
@Override
public void diskComponentAdded(final ILSMIndex index, boolean fullMergeIsRequested)
throws HyracksDataException, IndexException {
- List<ILSMComponent> immutableComponents = index.getImmutableComponents();
+ List<ILSMDiskComponent> immutableComponents = index.getImmutableComponents();
if (!areComponentsMergable(immutableComponents)) {
return;
}
if (fullMergeIsRequested) {
- ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
+ ILSMIndexAccessor accessor =
+ index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
accessor.scheduleFullMerge(index.getIOOperationCallback());
} else if (immutableComponents.size() >= numComponents) {
- ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
+ ILSMIndexAccessor accessor =
+ index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
accessor.scheduleMerge(index.getIOOperationCallback(), immutableComponents);
}
}
@@ -85,7 +86,7 @@
* there will be no new merge either in this situation.
*/
- List<ILSMComponent> immutableComponents = index.getImmutableComponents();
+ List<ILSMDiskComponent> immutableComponents = index.getImmutableComponents();
int totalImmutableComponentCount = immutableComponents.size();
// [case 1]
@@ -105,8 +106,8 @@
if (!areComponentsMergable(immutableComponents)) {
throw new IllegalStateException();
}
- ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
+ ILSMIndexAccessor accessor =
+ index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
accessor.scheduleMerge(index.getIOOperationCallback(), immutableComponents);
return true;
}
@@ -118,7 +119,7 @@
* @param immutableComponents
* @return true if all components are mergable, false otherwise.
*/
- private boolean areComponentsMergable(List<ILSMComponent> immutableComponents) {
+ private boolean areComponentsMergable(List<ILSMDiskComponent> immutableComponents) {
for (ILSMComponent c : immutableComponents) {
if (c.getState() != ComponentState.READABLE_UNWRITABLE) {
return false;
@@ -133,7 +134,7 @@
*
* @return true if there is an ongoing merge operation, false otherwise.
*/
- private boolean isMergeOngoing(List<ILSMComponent> immutableComponents) {
+ private boolean isMergeOngoing(List<ILSMDiskComponent> immutableComponents) {
int size = immutableComponents.size();
for (int i = 0; i < size; i++) {
if (immutableComponents.get(i).getState() == ComponentState.READABLE_MERGING) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/DiskComponentMetadata.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/DiskComponentMetadata.java
new file mode 100644
index 0000000..2da954d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/DiskComponentMetadata.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.common.impls;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
+import org.apache.hyracks.storage.am.lsm.common.api.IComponentMetadata;
+
+public class DiskComponentMetadata implements IComponentMetadata {
+
+ private final IMetadataPageManager mdpManager;
+
+ public DiskComponentMetadata(IMetadataPageManager mdpManager) {
+ this.mdpManager = mdpManager;
+ }
+
+ @Override
+ public void put(IValueReference key, IValueReference value) throws HyracksDataException {
+ mdpManager.put(mdpManager.createMetadataFrame(), key, value);
+ }
+
+ @Override
+ public void get(IValueReference key, IPointable value) throws HyracksDataException {
+ mdpManager.get(mdpManager.createMetadataFrame(), key, value);
+ }
+
+ @Override
+ public IValueReference get(IValueReference key) throws HyracksDataException {
+ IPointable value = VoidPointable.FACTORY.createPointable();
+ get(key, value);
+ return value;
+ }
+
+ public void put(MemoryComponentMetadata metadata) throws HyracksDataException {
+ metadata.copy(mdpManager);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ExternalIndexHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ExternalIndexHarness.java
index e4be66bb0..c6346cc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ExternalIndexHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ExternalIndexHarness.java
@@ -29,10 +29,11 @@
import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexInternal;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
@@ -42,7 +43,7 @@
public class ExternalIndexHarness extends LSMHarness {
private static final Logger LOGGER = Logger.getLogger(ExternalIndexHarness.class.getName());
- public ExternalIndexHarness(ILSMIndexInternal lsmIndex, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ public ExternalIndexHarness(ILSMIndex lsmIndex, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
boolean replicationEnabled) {
super(lsmIndex, mergePolicy, opTracker, replicationEnabled);
}
@@ -112,7 +113,7 @@
return true;
}
- private void exitComponents(ILSMIndexOperationContext ctx, LSMOperationType opType, ILSMComponent newComponent,
+ private void exitComponents(ILSMIndexOperationContext ctx, LSMOperationType opType, ILSMDiskComponent newComponent,
boolean failedOperation) throws HyracksDataException, IndexException {
/**
* FLUSH and MERGE operations should always exit the components
@@ -130,11 +131,11 @@
case INACTIVE:
if (replicationEnabled) {
componentsToBeReplicated.clear();
- componentsToBeReplicated.add(c);
+ componentsToBeReplicated.add((ILSMDiskComponent) c);
lsmIndex.scheduleReplication(null, componentsToBeReplicated, false,
ReplicationOperation.DELETE, opType);
}
- ((AbstractDiskLSMComponent) c).destroy();
+ ((ILSMDiskComponent) c).destroy();
break;
default:
break;
@@ -233,7 +234,7 @@
LOGGER.info("Started a merge operation for index: " + lsmIndex + " ...");
}
- ILSMComponent newComponent = null;
+ ILSMDiskComponent newComponent = null;
try {
newComponent = lsmIndex.merge(operation);
operation.getCallback().afterOperation(LSMOperationType.MERGE, ctx.getComponentHolder(), newComponent);
@@ -248,10 +249,10 @@
}
@Override
- public void addBulkLoadedComponent(ILSMComponent c) throws HyracksDataException, IndexException {
+ public void addBulkLoadedComponent(ILSMDiskComponent c) throws HyracksDataException, IndexException {
lsmIndex.markAsValid(c);
synchronized (opTracker) {
- lsmIndex.addComponent(c);
+ lsmIndex.addDiskComponent(c);
if (replicationEnabled) {
componentsToBeReplicated.clear();
componentsToBeReplicated.add(c);
@@ -267,11 +268,11 @@
// 1. this needs synchronization since others might be accessing the index (specifically merge operations that might change the lists of components)
// 2. the actions taken by the index itself are different
// 3. the component has already been marked valid by the bulk update operation
- public void addTransactionComponents(ILSMComponent newComponent) throws HyracksDataException, IndexException {
+ public void addTransactionComponents(ILSMDiskComponent newComponent) throws HyracksDataException, IndexException {
ITwoPCIndex index = (ITwoPCIndex) lsmIndex;
synchronized (opTracker) {
- List<ILSMComponent> newerList;
- List<ILSMComponent> olderList;
+ List<ILSMDiskComponent> newerList;
+ List<ILSMDiskComponent> olderList;
if (index.getCurrentVersion() == 0) {
newerList = index.getFirstComponentList();
olderList = index.getSecondComponentList();
@@ -281,11 +282,11 @@
}
// Exit components in old version of the index so they are ready to be
// deleted if they are not needed anymore
- for (ILSMComponent c : olderList) {
+ for (ILSMDiskComponent c : olderList) {
exitComponent(c);
}
// Enter components in the newer list
- for (ILSMComponent c : newerList) {
+ for (ILSMDiskComponent c : newerList) {
enterComponent(c);
}
if (newComponent != null) {
@@ -320,7 +321,7 @@
if (index.getFirstComponentList().containsAll(mergedComponents)) {
// exit un-needed components
for (ILSMComponent c : mergedComponents) {
- exitComponent(c);
+ exitComponent((ILSMDiskComponent) c);
}
// enter new component
enterComponent(newComponent);
@@ -329,7 +330,7 @@
if (index.getSecondComponentList().containsAll(mergedComponents)) {
// exit un-needed components
for (ILSMComponent c : mergedComponents) {
- exitComponent(c);
+ exitComponent((ILSMDiskComponent) c);
}
// enter new component
enterComponent(newComponent);
@@ -342,7 +343,7 @@
diskComponent.threadEnter(LSMOperationType.SEARCH, false);
}
- private void exitComponent(ILSMComponent diskComponent) throws HyracksDataException {
+ private void exitComponent(ILSMDiskComponent diskComponent) throws HyracksDataException {
diskComponent.threadExit(LSMOperationType.SEARCH, false, false);
if (diskComponent.getState() == ILSMComponent.ComponentState.INACTIVE) {
if (replicationEnabled) {
@@ -350,7 +351,7 @@
componentsToBeReplicated.add(diskComponent);
lsmIndex.scheduleReplication(null, componentsToBeReplicated, false, ReplicationOperation.DELETE, null);
}
- ((AbstractDiskLSMComponent) diskComponent).destroy();
+ diskComponent.destroy();
}
}
@@ -368,10 +369,10 @@
public void indexClear() throws HyracksDataException {
ITwoPCIndex index = (ITwoPCIndex) lsmIndex;
- for (ILSMComponent c : index.getFirstComponentList()) {
+ for (ILSMDiskComponent c : index.getFirstComponentList()) {
exitComponent(c);
}
- for (ILSMComponent c : index.getSecondComponentList()) {
+ for (ILSMDiskComponent c : index.getSecondComponentList()) {
exitComponent(c);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java
index f6d21d8..1f7d7a3 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java
@@ -24,7 +24,6 @@
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.IIndex;
import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
-import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
@@ -43,7 +42,7 @@
this.freePageManagerFactory = freePageManagerFactory;
}
- public abstract T createIndexInstance(FileReference file) throws IndexException, HyracksDataException;
+ public abstract T createIndexInstance(FileReference file) throws HyracksDataException;
public IBufferCache getBufferCache() {
return bufferCache;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterFactory.java
index b69ecf2..46a629a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterFactory.java
@@ -35,7 +35,7 @@
}
@Override
- public ILSMComponentFilter createLSMComponentFilter() {
+ public ILSMComponentFilter createFilter() {
return new LSMComponentFilter(tupleWriterFactory.createTupleWriter(), filterCmpFactories);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java
index a8b4c70..7f8e990 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java
@@ -42,7 +42,7 @@
}
@Override
- public void updateFilterInfo(ILSMComponentFilter filter, List<ITupleReference> filterTuples)
+ public void updateFilter(ILSMComponentFilter filter, List<ITupleReference> filterTuples)
throws HyracksDataException {
MultiComparator filterCmp = MultiComparator.create(filter.getFilterCmpFactories());
for (ITupleReference tuple : filterTuples) {
@@ -51,7 +51,7 @@
}
@Override
- public void writeFilterInfo(ILSMComponentFilter filter, ITreeIndex treeIndex) throws HyracksDataException {
+ public void writeFilter(ILSMComponentFilter filter, ITreeIndex treeIndex) throws HyracksDataException {
IMetadataPageManager treeMetaManager = (IMetadataPageManager) treeIndex.getPageManager();
ILSMComponentFilterReference filterFrame = filterFrameFactory.createFrame();
try {
@@ -67,7 +67,7 @@
}
@Override
- public boolean readFilterInfo(ILSMComponentFilter filter, ITreeIndex treeIndex) throws HyracksDataException {
+ public boolean readFilter(ILSMComponentFilter filter, ITreeIndex treeIndex) throws HyracksDataException {
IMetadataPageManager treeMetaManager = (IMetadataPageManager) treeIndex.getPageManager();
ILSMComponentFilterReference filterFrame = filterFrameFactory.createFrame();
treeMetaManager.get(treeMetaManager.createMetadataFrame(), FILTER_KEY, filterFrame);
@@ -78,7 +78,7 @@
List<ITupleReference> filterTuples = new ArrayList<>();
filterTuples.add(filterFrame.getMinTuple());
filterTuples.add(filterFrame.getMaxTuple());
- updateFilterInfo(filter, filterTuples);
+ updateFilter(filter, filterTuples);
return true;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index a6868f0..6bf9312 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -34,14 +34,15 @@
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.ComponentState;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexInternal;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
@@ -50,14 +51,14 @@
public class LSMHarness implements ILSMHarness {
private static final Logger LOGGER = Logger.getLogger(LSMHarness.class.getName());
- protected final ILSMIndexInternal lsmIndex;
+ protected final ILSMIndex lsmIndex;
protected final ILSMMergePolicy mergePolicy;
protected final ILSMOperationTracker opTracker;
protected final AtomicBoolean fullMergeIsRequested;
protected final boolean replicationEnabled;
- protected List<ILSMComponent> componentsToBeReplicated;
+ protected List<ILSMDiskComponent> componentsToBeReplicated;
- public LSMHarness(ILSMIndexInternal lsmIndex, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ public LSMHarness(ILSMIndex lsmIndex, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
boolean replicationEnabled) {
this.lsmIndex = lsmIndex;
this.opTracker = opTracker;
@@ -80,7 +81,7 @@
switch (opType) {
case FLUSH:
ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
- if (!((AbstractMemoryLSMComponent) flushingComponent).isModified()) {
+ if (!((AbstractLSMMemoryComponent) flushingComponent).isModified()) {
//The mutable component has not been modified by any writer. There is nothing to flush.
//since the component is empty, set its state back to READABLE_WRITABLE
if (((AbstractLSMIndex) lsmIndex)
@@ -90,7 +91,7 @@
}
return false;
}
- if (((AbstractMemoryLSMComponent) flushingComponent).getWriterCount() > 0) {
+ if (((AbstractLSMMemoryComponent) flushingComponent).getWriterCount() > 0) {
/*
* This case is a case where even though FLUSH log was flushed to disk and scheduleFlush is triggered,
* the current in-memory component (whose state was changed to READABLE_WRITABLE (RW)
@@ -172,11 +173,7 @@
lsmIndex.getIOOperationCallback().beforeOperation(LSMOperationType.FLUSH);
// Changing the flush status should *always* precede changing the mutable component.
lsmIndex.changeFlushStatusForCurrentMutableCompoent(false);
- // Flushing! => carry over the marker lsn to the next component
- long mostRecentMarkerLSN =
- ((AbstractLSMIndex) lsmIndex).getCurrentMemoryComponent().getMostRecentMarkerLSN();
lsmIndex.changeMutableComponent();
- ((AbstractLSMIndex) lsmIndex).getCurrentMemoryComponent().setMostRecentMarkerLSN(mostRecentMarkerLSN);
// Notify all waiting threads whenever a flush has been scheduled since they will check
// again if they can grab and enter the mutable component.
opTracker.notifyAll();
@@ -190,7 +187,7 @@
return true;
}
- private void exitComponents(ILSMIndexOperationContext ctx, LSMOperationType opType, ILSMComponent newComponent,
+ private void exitComponents(ILSMIndexOperationContext ctx, LSMOperationType opType, ILSMDiskComponent newComponent,
boolean failedOperation) throws HyracksDataException, IndexException {
/**
* FLUSH and MERGE operations should always exit the components
@@ -199,8 +196,8 @@
if (!ctx.isAccessingComponents() && opType != LSMOperationType.FLUSH && opType != LSMOperationType.MERGE) {
return;
}
- List<ILSMComponent> inactiveDiskComponents = null;
- List<ILSMComponent> inactiveDiskComponentsToBeDeleted = null;
+ List<ILSMDiskComponent> inactiveDiskComponents = null;
+ List<ILSMDiskComponent> inactiveDiskComponentsToBeDeleted = null;
try {
synchronized (opTracker) {
try {
@@ -237,7 +234,7 @@
}
break;
case INACTIVE:
- ((AbstractMemoryLSMComponent) c).reset();
+ ((AbstractLSMMemoryComponent) c).reset();
// Notify all waiting threads whenever the mutable component's has change to inactive. This is important because
// even though we switched the mutable components, it is possible that the component that we just switched
// to is still busy flushing its data to disk. Thus, the notification that was issued upon scheduling the flush
@@ -250,7 +247,7 @@
} else {
switch (c.getState()) {
case INACTIVE:
- lsmIndex.addInactiveDiskComponent(c);
+ lsmIndex.addInactiveDiskComponent((AbstractLSMDiskComponent) c);
break;
default:
break;
@@ -264,7 +261,7 @@
case FLUSH:
// newComponent is null if the flush op. was not performed.
if (newComponent != null) {
- lsmIndex.addComponent(newComponent);
+ lsmIndex.addDiskComponent(newComponent);
if (replicationEnabled) {
componentsToBeReplicated.clear();
componentsToBeReplicated.add(newComponent);
@@ -311,8 +308,8 @@
*/
inactiveDiskComponents = lsmIndex.getInactiveDiskComponents();
if (!inactiveDiskComponents.isEmpty()) {
- for (ILSMComponent inactiveComp : inactiveDiskComponents) {
- if (((AbstractDiskLSMComponent) inactiveComp).getFileReferenceCount() == 1) {
+ for (ILSMDiskComponent inactiveComp : inactiveDiskComponents) {
+ if (inactiveComp.getFileReferenceCount() == 1) {
if (inactiveDiskComponentsToBeDeleted == null) {
inactiveDiskComponentsToBeDeleted = new LinkedList<>();
}
@@ -338,7 +335,7 @@
}
for (ILSMComponent c : inactiveDiskComponentsToBeDeleted) {
- ((AbstractDiskLSMComponent) c).destroy();
+ ((AbstractLSMDiskComponent) c).destroy();
}
} catch (Throwable e) {
e.printStackTrace();
@@ -375,8 +372,8 @@
try {
lsmIndex.modify(ctx, tuple);
// The mutable component is always in the first index.
- AbstractMemoryLSMComponent mutableComponent = (AbstractMemoryLSMComponent) ctx.getComponentHolder().get(0);
- mutableComponent.setIsModified();
+ AbstractLSMMemoryComponent mutableComponent = (AbstractLSMMemoryComponent) ctx.getComponentHolder().get(0);
+ mutableComponent.setModified();
} catch (Exception e) {
failedOperation = true;
throw e;
@@ -429,7 +426,7 @@
LOGGER.info("Started a flush operation for index: " + lsmIndex + " ...");
}
- ILSMComponent newComponent = null;
+ ILSMDiskComponent newComponent = null;
try {
newComponent = lsmIndex.flush(operation);
operation.getCallback().afterOperation(LSMOperationType.FLUSH, null, newComponent);
@@ -477,7 +474,7 @@
LOGGER.info("Started a merge operation for index: " + lsmIndex + " ...");
}
- ILSMComponent newComponent = null;
+ ILSMDiskComponent newComponent = null;
try {
newComponent = lsmIndex.merge(operation);
operation.getCallback().afterOperation(LSMOperationType.MERGE, ctx.getComponentHolder(), newComponent);
@@ -495,10 +492,10 @@
}
@Override
- public void addBulkLoadedComponent(ILSMComponent c) throws HyracksDataException, IndexException {
+ public void addBulkLoadedComponent(ILSMDiskComponent c) throws HyracksDataException, IndexException {
lsmIndex.markAsValid(c);
synchronized (opTracker) {
- lsmIndex.addComponent(c);
+ lsmIndex.addDiskComponent(c);
if (replicationEnabled) {
componentsToBeReplicated.clear();
componentsToBeReplicated.add(c);
@@ -513,16 +510,16 @@
return opTracker;
}
- protected void triggerReplication(List<ILSMComponent> lsmComponents, boolean bulkload, LSMOperationType opType)
+ protected void triggerReplication(List<ILSMDiskComponent> lsmComponents, boolean bulkload, LSMOperationType opType)
throws HyracksDataException {
- ILSMIndexAccessorInternal accessor =
+ ILSMIndexAccessor accessor =
lsmIndex.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
accessor.scheduleReplication(lsmComponents, bulkload, opType);
}
@Override
- public void scheduleReplication(ILSMIndexOperationContext ctx, List<ILSMComponent> lsmComponents, boolean bulkload,
- LSMOperationType opType) throws HyracksDataException {
+ public void scheduleReplication(ILSMIndexOperationContext ctx, List<ILSMDiskComponent> lsmComponents,
+ boolean bulkload, LSMOperationType opType) throws HyracksDataException {
//enter the LSM components to be replicated to prevent them from being deleted until they are replicated
if (!getAndEnterComponents(ctx, LSMOperationType.REPLICATE, false)) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
index 92155e4..4199cfb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
@@ -27,15 +27,15 @@
import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
-public abstract class LSMTreeIndexAccessor implements ILSMIndexAccessorInternal {
+public abstract class LSMTreeIndexAccessor implements ILSMIndexAccessor {
protected ILSMHarness lsmHarness;
protected ILSMIndexOperationContext ctx;
@@ -124,7 +124,7 @@
}
@Override
- public void scheduleMerge(ILSMIOOperationCallback callback, List<ILSMComponent> components)
+ public void scheduleMerge(ILSMIOOperationCallback callback, List<ILSMDiskComponent> components)
throws HyracksDataException, IndexException {
ctx.setOperation(IndexOperation.MERGE);
ctx.getComponentsToBeMerged().clear();
@@ -133,7 +133,7 @@
}
@Override
- public void scheduleReplication(List<ILSMComponent> lsmComponents, boolean bulkload, LSMOperationType opType)
+ public void scheduleReplication(List<ILSMDiskComponent> lsmComponents, boolean bulkload, LSMOperationType opType)
throws HyracksDataException {
ctx.setOperation(IndexOperation.REPLICATE);
ctx.getComponentsToBeReplicated().clear();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MemoryComponentMetadata.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MemoryComponentMetadata.java
new file mode 100644
index 0000000..dcc9355
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MemoryComponentMetadata.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.common.impls;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
+import org.apache.hyracks.storage.am.lsm.common.api.IComponentMetadata;
+
+public class MemoryComponentMetadata implements IComponentMetadata {
+ private static final byte[] empty = new byte[0];
+ private final List<org.apache.commons.lang3.tuple.Pair<IValueReference, ArrayBackedValueStorage>> store =
+ new ArrayList<>();
+
+ /**
+ * Note: for memory metadata, it is expected that the key will be constant
+ */
+ @Override
+ public void put(IValueReference key, IValueReference value) {
+ ArrayBackedValueStorage stored = get(key);
+ if (stored == null) {
+ stored = new ArrayBackedValueStorage();
+ }
+ stored.assign(value);
+ store.add(Pair.of(key, stored));
+ }
+
+ /**
+ * Note: for memory metadata, it is expected that the key will be constant
+ */
+ @Override
+ public void get(IValueReference key, IPointable value) {
+ value.set(empty, 0, 0);
+ ArrayBackedValueStorage stored = get(key);
+ if (stored != null) {
+ value.set(stored);
+ }
+ }
+
+ @Override
+ public ArrayBackedValueStorage get(IValueReference key) {
+ for (Pair<IValueReference, ArrayBackedValueStorage> pair : store) {
+ if (pair.getKey().equals(key)) {
+ return pair.getValue();
+ }
+ }
+ return null;
+ }
+
+ public void copy(IMetadataPageManager mdpManager) throws HyracksDataException {
+ ITreeIndexMetadataFrame frame = mdpManager.createMetadataFrame();
+ for (Pair<IValueReference, ArrayBackedValueStorage> pair : store) {
+ mdpManager.put(frame, pair.getKey(), pair.getValue());
+ }
+ }
+
+ public void copy(DiskComponentMetadata metadata) throws HyracksDataException {
+ metadata.put(this);
+ }
+
+ public void reset() {
+ store.clear();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIOOperationCallback.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIOOperationCallback.java
index 0713d7d..dc69092 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIOOperationCallback.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIOOperationCallback.java
@@ -22,13 +22,15 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
-public enum NoOpIOOperationCallback implements ILSMIOOperationCallback, ILSMIOOperationCallbackProvider, ILSMIOOperationCallbackFactory {
+public enum NoOpIOOperationCallback
+ implements ILSMIOOperationCallback, ILSMIOOperationCallbackProvider, ILSMIOOperationCallbackFactory {
INSTANCE;
@Override
@@ -37,13 +39,13 @@
}
@Override
- public void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents, ILSMComponent newComponent)
- throws HyracksDataException {
+ public void afterOperation(LSMOperationType opType, List<ILSMComponent> oldComponents,
+ ILSMDiskComponent newComponent) throws HyracksDataException {
// Do nothing.
}
@Override
- public void afterFinalize(LSMOperationType opType, ILSMComponent newComponent) throws HyracksDataException {
+ public void afterFinalize(LSMOperationType opType, ILSMDiskComponent newComponent) throws HyracksDataException {
// Do nothing.
}
@@ -53,7 +55,7 @@
}
@Override
- public ILSMIOOperationCallback createIOOperationCallback() {
+ public ILSMIOOperationCallback createIoOpCallback() {
return INSTANCE;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/PrefixMergePolicy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/PrefixMergePolicy.java
index 36cb958..5f36339 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/PrefixMergePolicy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/PrefixMergePolicy.java
@@ -27,6 +27,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.ComponentState;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
@@ -42,15 +43,15 @@
public void diskComponentAdded(final ILSMIndex index, boolean fullMergeIsRequested)
throws HyracksDataException, IndexException {
- List<ILSMComponent> immutableComponents = new ArrayList<ILSMComponent>(index.getImmutableComponents());
+ ArrayList<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getImmutableComponents());
if (!areComponentsReadableWritableState(immutableComponents)) {
return;
}
if (fullMergeIsRequested) {
- ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
+ ILSMIndexAccessor accessor =
+ index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
accessor.scheduleFullMerge(index.getIOOperationCallback());
return;
}
@@ -130,7 +131,7 @@
* there will be no new merge either in this situation.
*/
- List<ILSMComponent> immutableComponents = index.getImmutableComponents();
+ List<ILSMDiskComponent> immutableComponents = index.getImmutableComponents();
int mergableImmutableComponentCount = getMergableImmutableComponentCount(immutableComponents);
// [case 1]
@@ -166,7 +167,7 @@
* @param immutableComponents
* @return true if there is an ongoing merge operation, false otherwise.
*/
- private boolean isMergeOngoing(List<ILSMComponent> immutableComponents) {
+ private boolean isMergeOngoing(List<ILSMDiskComponent> immutableComponents) {
int size = immutableComponents.size();
for (int i = 0; i < size; i++) {
if (immutableComponents.get(i).getState() == ComponentState.READABLE_MERGING) {
@@ -184,10 +185,10 @@
* @param immutableComponents
* @return the number of mergable component
*/
- private int getMergableImmutableComponentCount(List<ILSMComponent> immutableComponents) {
+ private int getMergableImmutableComponentCount(List<ILSMDiskComponent> immutableComponents) {
int count = 0;
for (ILSMComponent c : immutableComponents) {
- long componentSize = ((AbstractDiskLSMComponent) c).getComponentSize();
+ long componentSize = ((ILSMDiskComponent) c).getComponentSize();
//stop when the first non-mergable component is found.
if (c.getState() != ComponentState.READABLE_UNWRITABLE || componentSize > maxMergableComponentSize) {
break;
@@ -203,7 +204,7 @@
* @param immutableComponents
* @return true if all components are of READABLE_UNWRITABLE state, false otherwise.
*/
- private boolean areComponentsReadableWritableState(List<ILSMComponent> immutableComponents) {
+ private boolean areComponentsReadableWritableState(List<ILSMDiskComponent> immutableComponents) {
for (ILSMComponent c : immutableComponents) {
if (c.getState() != ComponentState.READABLE_UNWRITABLE) {
return false;
@@ -225,7 +226,7 @@
// all such components for which the sum of their sizes exceeds MaxMrgCompSz. Schedule a merge of those components into a new component.
// 2. If a merge from 1 doesn't happen, see if the set of candidate components for merging exceeds MaxTolCompCnt. If so, schedule
// a merge all of the current candidates into a new single component.
- List<ILSMComponent> immutableComponents = new ArrayList<ILSMComponent>(index.getImmutableComponents());
+ List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getImmutableComponents());
// Reverse the components order so that we look at components from oldest to newest.
Collections.reverse(immutableComponents);
@@ -233,7 +234,7 @@
int startIndex = -1;
for (int i = 0; i < immutableComponents.size(); i++) {
ILSMComponent c = immutableComponents.get(i);
- long componentSize = ((AbstractDiskLSMComponent) c).getComponentSize();
+ long componentSize = ((ILSMDiskComponent) c).getComponentSize();
if (componentSize > maxMergableComponentSize) {
startIndex = i;
totalSize = 0;
@@ -243,14 +244,14 @@
boolean isLastComponent = i + 1 == immutableComponents.size() ? true : false;
if (totalSize > maxMergableComponentSize
|| (isLastComponent && i - startIndex >= maxToleranceComponentCount)) {
- List<ILSMComponent> mergableComponents = new ArrayList<ILSMComponent>();
+ List<ILSMDiskComponent> mergableComponents = new ArrayList<>();
for (int j = startIndex + 1; j <= i; j++) {
mergableComponents.add(immutableComponents.get(j));
}
// Reverse the components order back to its original order
Collections.reverse(mergableComponents);
- ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
+ ILSMIndexAccessor accessor =
+ index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
accessor.scheduleMerge(index.getIOOperationCallback(), mergableComponents);
return true;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java
index 9b255d7..d5a8488 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java
@@ -26,7 +26,6 @@
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexInternal;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
@@ -56,13 +55,11 @@
public void completeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException {
// Flush will only be handled by last exiting thread.
- if (opType == LSMOperationType.MODIFICATION) {
- if (threadRefCount.decrementAndGet() == 0
- && ((ILSMIndexInternal) index).hasFlushRequestForCurrentMutableComponent()) {
- ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
- accessor.scheduleFlush(NoOpIOOperationCallback.INSTANCE);
- }
+ if (opType == LSMOperationType.MODIFICATION && threadRefCount.decrementAndGet() == 0
+ && index.hasFlushRequestForCurrentMutableComponent()) {
+ ILSMIndexAccessor accessor =
+ index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ accessor.scheduleFlush(NoOpIOOperationCallback.INSTANCE);
}
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/utils/ComponentMetadataUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/utils/ComponentMetadataUtil.java
new file mode 100644
index 0000000..b55e8ad
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/utils/ComponentMetadataUtil.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.common.utils;
+
+import java.util.List;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.LongPointable;
+import org.apache.hyracks.storage.am.common.freepage.MutableArrayValueReference;
+import org.apache.hyracks.storage.am.lsm.common.api.IComponentMetadata;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
+
+public class ComponentMetadataUtil {
+
+ public static final MutableArrayValueReference MARKER_LSN_KEY =
+ new MutableArrayValueReference("Marker".getBytes());
+ public static final long NOT_FOUND = -1L;
+
+ private ComponentMetadataUtil() {
+ }
+
+ /**
+ * Get a long value from the metadata of a component or return a default value
+ *
+ * @param metadata
+ * the component's metadata
+ * @param key
+ * the key
+ * @param defaultValue
+ * the default value
+ * @return
+ * the long value if found, the default value otherwise
+ * @throws HyracksDataException
+ * If the comopnent was a disk component and an IO error was encountered
+ */
+ public static long getLong(IComponentMetadata metadata, IValueReference key, long defaultValue)
+ throws HyracksDataException {
+ IValueReference value = metadata.get(key);
+ return value == null || value.getLength() == 0 ? defaultValue
+ : LongPointable.getLong(value.getByteArray(), value.getStartOffset());
+ }
+
+ /**
+ * Get a value from an index's metadata pages. It first, searches the current in memory component
+ * then searches the other components. in reverse order.
+ * Note: This method locks on the OpTracker of the index
+ *
+ * @param index
+ * @param key
+ * @param pointable
+ * @throws HyracksDataException
+ */
+ public static void get(ILSMIndex index, IValueReference key, IPointable pointable) throws HyracksDataException {
+ // Lock the opTracker to ensure index components don't change
+ synchronized (index.getOperationTracker()) {
+ index.getCurrentMemoryComponent().getMetadata().get(key, pointable);
+ if (pointable.getLength() == 0) {
+ // was not found in the in current mutable component, search in the other in memory components
+ fromImmutableMemoryComponents(index, key, pointable);
+ if (pointable.getLength() == 0) {
+ // was not found in the in all in memory components, search in the disk components
+ fromDiskComponents(index, key, pointable);
+ }
+ }
+ }
+ }
+
+ private static void fromDiskComponents(ILSMIndex index, IValueReference key, IPointable pointable)
+ throws HyracksDataException {
+ for (ILSMDiskComponent c : index.getImmutableComponents()) {
+ c.getMetadata().get(key, pointable);
+ if (pointable.getLength() != 0) {
+ // Found
+ return;
+ }
+ }
+ }
+
+ private static void fromImmutableMemoryComponents(ILSMIndex index, IValueReference key, IPointable pointable) {
+ List<ILSMMemoryComponent> memComponents = index.getMemoryComponents();
+ int numOtherMemComponents = memComponents.size() - 1;
+ int next = index.getCurrentMemoryComponentIndex();
+ for (int i = 0; i < numOtherMemComponents; i++) {
+ next = next - 1;
+ if (next < 0) {
+ next = memComponents.size() - 1;
+ }
+ ILSMMemoryComponent c = index.getMemoryComponents().get(next);
+ if (c.isReadable()) {
+ c.getMetadata().get(key, pointable);
+ if (pointable.getLength() != 0) {
+ // Found
+ return;
+ }
+ }
+ }
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml
index e03cddc..010a343 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml
@@ -16,18 +16,15 @@
! specific language governing permissions and limitations
! under the License.
!-->
-
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>hyracks-storage-am-lsm-invertedindex</artifactId>
-
<parent>
<artifactId>hyracks</artifactId>
<groupId>org.apache.hyracks</groupId>
<version>0.3.1-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
-
<licenses>
<license>
<name>Apache License, Version 2.0</name>
@@ -36,11 +33,9 @@
<comments>A business-friendly OSS license</comments>
</license>
</licenses>
-
<properties>
<root.dir>${basedir}/../..</root.dir>
</properties>
-
<dependencies>
<dependency>
<groupId>org.apache.hyracks</groupId>
@@ -93,4 +88,4 @@
<version>${project.version}</version>
</dependency>
</dependencies>
-</project>
+</project>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java
index f39bab6..449c8f9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java
@@ -28,16 +28,16 @@
import org.apache.hyracks.storage.am.common.api.IndexException;
public interface IInvertedIndex extends IIndex {
- public IInvertedListCursor createInvertedListCursor();
+ IInvertedListCursor createInvertedListCursor();
- public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey, IIndexOperationContext ictx)
+ void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey, IIndexOperationContext ictx)
throws HyracksDataException, IndexException;
- public ITypeTraits[] getInvListTypeTraits();
+ ITypeTraits[] getInvListTypeTraits();
- public IBinaryComparatorFactory[] getInvListCmpFactories();
+ IBinaryComparatorFactory[] getInvListCmpFactories();
- public ITypeTraits[] getTokenTypeTraits();
+ ITypeTraits[] getTokenTypeTraits();
- public IBinaryComparatorFactory[] getTokenCmpFactories();
+ IBinaryComparatorFactory[] getTokenCmpFactories();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java
index 28cfadf..ac06d1c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java
@@ -90,7 +90,7 @@
invIndexOpDesc.getTokenComparatorFactories(), invIndexOpDesc.getTokenizerFactory(),
diskBufferCache, fileRef.getFile().getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy,
opTrackerFactory.getOperationTracker(ctx.getJobletContext().getApplicationContext()), ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback(), invertedIndexFields, filterTypeTraits,
+ ioOpCallbackFactory.createIoOpCallback(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
invertedIndexFieldsForNonBulkLoadOps, durable, (IMetadataPageManagerFactory) opDesc
.getPageManagerFactory());
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java
index 5138642..4ee3a21 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java
@@ -89,7 +89,7 @@
invIndexOpDesc.getTokenComparatorFactories(), invIndexOpDesc.getTokenizerFactory(),
diskBufferCache, fileRef.getFile().getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy,
opTrackerFactory.getOperationTracker(ctx.getJobletContext().getApplicationContext()), ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback(), invertedIndexFields, filterTypeTraits,
+ ioOpCallbackFactory.createIoOpCallback(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
invertedIndexFieldsForNonBulkLoadOps, durable, opDesc.getPageManagerFactory());
return invIndex;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 403fa09..fc0e4ec 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -55,14 +55,14 @@
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -94,7 +94,7 @@
// On-disk components.
// For creating inverted indexes in flush and merge.
- protected final ILSMComponentFactory componentFactory;
+ protected final ILSMDiskComponentFactory componentFactory;
// Type traits and comparators for tokens and inverted-list elements.
protected final ITypeTraits[] invListTypeTraits;
@@ -109,13 +109,14 @@
OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory,
BloomFilterFactory bloomFilterFactory, ILSMComponentFilterFactory filterFactory,
ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
- double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider,
- ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
- ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
- IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields,
- int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps,
- boolean durable) throws IndexException, HyracksDataException {
+ double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager,
+ IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits,
+ IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
+ IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, int[] filterFields,
+ int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable)
+ throws IndexException, HyracksDataException {
super(ioManager, virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
filterManager, filterFields, durable);
@@ -134,17 +135,15 @@
int i = 0;
for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
- InMemoryInvertedIndex memInvIndex = createInMemoryInvertedIndex(virtualBufferCache,
- new VirtualFreePageManager(virtualBufferCache), i);
+ InMemoryInvertedIndex memInvIndex =
+ createInMemoryInvertedIndex(virtualBufferCache, new VirtualFreePageManager(virtualBufferCache), i);
BTree deleteKeysBTree = BTreeUtils.createBTree(virtualBufferCache,
- new VirtualFreePageManager(virtualBufferCache),
- virtualBufferCache.getFileMapProvider(), invListTypeTraits, invListCmpFactories,
- BTreeLeafFrameType.REGULAR_NSM, ioManager.resolveAbsolutePath(fileManager.getBaseDir()
- + "_virtual_del_"
- + i));
- LSMInvertedIndexMemoryComponent mutableComponent = new LSMInvertedIndexMemoryComponent(memInvIndex,
- deleteKeysBTree, virtualBufferCache, i == 0 ? true : false,
- filterFactory == null ? null : filterFactory.createLSMComponentFilter());
+ new VirtualFreePageManager(virtualBufferCache), virtualBufferCache.getFileMapProvider(),
+ invListTypeTraits, invListCmpFactories, BTreeLeafFrameType.REGULAR_NSM,
+ ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_del_" + i));
+ LSMInvertedIndexMemoryComponent mutableComponent =
+ new LSMInvertedIndexMemoryComponent(memInvIndex, deleteKeysBTree, virtualBufferCache,
+ i == 0 ? true : false, filterFactory == null ? null : filterFactory.createFilter());
memoryComponents.add(mutableComponent);
++i;
}
@@ -168,7 +167,7 @@
}
try {
- List<ILSMComponent> immutableComponents = diskComponents;
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
immutableComponents.clear();
List<LSMComponentFileReferences> validFileReferences = fileManager.cleanupAndGetValidFiles();
for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
@@ -196,8 +195,8 @@
}
clearMemoryComponents();
- List<ILSMComponent> immutableComponents = diskComponents;
- for (ILSMComponent c : immutableComponents) {
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
+ for (ILSMDiskComponent c : immutableComponents) {
LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
component.getBloomFilter().deactivate();
component.getInvIndex().deactivate();
@@ -216,7 +215,8 @@
}
if (flushOnExit) {
BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
- ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ ILSMIndexAccessor accessor =
+ createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
accessor.scheduleFlush(cb);
try {
cb.waitForIO();
@@ -225,8 +225,8 @@
}
}
- List<ILSMComponent> immutableComponents = diskComponents;
- for (ILSMComponent c : immutableComponents) {
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
+ for (ILSMDiskComponent c : immutableComponents) {
LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
component.getBloomFilter().deactivate();
component.getInvIndex().deactivate();
@@ -252,8 +252,8 @@
mutableComponent.getInvIndex().destroy();
mutableComponent.getDeletedKeysBTree().destroy();
}
- List<ILSMComponent> immutableComponents = diskComponents;
- for (ILSMComponent c : immutableComponents) {
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
+ for (ILSMDiskComponent c : immutableComponents) {
LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
component.getInvIndex().destroy();
component.getDeletedKeysBTree().destroy();
@@ -264,7 +264,7 @@
@Override
public void getOperationalComponents(ILSMIndexOperationContext ctx) throws HyracksDataException {
- List<ILSMComponent> immutableComponents = diskComponents;
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
int cmc = currentMutableComponentId.get();
ctx.setCurrentMutableComponentId(cmc);
@@ -322,7 +322,8 @@
* - Insert key into deleted-keys BTree.
*/
@Override
- public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
+ public void modify(IIndexOperationContext ictx, ITupleReference tuple)
+ throws HyracksDataException, IndexException {
LSMInvertedIndexOpContext ctx = (LSMInvertedIndexOpContext) ictx;
// TODO: This is a hack to support logging properly in ASTERIX.
// The proper undo operations are only dependent on the after image so
@@ -386,8 +387,9 @@
IIndexAccessor invIndexAccessor = ((LSMInvertedIndexMemoryComponent) component).getInvIndex()
.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
indexAccessors.add(invIndexAccessor);
- IIndexAccessor deletedKeysAccessor = ((LSMInvertedIndexMemoryComponent) component).getDeletedKeysBTree()
- .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ IIndexAccessor deletedKeysAccessor =
+ ((LSMInvertedIndexMemoryComponent) component).getDeletedKeysBTree()
+ .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
deletedKeysBTreeAccessors.add(deletedKeysAccessor);
} else {
IIndexAccessor invIndexAccessor = ((LSMInvertedIndexDiskComponent) component).getInvIndex()
@@ -420,8 +422,8 @@
.getDeletedKeysBTree().getLeafFrameFactory(),
ictx, includeMutableComponent, lsmHarness, operationalComponents);
} else {
- LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) memoryComponents
- .get(currentMutableComponentId.get());
+ LSMInvertedIndexMemoryComponent mutableComponent =
+ (LSMInvertedIndexMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
InMemoryInvertedIndex memInvIndex = (InMemoryInvertedIndex) mutableComponent.getInvIndex();
MultiComparator tokensAndKeysCmp = MultiComparator.create(memInvIndex.getBTree().getComparatorFactories());
initState = new LSMInvertedIndexRangeSearchCursorInitialState(tokensAndKeysCmp, keyCmp, keysOnlyTuple,
@@ -452,8 +454,8 @@
throws HyracksDataException {
ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- LSMInvertedIndexOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
+ LSMInvertedIndexOpContext opCtx =
+ createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
opCtx.setOperation(IndexOperation.FLUSH);
opCtx.getComponentHolder().add(flushingComponent);
ioScheduler.scheduleOperation(new LSMInvertedIndexFlushOperation(
@@ -463,18 +465,18 @@
}
@Override
- public ILSMComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
LSMInvertedIndexFlushOperation flushOp = (LSMInvertedIndexFlushOperation) operation;
// Create an inverted index instance to be bulk loaded.
- LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory,
- flushOp.getDictBTreeFlushTarget(), flushOp.getDeletedKeysBTreeFlushTarget(),
- flushOp.getBloomFilterFlushTarget(), true);
+ LSMInvertedIndexDiskComponent component =
+ createDiskInvIndexComponent(componentFactory, flushOp.getDictBTreeFlushTarget(),
+ flushOp.getDeletedKeysBTreeFlushTarget(), flushOp.getBloomFilterFlushTarget(), true);
IInvertedIndex diskInvertedIndex = component.getInvIndex();
// Create a scan cursor on the BTree underlying the in-memory inverted index.
- LSMInvertedIndexMemoryComponent flushingComponent = (LSMInvertedIndexMemoryComponent) flushOp
- .getFlushingComponent();
+ LSMInvertedIndexMemoryComponent flushingComponent =
+ (LSMInvertedIndexMemoryComponent) flushOp.getFlushingComponent();
InMemoryInvertedIndexAccessor memInvIndexAccessor = (InMemoryInvertedIndexAccessor) flushingComponent
.getInvIndex().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
BTreeAccessor memBTreeAccessor = memInvIndexAccessor.getBTreeAccessor();
@@ -496,10 +498,11 @@
List<ITupleReference> filterTuples = new ArrayList<>();
filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
- filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilterInfo(component.getLSMComponentFilter(),
+ filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
+ filterManager.writeFilter(component.getLSMComponentFilter(),
((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
}
+ flushingComponent.getMetadata().copy(component.getMetadata());
invIndexBulkLoader.end();
IIndexAccessor deletedKeysBTreeAccessor = flushingComponent.getDeletedKeysBTree()
@@ -518,8 +521,8 @@
}
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numBTreeTuples);
- BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
- bloomFilterFalsePositiveRate);
+ BloomFilterSpecification bloomFilterSpec =
+ BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate);
// Create an BTree instance for the deleted keys.
BTree diskDeletedKeysBTree = component.getDeletedKeysBTree();
@@ -551,8 +554,8 @@
@Override
public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException, IndexException {
- LSMInvertedIndexOpContext ictx = createOpContext(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
+ LSMInvertedIndexOpContext ictx =
+ createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
ictx.setOperation(IndexOperation.MERGE);
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
IIndexCursor cursor = new LSMInvertedIndexRangeSearchCursor(ictx);
@@ -561,20 +564,21 @@
OnDiskInvertedIndex firstInvIndex = (OnDiskInvertedIndex) firstComponent.getInvIndex();
String firstFileName = firstInvIndex.getBTree().getFileReference().getFile().getName();
- LSMInvertedIndexDiskComponent lastComponent = (LSMInvertedIndexDiskComponent) mergingComponents
- .get(mergingComponents.size() - 1);
+ LSMInvertedIndexDiskComponent lastComponent =
+ (LSMInvertedIndexDiskComponent) mergingComponents.get(mergingComponents.size() - 1);
OnDiskInvertedIndex lastInvIndex = (OnDiskInvertedIndex) lastComponent.getInvIndex();
String lastFileName = lastInvIndex.getBTree().getFileReference().getFile().getName();
- LSMComponentFileReferences relMergeFileRefs = fileManager.getRelMergeFileReference(firstFileName, lastFileName);
- ILSMIndexAccessorInternal accessor = new LSMInvertedIndexAccessor(lsmHarness, ctx);
+ LSMComponentFileReferences relMergeFileRefs =
+ fileManager.getRelMergeFileReference(firstFileName, lastFileName);
+ ILSMIndexAccessor accessor = new LSMInvertedIndexAccessor(lsmHarness, ctx);
ioScheduler.scheduleOperation(new LSMInvertedIndexMergeOperation(accessor, mergingComponents, cursor,
relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(),
relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir()));
}
@Override
- public ILSMComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
LSMInvertedIndexMergeOperation mergeOp = (LSMInvertedIndexMergeOperation) operation;
IIndexCursor cursor = mergeOp.getCursor();
@@ -585,9 +589,9 @@
search(opCtx, cursor, mergePred);
// Create an inverted index instance.
- LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory,
- mergeOp.getDictBTreeMergeTarget(), mergeOp.getDeletedKeysBTreeMergeTarget(),
- mergeOp.getBloomFilterMergeTarget(), true);
+ LSMInvertedIndexDiskComponent component =
+ createDiskInvIndexComponent(componentFactory, mergeOp.getDictBTreeMergeTarget(),
+ mergeOp.getDeletedKeysBTreeMergeTarget(), mergeOp.getBloomFilterMergeTarget(), true);
IInvertedIndex mergedDiskInvertedIndex = component.getInvIndex();
@@ -597,8 +601,8 @@
.get(diskComponents.size() - 1)) {
// Keep the deleted tuples since the oldest disk component is not included in the merge operation
- LSMInvertedIndexDeletedKeysBTreeMergeCursor btreeCursor = new LSMInvertedIndexDeletedKeysBTreeMergeCursor(
- opCtx);
+ LSMInvertedIndexDeletedKeysBTreeMergeCursor btreeCursor =
+ new LSMInvertedIndexDeletedKeysBTreeMergeCursor(opCtx);
search(opCtx, btreeCursor, mergePred);
BTree btree = component.getDeletedKeysBTree();
@@ -611,8 +615,8 @@
}
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
- BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
- bloomFilterFalsePositiveRate);
+ BloomFilterSpecification bloomFilterSpec =
+ BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate);
IIndexBulkLoader builder = component.getBloomFilter().createBuilder(numElements,
bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
try {
@@ -655,8 +659,8 @@
filterTuples.add(max);
}
}
- filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilterInfo(component.getLSMComponentFilter(),
+ filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
+ filterManager.writeFilter(component.getLSMComponentFilter(),
((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
}
invIndexBulkLoader.end();
@@ -664,12 +668,6 @@
return component;
}
- private ILSMComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
- LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- return createDiskInvIndexComponent(componentFactory, componentFileRefs.getInsertIndexFileReference(),
- componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true);
- }
-
@Override
public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
boolean checkIfEmptyIndex) throws IndexException {
@@ -681,7 +679,7 @@
}
public class LSMInvertedIndexBulkLoader implements IIndexBulkLoader {
- private final ILSMComponent component;
+ private final ILSMDiskComponent component;
private final IIndexBulkLoader invIndexBulkLoader;
private final IIndexBulkLoader deletedKeysBTreeBulkLoader;
private boolean cleanedUpArtifacts = false;
@@ -763,7 +761,7 @@
public void end() throws IndexException, HyracksDataException {
if (!cleanedUpArtifacts) {
if (component.getLSMComponentFilter() != null) {
- filterManager.writeFilterInfo(component.getLSMComponentFilter(),
+ filterManager.writeFilter(component.getLSMComponentFilter(),
(((OnDiskInvertedIndex) ((LSMInvertedIndexDiskComponent) component).getInvIndex())
.getBTree()));
}
@@ -789,20 +787,27 @@
deletedKeysBTreeBulkLoader.abort();
}
}
+
+ private ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
+ LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
+ return createDiskInvIndexComponent(componentFactory, componentFileRefs.getInsertIndexFileReference(),
+ componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(),
+ true);
+ }
}
protected InMemoryInvertedIndex createInMemoryInvertedIndex(IVirtualBufferCache virtualBufferCache,
VirtualFreePageManager virtualFreePageManager, int id) throws IndexException, HyracksDataException {
return InvertedIndexUtils.createInMemoryBTreeInvertedindex(virtualBufferCache, virtualFreePageManager,
- invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, ioManager
- .resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_vocab_" + id));
+ invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
+ ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_vocab_" + id));
}
- protected LSMInvertedIndexDiskComponent createDiskInvIndexComponent(ILSMComponentFactory factory,
+ protected LSMInvertedIndexDiskComponent createDiskInvIndexComponent(ILSMDiskComponentFactory factory,
FileReference dictBTreeFileRef, FileReference btreeFileRef, FileReference bloomFilterFileRef,
boolean create) throws HyracksDataException, IndexException {
- LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) factory.createLSMComponentInstance(
- new LSMComponentFileReferences(dictBTreeFileRef, btreeFileRef, bloomFilterFileRef));
+ LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) factory
+ .createComponent(new LSMComponentFileReferences(dictBTreeFileRef, btreeFileRef, bloomFilterFileRef));
if (create) {
component.getInvIndex().create();
component.getDeletedKeysBTree().create();
@@ -813,14 +818,14 @@
component.getBloomFilter().activate();
// Will be closed during cleanup of merge().
if (component.getLSMComponentFilter() != null && !create) {
- filterManager.readFilterInfo(component.getLSMComponentFilter(),
+ filterManager.readFilter(component.getLSMComponentFilter(),
((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
}
return component;
}
@Override
- public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
+ public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) throws HyracksDataException {
return new LSMInvertedIndexAccessor(lsmHarness, createOpContext(modificationCallback, searchCallback));
}
@@ -878,7 +883,7 @@
}
@Override
- public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException {
+ public void markAsValid(ILSMDiskComponent lsmComponent) throws HyracksDataException {
LSMInvertedIndexDiskComponent invIndexComponent = (LSMInvertedIndexDiskComponent) lsmComponent;
OnDiskInvertedIndex invIndex = (OnDiskInvertedIndex) invIndexComponent.getInvIndex();
IBufferCache bufferCache = invIndex.getBufferCache();
@@ -895,8 +900,8 @@
@Override
public void validate() throws HyracksDataException {
validateMemoryComponents();
- List<ILSMComponent> immutableComponents = diskComponents;
- for (ILSMComponent c : immutableComponents) {
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
+ for (ILSMDiskComponent c : immutableComponents) {
LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
component.getInvIndex().validate();
component.getDeletedKeysBTree().validate();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
index 4c4a552..cef4257 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
@@ -26,17 +26,17 @@
import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
-public class LSMInvertedIndexAccessor implements ILSMIndexAccessorInternal, IInvertedIndexAccessor {
+public class LSMInvertedIndexAccessor implements ILSMIndexAccessor, IInvertedIndexAccessor {
protected final ILSMHarness lsmHarness;
protected final ILSMIndexOperationContext ctx;
@@ -93,7 +93,7 @@
}
@Override
- public void scheduleMerge(ILSMIOOperationCallback callback, List<ILSMComponent> components)
+ public void scheduleMerge(ILSMIOOperationCallback callback, List<ILSMDiskComponent> components)
throws HyracksDataException, IndexException {
ctx.setOperation(IndexOperation.MERGE);
ctx.getComponentsToBeMerged().clear();
@@ -102,7 +102,7 @@
}
@Override
- public void scheduleReplication(List<ILSMComponent> lsmComponents, boolean bulkload, LSMOperationType opType)
+ public void scheduleReplication(List<ILSMDiskComponent> lsmComponents, boolean bulkload, LSMOperationType opType)
throws HyracksDataException {
ctx.setOperation(IndexOperation.REPLICATE);
ctx.getComponentsToBeReplicated().clear();
@@ -127,8 +127,8 @@
}
@Override
- public void rangeSearch(IIndexCursor cursor, ISearchPredicate searchPred) throws IndexException,
- HyracksDataException {
+ public void rangeSearch(IIndexCursor cursor, ISearchPredicate searchPred)
+ throws IndexException, HyracksDataException {
search(cursor, searchPred);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java
index 799a3da..8a12dea 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java
@@ -21,20 +21,21 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import org.apache.hyracks.storage.am.btree.impls.BTree;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
-import org.apache.hyracks.storage.am.lsm.common.impls.AbstractDiskLSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex;
-public class LSMInvertedIndexDiskComponent extends AbstractDiskLSMComponent {
+public class LSMInvertedIndexDiskComponent extends AbstractLSMDiskComponent {
private final IInvertedIndex invIndex;
private final BTree deletedKeysBTree;
private final BloomFilter bloomFilter;
public LSMInvertedIndexDiskComponent(IInvertedIndex invIndex, BTree deletedKeysBTree, BloomFilter bloomFilter,
- ILSMComponentFilter filter) {
- super(filter);
+ ILSMComponentFilter filter) throws HyracksDataException {
+ super((IMetadataPageManager) deletedKeysBTree.getPageManager(), filter);
this.invIndex = invIndex;
this.deletedKeysBTree = deletedKeysBTree;
this.bloomFilter = bloomFilter;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java
index a6f80ce..270d3a1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java
@@ -22,16 +22,13 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import org.apache.hyracks.storage.am.btree.impls.BTree;
-import org.apache.hyracks.storage.am.common.api.IndexException;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexFactory;
-import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-public class LSMInvertedIndexDiskComponentFactory implements ILSMComponentFactory {
+public class LSMInvertedIndexDiskComponentFactory implements ILSMDiskComponentFactory {
private final OnDiskInvertedIndexFactory diskInvIndexFactory;
private final TreeIndexFactory<BTree> btreeFactory;
private final BloomFilterFactory bloomFilterFactory;
@@ -47,16 +44,11 @@
}
@Override
- public ILSMComponent createLSMComponentInstance(LSMComponentFileReferences cfr) throws IndexException,
- HyracksDataException {
- return new LSMInvertedIndexDiskComponent(diskInvIndexFactory.createIndexInstance(cfr
- .getInsertIndexFileReference()), btreeFactory.createIndexInstance(cfr.getDeleteIndexFileReference()),
+ public LSMInvertedIndexDiskComponent createComponent(LSMComponentFileReferences cfr) throws HyracksDataException {
+ return new LSMInvertedIndexDiskComponent(
+ diskInvIndexFactory.createIndexInstance(cfr.getInsertIndexFileReference()),
+ btreeFactory.createIndexInstance(cfr.getDeleteIndexFileReference()),
bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()),
- filterFactory == null ? null : filterFactory.createLSMComponentFilter());
- }
-
- @Override
- public IBufferCache getBufferCache() {
- return diskInvIndexFactory.getBufferCache();
+ filterFactory == null ? null : filterFactory.createFilter());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
index b38d23a..03ba304 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
@@ -30,10 +30,10 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
public class LSMInvertedIndexFlushOperation implements ILSMIOOperation, Comparable<LSMInvertedIndexFlushOperation> {
- private final ILSMIndexAccessorInternal accessor;
+ private final ILSMIndexAccessor accessor;
private final ILSMComponent flushingComponent;
private final FileReference dictBTreeFlushTarget;
private final FileReference deletedKeysBTreeFlushTarget;
@@ -41,7 +41,7 @@
private final ILSMIOOperationCallback callback;
private final String indexIdentifier;
- public LSMInvertedIndexFlushOperation(ILSMIndexAccessorInternal accessor, ILSMComponent flushingComponent,
+ public LSMInvertedIndexFlushOperation(ILSMIndexAccessor accessor, ILSMComponent flushingComponent,
FileReference dictBTreeFlushTarget, FileReference deletedKeysBTreeFlushTarget,
FileReference bloomFilterFlushTarget, ILSMIOOperationCallback callback, String indexIdentifier) {
this.accessor = accessor;
@@ -60,7 +60,7 @@
@Override
public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ Set<IODeviceHandle> devs = new HashSet<>();
devs.add(dictBTreeFlushTarget.getDeviceHandle());
devs.add(deletedKeysBTreeFlushTarget.getDeviceHandle());
devs.add(bloomFilterFlushTarget.getDeviceHandle());
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java
index d5a9237..c164cf9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java
@@ -23,10 +23,10 @@
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
-import org.apache.hyracks.storage.am.lsm.common.impls.AbstractMemoryLSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
-public class LSMInvertedIndexMemoryComponent extends AbstractMemoryLSMComponent {
+public class LSMInvertedIndexMemoryComponent extends AbstractLSMMemoryComponent {
private final IInvertedIndex invIndex;
private final BTree deletedKeysBTree;
@@ -47,7 +47,7 @@
}
@Override
- protected void reset() throws HyracksDataException {
+ public void reset() throws HyracksDataException {
super.reset();
invIndex.deactivate();
invIndex.destroy();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
index 7cbf544..c22af32 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
@@ -31,11 +31,11 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex;
public class LSMInvertedIndexMergeOperation implements ILSMIOOperation {
- private final ILSMIndexAccessorInternal accessor;
+ private final ILSMIndexAccessor accessor;
private final List<ILSMComponent> mergingComponents;
private final IIndexCursor cursor;
private final FileReference dictBTreeMergeTarget;
@@ -44,7 +44,7 @@
private final ILSMIOOperationCallback callback;
private final String indexIdentifier;
- public LSMInvertedIndexMergeOperation(ILSMIndexAccessorInternal accessor, List<ILSMComponent> mergingComponents,
+ public LSMInvertedIndexMergeOperation(ILSMIndexAccessor accessor, List<ILSMComponent> mergingComponents,
IIndexCursor cursor, FileReference dictBTreeMergeTarget, FileReference deletedKeysBTreeMergeTarget,
FileReference bloomFilterMergeTarget, ILSMIOOperationCallback callback, String indexIdentifier) {
this.accessor = accessor;
@@ -59,7 +59,7 @@
@Override
public Set<IODeviceHandle> getReadDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ Set<IODeviceHandle> devs = new HashSet<>();
for (Object o : mergingComponents) {
LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) o;
OnDiskInvertedIndex invIndex = (OnDiskInvertedIndex) component.getInvIndex();
@@ -72,7 +72,7 @@
@Override
public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ Set<IODeviceHandle> devs = new HashSet<>();
devs.add(dictBTreeMergeTarget.getDeviceHandle());
devs.add(deletedKeysBTreeMergeTarget.getDeviceHandle());
devs.add(bloomFilterMergeTarget.getDeviceHandle());
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
index 828e296..c8ff02f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
@@ -31,7 +31,9 @@
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor;
@@ -41,8 +43,8 @@
private IndexOperation op;
private final List<ILSMComponent> componentHolder;
- private final List<ILSMComponent> componentsToBeMerged;
- private final List<ILSMComponent> componentsToBeReplicated;
+ private final List<ILSMDiskComponent> componentsToBeMerged;
+ private final List<ILSMDiskComponent> componentsToBeReplicated;
private IModificationOperationCallback modificationCallback;
private ISearchOperationCallback searchCallback;
@@ -64,12 +66,12 @@
public ISearchPredicate searchPredicate;
- public LSMInvertedIndexOpContext(List<ILSMComponent> mutableComponents,
+ public LSMInvertedIndexOpContext(List<ILSMMemoryComponent> mutableComponents,
IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback,
int[] invertedIndexFields, int[] filterFields) throws HyracksDataException {
- this.componentHolder = new LinkedList<ILSMComponent>();
- this.componentsToBeMerged = new LinkedList<ILSMComponent>();
- this.componentsToBeReplicated = new LinkedList<ILSMComponent>();
+ this.componentHolder = new LinkedList<>();
+ this.componentsToBeMerged = new LinkedList<>();
+ this.componentsToBeReplicated = new LinkedList<>();
this.modificationCallback = modificationCallback;
this.searchCallback = searchCallback;
@@ -77,8 +79,8 @@
deletedKeysBTreeAccessors = new IIndexAccessor[mutableComponents.size()];
for (int i = 0; i < mutableComponents.size(); i++) {
- LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) mutableComponents
- .get(i);
+ LSMInvertedIndexMemoryComponent mutableComponent =
+ (LSMInvertedIndexMemoryComponent) mutableComponents.get(i);
mutableInvIndexAccessors[i] = (IInvertedIndexAccessor) mutableComponent.getInvIndex()
.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
deletedKeysBTreeAccessors[i] = mutableComponent.getDeletedKeysBTree()
@@ -149,7 +151,7 @@
}
@Override
- public List<ILSMComponent> getComponentsToBeMerged() {
+ public List<ILSMDiskComponent> getComponentsToBeMerged() {
return componentsToBeMerged;
}
@@ -164,7 +166,7 @@
}
@Override
- public List<ILSMComponent> getComponentsToBeReplicated() {
+ public List<ILSMDiskComponent> getComponentsToBeReplicated() {
return componentsToBeReplicated;
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
index bef5248..583c5f4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
@@ -23,7 +23,6 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.btree.exceptions.BTreeException;
import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
@@ -59,7 +58,7 @@
public InMemoryInvertedIndex(IBufferCache virtualBufferCache, IPageManager virtualFreePageManager,
ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
- IBinaryTokenizerFactory tokenizerFactory, FileReference btreeFileRef) throws BTreeException {
+ IBinaryTokenizerFactory tokenizerFactory, FileReference btreeFileRef) throws HyracksDataException {
this.tokenTypeTraits = tokenTypeTraits;
this.tokenCmpFactories = tokenCmpFactories;
this.invListTypeTraits = invListTypeTraits;
@@ -168,8 +167,8 @@
@Override
public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) throws HyracksDataException {
- return new InMemoryInvertedIndexAccessor(this, new InMemoryInvertedIndexOpContext(btree, tokenCmpFactories,
- tokenizerFactory));
+ return new InMemoryInvertedIndexAccessor(this,
+ new InMemoryInvertedIndexOpContext(btree, tokenCmpFactories, tokenizerFactory));
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java
index f9473db..0bea41f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java
@@ -26,12 +26,11 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.btree.exceptions.BTreeException;
import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
-import org.apache.hyracks.storage.am.common.api.IPageManager;
import org.apache.hyracks.storage.am.common.api.IIndexAccessor;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
+import org.apache.hyracks.storage.am.common.api.IPageManager;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
@@ -53,7 +52,7 @@
public PartitionedInMemoryInvertedIndex(IBufferCache memBufferCache, IPageManager memFreePageManager,
ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
- IBinaryTokenizerFactory tokenizerFactory, FileReference btreeFileRef) throws BTreeException {
+ IBinaryTokenizerFactory tokenizerFactory, FileReference btreeFileRef) throws HyracksDataException {
super(memBufferCache, memFreePageManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, tokenizerFactory, btreeFileRef);
}
@@ -63,7 +62,8 @@
throws HyracksDataException, IndexException {
super.insert(tuple, btreeAccessor, ictx);
PartitionedInMemoryInvertedIndexOpContext ctx = (PartitionedInMemoryInvertedIndexOpContext) ictx;
- PartitionedInvertedIndexTokenizingTupleIterator tupleIter = (PartitionedInvertedIndexTokenizingTupleIterator) ctx.tupleIter;
+ PartitionedInvertedIndexTokenizingTupleIterator tupleIter =
+ (PartitionedInvertedIndexTokenizingTupleIterator) ctx.tupleIter;
updatePartitionIndexes(tupleIter.getNumTokens());
}
@@ -91,8 +91,8 @@
@Override
public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) throws HyracksDataException {
- return new PartitionedInMemoryInvertedIndexAccessor(this, new PartitionedInMemoryInvertedIndexOpContext(btree,
- tokenCmpFactories, tokenizerFactory));
+ return new PartitionedInMemoryInvertedIndexAccessor(this,
+ new PartitionedInMemoryInvertedIndexOpContext(btree, tokenCmpFactories, tokenizerFactory));
}
@Override
@@ -131,8 +131,8 @@
// using the last existing partition and re-searching the BTree with an open interval as low key.
for (short i = partitionStartIndex; i <= partitionEndIndex; i++) {
partSearcher.setNumTokensBoundsInSearchKeys(i, i);
- InMemoryInvertedListCursor inMemListCursor = (InMemoryInvertedListCursor) partSearcher
- .getCachedInvertedListCursor();
+ InMemoryInvertedListCursor inMemListCursor =
+ (InMemoryInvertedListCursor) partSearcher.getCachedInvertedListCursor();
inMemListCursor.prepare(ctx.btreeAccessor, ctx.btreePred, ctx.tokenFieldsCmp, ctx.btreeCmp);
inMemListCursor.reset(searchKey);
invListPartitions.addInvertedListCursor(inMemListCursor, i);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
index 4884190..70a5024 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
@@ -117,8 +117,7 @@
IInvertedListBuilder invListBuilder, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, FileReference btreeFile, FileReference invListsFile,
- IPageManagerFactory pageManagerFactory)
- throws IndexException {
+ IPageManagerFactory pageManagerFactory) throws HyracksDataException {
this.bufferCache = bufferCache;
this.fileMapProvider = fileMapProvider;
this.invListBuilder = invListBuilder;
@@ -127,8 +126,8 @@
this.tokenTypeTraits = tokenTypeTraits;
this.tokenCmpFactories = tokenCmpFactories;
this.btree = BTreeUtils.createBTree(bufferCache, fileMapProvider, getBTreeTypeTraits(tokenTypeTraits),
- tokenCmpFactories, BTreeLeafFrameType.REGULAR_NSM, btreeFile, pageManagerFactory.createPageManager(
- bufferCache));
+ tokenCmpFactories, BTreeLeafFrameType.REGULAR_NSM, btreeFile,
+ pageManagerFactory.createPageManager(bufferCache));
this.numTokenFields = btree.getComparatorFactories().length;
this.numInvListKeys = invListCmpFactories.length;
this.invListsFile = invListsFile;
@@ -280,14 +279,14 @@
}
public void resetInvertedListCursor(ITupleReference btreeTuple, IInvertedListCursor listCursor) {
- int startPageId = IntegerPointable.getInteger(btreeTuple.getFieldData(invListStartPageIdField), btreeTuple
- .getFieldStart(invListStartPageIdField));
- int endPageId = IntegerPointable.getInteger(btreeTuple.getFieldData(invListEndPageIdField), btreeTuple
- .getFieldStart(invListEndPageIdField));
- int startOff = IntegerPointable.getInteger(btreeTuple.getFieldData(invListStartOffField), btreeTuple
- .getFieldStart(invListStartOffField));
- int numElements = IntegerPointable.getInteger(btreeTuple.getFieldData(invListNumElementsField), btreeTuple
- .getFieldStart(invListNumElementsField));
+ int startPageId = IntegerPointable.getInteger(btreeTuple.getFieldData(invListStartPageIdField),
+ btreeTuple.getFieldStart(invListStartPageIdField));
+ int endPageId = IntegerPointable.getInteger(btreeTuple.getFieldData(invListEndPageIdField),
+ btreeTuple.getFieldStart(invListEndPageIdField));
+ int startOff = IntegerPointable.getInteger(btreeTuple.getFieldData(invListStartOffField),
+ btreeTuple.getFieldStart(invListStartOffField));
+ int numElements = IntegerPointable.getInteger(btreeTuple.getFieldData(invListNumElementsField),
+ btreeTuple.getFieldStart(invListNumElementsField));
listCursor.reset(startPageId, endPageId, startOff, numElements);
}
@@ -312,8 +311,7 @@
private IFIFOPageQueue queue;
public OnDiskInvertedIndexBulkLoader(float btreeFillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex, int startPageId) throws IndexException,
- HyracksDataException {
+ boolean checkIfEmptyIndex, int startPageId) throws IndexException, HyracksDataException {
this.verifyInput = verifyInput;
this.tokenCmp = MultiComparator.create(btree.getComparatorFactories());
this.invListCmp = MultiComparator.create(invListCmpFactories);
@@ -326,8 +324,8 @@
this.btreeTupleReference = new ArrayTupleReference();
this.lastTupleBuilder = new ArrayTupleBuilder(numTokenFields + numInvListKeys);
this.lastTuple = new ArrayTupleReference();
- this.btreeBulkloader = btree.createBulkLoader(btreeFillFactor, verifyInput, numElementsHint,
- checkIfEmptyIndex);
+ this.btreeBulkloader =
+ btree.createBulkLoader(btreeFillFactor, verifyInput, numElementsHint, checkIfEmptyIndex);
currentPageId = startPageId;
currentPage = bufferCache.confiscatePage(BufferedFileHandle.getDiskPageId(fileId, currentPageId));
invListBuilder.setTargetBuffer(currentPage.getBuffer().array(), 0);
@@ -507,9 +505,10 @@
}
@Override
- public void search(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException,
- IndexException {
- searcher.search((OnDiskInvertedIndexSearchCursor) cursor, (InvertedIndexSearchPredicate) searchPred, opCtx);
+ public void search(IIndexCursor cursor, ISearchPredicate searchPred)
+ throws HyracksDataException, IndexException {
+ searcher.search((OnDiskInvertedIndexSearchCursor) cursor, (InvertedIndexSearchPredicate) searchPred,
+ opCtx);
}
@Override
@@ -529,8 +528,8 @@
}
@Override
- public void rangeSearch(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException,
- IndexException {
+ public void rangeSearch(IIndexCursor cursor, ISearchPredicate searchPred)
+ throws HyracksDataException, IndexException {
OnDiskInvertedIndexRangeSearchCursor rangeSearchCursor = (OnDiskInvertedIndexRangeSearchCursor) cursor;
rangeSearchCursor.open(null, searchPred);
}
@@ -617,8 +616,8 @@
public void validate() throws HyracksDataException {
btree.validate();
// Scan the btree and validate the order of elements in each inverted-list.
- IIndexAccessor btreeAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
+ IIndexAccessor btreeAccessor =
+ btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
IIndexCursor btreeCursor = btreeAccessor.createSearchCursor(false);
MultiComparator btreeCmp = MultiComparator.create(btree.getComparatorFactories());
RangePredicate rangePred = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp);
@@ -628,8 +627,9 @@
}
PermutingTupleReference tokenTuple = new PermutingTupleReference(fieldPermutation);
- IInvertedIndexAccessor invIndexAccessor = (IInvertedIndexAccessor) createAccessor(
- NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ IInvertedIndexAccessor invIndexAccessor =
+ (IInvertedIndexAccessor) createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
IInvertedListCursor invListCursor = invIndexAccessor.createInvertedListCursor();
MultiComparator invListCmp = MultiComparator.create(invListCmpFactories);
@@ -706,5 +706,4 @@
public boolean hasMemoryComponents() {
return true;
}
-
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java
index 14ceee8..dacef3c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java
@@ -24,7 +24,6 @@
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
-import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.impls.IndexFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNameMapper;
@@ -57,7 +56,7 @@
}
@Override
- public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws IndexException, HyracksDataException {
+ public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws HyracksDataException {
String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getAbsolutePath());
FileReference invListsFile = ioManager.resolveAbsolutePath(invListsFilePath);
IInvertedListBuilder invListBuilder = invListBuilderFactory.create();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
index 6d47ddc..a3a4de4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
@@ -50,8 +50,7 @@
IInvertedListBuilder invListBuilder, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, FileReference btreeFile, FileReference invListsFile,
- IPageManagerFactory pageManagerFactory)
- throws IndexException {
+ IPageManagerFactory pageManagerFactory) throws HyracksDataException {
super(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory);
}
@@ -99,7 +98,8 @@
while (ctx.btreeCursor.hasNext()) {
ctx.btreeCursor.next();
ITupleReference btreeTuple = ctx.btreeCursor.getTuple();
- short numTokens = ShortPointable.getShort(btreeTuple.getFieldData(PARTITIONING_NUM_TOKENS_FIELD), btreeTuple.getFieldStart(PARTITIONING_NUM_TOKENS_FIELD));
+ short numTokens = ShortPointable.getShort(btreeTuple.getFieldData(PARTITIONING_NUM_TOKENS_FIELD),
+ btreeTuple.getFieldStart(PARTITIONING_NUM_TOKENS_FIELD));
IInvertedListCursor invListCursor = partSearcher.getCachedInvertedListCursor();
resetInvertedListCursor(btreeTuple, invListCursor);
cursorsOrderedByTokens.add(invListCursor);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java
index b97cad0..ebc0152 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java
@@ -24,7 +24,6 @@
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
-import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNameMapper;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder;
@@ -35,17 +34,16 @@
public class PartitionedOnDiskInvertedIndexFactory extends OnDiskInvertedIndexFactory {
public PartitionedOnDiskInvertedIndexFactory(IIOManager ioManager, IBufferCache bufferCache,
- IFileMapProvider fileMapProvider,
- IInvertedListBuilderFactory invListBuilderFactory, ITypeTraits[] invListTypeTraits,
- IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
- IBinaryComparatorFactory[] tokenCmpFactories, IInvertedIndexFileNameMapper fileNameMapper,
- IPageManagerFactory pageManagerFactory) {
+ IFileMapProvider fileMapProvider, IInvertedListBuilderFactory invListBuilderFactory,
+ ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
+ ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
+ IInvertedIndexFileNameMapper fileNameMapper, IPageManagerFactory pageManagerFactory) {
super(ioManager, bufferCache, fileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories,
tokenTypeTraits, tokenCmpFactories, fileNameMapper, pageManagerFactory);
}
@Override
- public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws IndexException, HyracksDataException {
+ public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws HyracksDataException {
String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getAbsolutePath());
FileReference invListsFile = ioManager.resolveAbsolutePath(invListsFilePath);
IInvertedListBuilder invListBuilder = invListBuilderFactory.create();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
index b8dd258..c6a552c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
@@ -27,7 +27,6 @@
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
-import org.apache.hyracks.storage.am.btree.exceptions.BTreeException;
import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import org.apache.hyracks.storage.am.btree.util.BTreeUtils;
@@ -69,7 +68,7 @@
IPageManager virtualFreePageManager, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
- FileReference btreeFileRef) throws BTreeException {
+ FileReference btreeFileRef) throws HyracksDataException {
return new InMemoryInvertedIndex(memBufferCache, virtualFreePageManager, invListTypeTraits,
invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, btreeFileRef);
}
@@ -78,7 +77,7 @@
IPageManager virtualFreePageManager, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
- FileReference btreeFileRef) throws BTreeException {
+ FileReference btreeFileRef) throws HyracksDataException {
return new PartitionedInMemoryInvertedIndex(memBufferCache, virtualFreePageManager, invListTypeTraits,
invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, btreeFileRef);
}
@@ -87,8 +86,7 @@
IFileMapProvider fileMapProvider, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile,
- IPageManagerFactory pageManagerFactory) throws IndexException,
- HyracksDataException {
+ IPageManagerFactory pageManagerFactory) throws IndexException, HyracksDataException {
IInvertedListBuilder builder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
FileReference btreeFile = getBTreeFile(ioManager, invListsFile);
return new OnDiskInvertedIndex(bufferCache, fileMapProvider, builder, invListTypeTraits, invListCmpFactories,
@@ -96,12 +94,10 @@
}
public static PartitionedOnDiskInvertedIndex createPartitionedOnDiskInvertedIndex(IIOManager ioManager,
- IBufferCache bufferCache,
- IFileMapProvider fileMapProvider, ITypeTraits[] invListTypeTraits,
+ IBufferCache bufferCache, IFileMapProvider fileMapProvider, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile,
- IPageManagerFactory pageManagerFactory) throws IndexException,
- HyracksDataException {
+ IPageManagerFactory pageManagerFactory) throws IndexException, HyracksDataException {
IInvertedListBuilder builder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
FileReference btreeFile = getBTreeFile(ioManager, invListsFile);
return new PartitionedOnDiskInvertedIndex(bufferCache, fileMapProvider, builder, invListTypeTraits,
@@ -113,30 +109,30 @@
return ioManager.resolveAbsolutePath(invListsFile.getFile().getPath() + "_btree");
}
- public static BTreeFactory createDeletedKeysBTreeFactory(IIOManager ioManager, IFileMapProvider diskFileMapProvider,
- ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
- IBufferCache diskBufferCache, IPageManagerFactory freePageManagerFactory) throws BTreeException {
+ public static BTreeFactory createDeletedKeysBTreeFactory(IIOManager ioManager,
+ IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits,
+ IBinaryComparatorFactory[] invListCmpFactories, IBufferCache diskBufferCache,
+ IPageManagerFactory freePageManagerFactory) throws HyracksDataException {
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(invListTypeTraits);
- ITreeIndexFrameFactory leafFrameFactory = BTreeUtils.getLeafFrameFactory(tupleWriterFactory,
- BTreeLeafFrameType.REGULAR_NSM);
+ ITreeIndexFrameFactory leafFrameFactory =
+ BTreeUtils.getLeafFrameFactory(tupleWriterFactory, BTreeLeafFrameType.REGULAR_NSM);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- BTreeFactory deletedKeysBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
- freePageManagerFactory, interiorFrameFactory, leafFrameFactory, invListCmpFactories,
- invListCmpFactories.length);
+ BTreeFactory deletedKeysBTreeFactory =
+ new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+ interiorFrameFactory, leafFrameFactory, invListCmpFactories, invListCmpFactories.length);
return deletedKeysBTreeFactory;
}
public static LSMInvertedIndex createLSMInvertedIndex(IIOManager ioManager,
- List<IVirtualBufferCache> virtualBufferCaches,
- IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits,
- IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
- IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
- IBufferCache diskBufferCache, String absoluteOnDiskDir, double bloomFilterFalsePositiveRate,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps,
- int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, IMetadataPageManagerFactory pageManagerFactory)
- throws IndexException, HyracksDataException {
+ List<IVirtualBufferCache> virtualBufferCaches, IFileMapProvider diskFileMapProvider,
+ ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
+ ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
+ IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String absoluteOnDiskDir,
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
+ int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable,
+ IMetadataPageManagerFactory pageManagerFactory) throws IndexException, HyracksDataException {
BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider,
invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory);
@@ -145,15 +141,15 @@
for (int i = 0; i < invListCmpFactories.length; i++) {
bloomFilterKeyFields[i] = i;
}
- BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider,
- bloomFilterKeyFields);
+ BloomFilterFactory bloomFilterFactory =
+ new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
FileReference onDiskDirFileRef = ioManager.resolveAbsolutePath(absoluteOnDiskDir);
LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider,
onDiskDirFileRef, deletedKeysBTreeFactory);
- IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory(
- invListTypeTraits);
+ IInvertedListBuilderFactory invListBuilderFactory =
+ new FixedSizeElementInvertedListBuilderFactory(invListTypeTraits);
OnDiskInvertedIndexFactory invIndexFactory = new OnDiskInvertedIndexFactory(ioManager, diskBufferCache,
diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, fileManager, pageManagerFactory);
@@ -168,12 +164,11 @@
filterManager = new LSMComponentFilterManager(filterFrameFactory);
}
LSMInvertedIndex invIndex = new LSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory,
- deletedKeysBTreeFactory,
- bloomFilterFactory, filterFactory, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate,
- fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
- tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback,
- invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps,
- durable);
+ deletedKeysBTreeFactory, bloomFilterFactory, filterFactory, filterFrameFactory, filterManager,
+ bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories,
+ tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler,
+ ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps,
+ invertedIndexFieldsForNonBulkLoadOps, durable);
return invIndex;
}
@@ -186,8 +181,7 @@
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable,
- IPageManagerFactory pageManagerFactory)
- throws IndexException, HyracksDataException {
+ IPageManagerFactory pageManagerFactory) throws IndexException, HyracksDataException {
BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider,
invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory);
@@ -196,15 +190,15 @@
for (int i = 0; i < invListCmpFactories.length; i++) {
bloomFilterKeyFields[i] = i;
}
- BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider,
- bloomFilterKeyFields);
+ BloomFilterFactory bloomFilterFactory =
+ new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
FileReference onDiskDirFileRef = ioManager.resolveAbsolutePath(absoluteOnDiskDir);
LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider,
onDiskDirFileRef, deletedKeysBTreeFactory);
- IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory(
- invListTypeTraits);
+ IInvertedListBuilderFactory invListBuilderFactory =
+ new FixedSizeElementInvertedListBuilderFactory(invListTypeTraits);
PartitionedOnDiskInvertedIndexFactory invIndexFactory = new PartitionedOnDiskInvertedIndexFactory(ioManager,
diskBufferCache, diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories,
tokenTypeTraits, tokenCmpFactories, fileManager, pageManagerFactory);
@@ -219,11 +213,10 @@
filterManager = new LSMComponentFilterManager(filterFrameFactory);
}
PartitionedLSMInvertedIndex invIndex = new PartitionedLSMInvertedIndex(ioManager, virtualBufferCaches,
- invIndexFactory,
- deletedKeysBTreeFactory, bloomFilterFactory, filterFactory, filterFrameFactory, filterManager,
- bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories,
- tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler,
- ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps,
+ invIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterFactory, filterFrameFactory,
+ filterManager, bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits,
+ invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker,
+ ioScheduler, ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps,
invertedIndexFieldsForNonBulkLoadOps, durable);
return invIndex;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
index 7c57bed..1b2ff6f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
@@ -100,7 +100,7 @@
typeTraits,
rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback(), linearizeCmpFactory, btreeFields, version, durable,
+ ioOpCallbackFactory.createIoOpCallback(), linearizeCmpFactory, btreeFields, version, durable,
isPointMBR, (IMetadataPageManagerFactory) opDesc.getPageManagerFactory());
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
index 3f69016..63633aa 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
@@ -89,7 +89,7 @@
diskFileMapProvider,
typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback(), linearizeCmpFactory, rtreeFields, btreeFields,
+ ioOpCallbackFactory.createIoOpCallback(), linearizeCmpFactory, rtreeFields, btreeFields,
filterTypeTraits, filterCmpFactories, filterFields, durable, isPointMBR,
(IMetadataPageManagerFactory) opDesc.getPageManagerFactory());
} catch (TreeIndexException e) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
index 8a807c2..6e0ffaf 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
@@ -70,7 +70,7 @@
return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(ctx.getIOManager(), virtualBufferCaches, file,
diskBufferCache, diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories,
valueProviderFactories, rtreePolicyType, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory
- .createIOOperationCallback(), linearizeCmpFactory, rtreeFields, filterTypeTraits,
+ .createIoOpCallback(), linearizeCmpFactory, rtreeFields, filterTypeTraits,
filterCmpFactories, filterFields, durable, isPointMBR, (IMetadataPageManagerFactory) opDesc
.getPageManagerFactory());
} catch (TreeIndexException e) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index fc5b6b8..480e1e2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -41,8 +41,9 @@
import org.apache.hyracks.storage.am.common.impls.AbstractSearchPredicate;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
@@ -71,7 +72,7 @@
// On-disk components.
// For creating RTree's used in flush and merge.
- protected final ILSMComponentFactory componentFactory;
+ protected final ILSMDiskComponentFactory componentFactory;
protected IBinaryComparatorFactory[] btreeCmpFactories;
protected IBinaryComparatorFactory[] rtreeCmpFactories;
@@ -86,30 +87,31 @@
public AbstractLSMRTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
- ILSMIndexFileManager fileManager, ILSMComponentFactory componentFactory,
+ ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory,
IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFactory filterFactory,
ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
- int[] rtreeFields, int[] filterFields, boolean durable, boolean isPointMBR) throws HyracksDataException {
- super(ioManager, virtualBufferCaches, componentFactory.getBufferCache(), fileManager, diskFileMapProvider,
+ int[] rtreeFields, int[] filterFields, boolean durable, boolean isPointMBR, IBufferCache diskBufferCache)
+ throws HyracksDataException {
+ super(ioManager, virtualBufferCaches, diskBufferCache, fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
filterManager, filterFields, durable);
int i = 0;
for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
RTree memRTree = new RTree(virtualBufferCache, virtualBufferCache.getFileMapProvider(),
- new VirtualFreePageManager(virtualBufferCache), rtreeInteriorFrameFactory,
- rtreeLeafFrameFactory, rtreeCmpFactories, fieldCount,
+ new VirtualFreePageManager(virtualBufferCache), rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
+ rtreeCmpFactories, fieldCount,
ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_r_" + i), isPointMBR);
BTree memBTree = new BTree(virtualBufferCache, virtualBufferCache.getFileMapProvider(),
- new VirtualFreePageManager(virtualBufferCache), btreeInteriorFrameFactory,
- btreeLeafFrameFactory, btreeCmpFactories, btreeCmpFactories.length,
+ new VirtualFreePageManager(virtualBufferCache), btreeInteriorFrameFactory, btreeLeafFrameFactory,
+ btreeCmpFactories, btreeCmpFactories.length,
ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_b_" + i));
- LSMRTreeMemoryComponent mutableComponent = new LSMRTreeMemoryComponent(memRTree, memBTree,
- virtualBufferCache, i == 0 ? true : false,
- filterFactory == null ? null : filterFactory.createLSMComponentFilter());
+ LSMRTreeMemoryComponent mutableComponent =
+ new LSMRTreeMemoryComponent(memRTree, memBTree, virtualBufferCache, i == 0 ? true : false,
+ filterFactory == null ? null : filterFactory.createFilter());
memoryComponents.add(mutableComponent);
++i;
}
@@ -134,15 +136,14 @@
public AbstractLSMRTree(IIOManager ioManager, ITreeIndexFrameFactory rtreeInteriorFrameFactory,
ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory,
ITreeIndexFrameFactory btreeLeafFrameFactory, ILSMIndexFileManager fileManager,
- ILSMComponentFactory componentFactory, IFileMapProvider diskFileMapProvider, int fieldCount,
+ ILSMDiskComponentFactory componentFactory, IFileMapProvider diskFileMapProvider,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean durable,
- boolean isPointMBR) {
- super(ioManager, componentFactory.getBufferCache(), fileManager, diskFileMapProvider,
- bloomFilterFalsePositiveRate,
- mergePolicy, opTracker, ioScheduler, ioOpCallback, durable);
+ boolean isPointMBR, IBufferCache diskBufferCache) {
+ super(ioManager, diskBufferCache, fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy,
+ opTracker, ioScheduler, ioOpCallback, durable);
this.rtreeInteriorFrameFactory = rtreeInteriorFrameFactory;
this.rtreeLeafFrameFactory = rtreeLeafFrameFactory;
this.btreeInteriorFrameFactory = btreeInteriorFrameFactory;
@@ -183,7 +184,8 @@
if (flushOnExit) {
BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
- ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ ILSMIndexAccessor accessor =
+ createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
accessor.scheduleFlush(cb);
try {
cb.waitForIO();
@@ -212,7 +214,7 @@
@Override
public void getOperationalComponents(ILSMIndexOperationContext ctx) throws HyracksDataException {
List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
- List<ILSMComponent> immutableComponents = diskComponents;
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
int cmc = currentMutableComponentId.get();
ctx.setCurrentMutableComponentId(cmc);
operationalComponents.clear();
@@ -264,21 +266,21 @@
protected LSMComponentFileReferences getMergeTargetFileName(List<ILSMComponent> mergingDiskComponents)
throws HyracksDataException {
RTree firstTree = ((LSMRTreeDiskComponent) mergingDiskComponents.get(0)).getRTree();
- RTree lastTree = ((LSMRTreeDiskComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1))
- .getRTree();
+ RTree lastTree =
+ ((LSMRTreeDiskComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1)).getRTree();
FileReference firstFile = firstTree.getFileReference();
FileReference lastFile = lastTree.getFileReference();
- LSMComponentFileReferences fileRefs = fileManager.getRelMergeFileReference(firstFile.getFile().getName(),
- lastFile.getFile().getName());
+ LSMComponentFileReferences fileRefs =
+ fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
return fileRefs;
}
- protected LSMRTreeDiskComponent createDiskComponent(ILSMComponentFactory factory, FileReference insertFileRef,
+ protected LSMRTreeDiskComponent createDiskComponent(ILSMDiskComponentFactory factory, FileReference insertFileRef,
FileReference deleteFileRef, FileReference bloomFilterFileRef, boolean createComponent)
throws HyracksDataException, IndexException {
// Create new tree instance.
- LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) factory.createLSMComponentInstance(
- new LSMComponentFileReferences(insertFileRef, deleteFileRef, bloomFilterFileRef));
+ LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) factory
+ .createComponent(new LSMComponentFileReferences(insertFileRef, deleteFileRef, bloomFilterFileRef));
// Tree will be closed during cleanup of merge().
if (createComponent) {
component.getRTree().create();
@@ -293,55 +295,56 @@
component.getBloomFilter().activate();
}
if (component.getLSMComponentFilter() != null && !createComponent) {
- filterManager.readFilterInfo(component.getLSMComponentFilter(), component.getRTree());
+ filterManager.readFilter(component.getLSMComponentFilter(), component.getRTree());
}
return component;
}
@Override
public ITreeIndexFrameFactory getLeafFrameFactory() {
- LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) memoryComponents
- .get(currentMutableComponentId.get());
+ LSMRTreeMemoryComponent mutableComponent =
+ (LSMRTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
return mutableComponent.getRTree().getLeafFrameFactory();
}
@Override
public ITreeIndexFrameFactory getInteriorFrameFactory() {
- LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) memoryComponents
- .get(currentMutableComponentId.get());
+ LSMRTreeMemoryComponent mutableComponent =
+ (LSMRTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
return mutableComponent.getRTree().getInteriorFrameFactory();
}
@Override
public IPageManager getPageManager() {
- LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) memoryComponents
- .get(currentMutableComponentId.get());
+ LSMRTreeMemoryComponent mutableComponent =
+ (LSMRTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
return mutableComponent.getRTree().getPageManager();
}
@Override
public int getFieldCount() {
- LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) memoryComponents
- .get(currentMutableComponentId.get());
+ LSMRTreeMemoryComponent mutableComponent =
+ (LSMRTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
return mutableComponent.getRTree().getFieldCount();
}
@Override
public int getRootPageId() {
- LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) memoryComponents
- .get(currentMutableComponentId.get());
+ LSMRTreeMemoryComponent mutableComponent =
+ (LSMRTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
return mutableComponent.getRTree().getRootPageId();
}
@Override
public int getFileId() {
- LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) memoryComponents
- .get(currentMutableComponentId.get());
+ LSMRTreeMemoryComponent mutableComponent =
+ (LSMRTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
return mutableComponent.getRTree().getFileId();
}
@Override
- public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
+ public void modify(IIndexOperationContext ictx, ITupleReference tuple)
+ throws HyracksDataException, IndexException {
LSMRTreeOpContext ctx = (LSMRTreeOpContext) ictx;
if (ctx.getOperation() == IndexOperation.PHYSICALDELETE) {
throw new UnsupportedOperationException("Physical delete not supported in the LSM-RTree");
@@ -380,8 +383,8 @@
protected LSMRTreeOpContext createOpContext(IModificationOperationCallback modCallback,
ISearchOperationCallback searchCallback) {
return new LSMRTreeOpContext(memoryComponents, rtreeLeafFrameFactory, rtreeInteriorFrameFactory,
- btreeLeafFrameFactory, btreeInteriorFrameFactory, rtreeCmpFactories, btreeCmpFactories, modCallback,
- searchCallback, rtreeFields, filterFields, lsmHarness, comparatorFields, linearizerArray);
+ btreeLeafFrameFactory, modCallback, searchCallback, rtreeFields, filterFields, lsmHarness,
+ comparatorFields, linearizerArray);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
index bfaba68..f846c6c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
@@ -46,11 +46,12 @@
import org.apache.hyracks.storage.am.common.api.TreeIndexException;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -76,7 +77,7 @@
// A second disk component list that will be used when a transaction is
// committed and will be seen by subsequent accessors
- private final List<ILSMComponent> secondDiskComponents;
+ private final List<ILSMDiskComponent> secondDiskComponents;
// A pointer that points to the current most recent list (either
// diskComponents = 0, or secondDiskComponents = 1). It starts with -1 to
// indicate first time activation
@@ -84,49 +85,34 @@
private final int fieldCount;
public ExternalRTree(IIOManager ioManager, ITreeIndexFrameFactory rtreeInteriorFrameFactory,
- ITreeIndexFrameFactory rtreeLeafFrameFactory,
- ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
- ILSMIndexFileManager fileNameManager, TreeIndexFactory<RTree> diskRTreeFactory,
- TreeIndexFactory<BTree> diskBTreeFactory, BloomFilterFactory bloomFilterFactory,
- double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount,
- IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
- ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields, int version, boolean durable,
- boolean isPointMBR) {
+ ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory,
+ ITreeIndexFrameFactory btreeLeafFrameFactory, ILSMIndexFileManager fileNameManager,
+ TreeIndexFactory<RTree> diskRTreeFactory, TreeIndexFactory<BTree> diskBTreeFactory,
+ BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
+ IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories,
+ IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
+ int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ int[] buddyBTreeFields, int version, boolean durable, boolean isPointMBR) {
super(ioManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory,
- btreeLeafFrameFactory,
- fileNameManager, diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate,
- diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields,
- linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, buddyBTreeFields, durable,
- isPointMBR);
+ btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, diskBTreeFactory, bloomFilterFactory,
+ bloomFilterFalsePositiveRate, diskFileMapProvider, rtreeCmpFactories, btreeCmpFactories, linearizer,
+ comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, buddyBTreeFields,
+ durable, isPointMBR);
this.secondDiskComponents = new LinkedList<>();
this.version = version;
this.fieldCount = fieldCount;
}
- // This method is used to create a target for a bulk modify operation. This
- // component must then eventually be either committed or deleted
- private ILSMComponent createTransactionTarget() throws HyracksDataException, IndexException {
- LSMComponentFileReferences componentFileRefs;
- try {
- componentFileRefs = fileManager.getNewTransactionFileReference();
- } catch (IOException e) {
- throw new HyracksDataException("Failed to create transaction components", e);
- }
- return createDiskComponent(componentFactory, componentFileRefs.getInsertIndexFileReference(),
- componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true);
- }
-
// The subsume merged components is overridden to account for:
// 1. the number of readers of components
// 2. maintaining two versions of the index
@Override
- public void subsumeMergedComponents(ILSMComponent newComponent, List<ILSMComponent> mergedComponents)
+ public void subsumeMergedComponents(ILSMDiskComponent newComponent, List<ILSMComponent> mergedComponents)
throws HyracksDataException {
// determine which list is the new one
- List<ILSMComponent> newerList;
- List<ILSMComponent> olderList;
+ List<ILSMDiskComponent> newerList;
+ List<ILSMDiskComponent> olderList;
if (version == 0) {
newerList = diskComponents;
olderList = secondDiskComponents;
@@ -150,7 +136,7 @@
// is needed.
// It only needs to return the newer list
@Override
- public List<ILSMComponent> getImmutableComponents() {
+ public List<ILSMDiskComponent> getImmutableComponents() {
if (version == 0) {
return diskComponents;
} else {
@@ -169,7 +155,7 @@
// is successful
// it will therefore add the component to the first list and enter it.
@Override
- public void addComponent(ILSMComponent c) throws HyracksDataException {
+ public void addDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
if (version == 0) {
diskComponents.add(0, c);
} else if (version == 1) {
@@ -179,11 +165,11 @@
// This function is used when a new component is to be committed.
@Override
- public void commitTransactionDiskComponent(ILSMComponent newComponent) throws HyracksDataException {
+ public void commitTransactionDiskComponent(ILSMDiskComponent newComponent) throws HyracksDataException {
// determine which list is the new one and flip the pointer
- List<ILSMComponent> newerList;
- List<ILSMComponent> olderList;
+ List<ILSMDiskComponent> newerList;
+ List<ILSMDiskComponent> olderList;
if (version == 0) {
newerList = diskComponents;
olderList = secondDiskComponents;
@@ -281,7 +267,7 @@
// This can be done in a better way by creating a method boolean
// keepDeletedTuples(mergedComponents);
@Override
- public ILSMComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation;
ITreeIndexCursor cursor = mergeOp.getCursor();
ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
@@ -313,8 +299,8 @@
}
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
- BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
- bloomFilterFalsePositiveRate);
+ BloomFilterSpecification bloomFilterSpec =
+ BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate);
IIndexBulkLoader builder = mergedComponent.getBloomFilter().createBuilder(numElements,
bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
@@ -444,7 +430,8 @@
// Not supported
@Override
- public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
+ public void modify(IIndexOperationContext ictx, ITupleReference tuple)
+ throws HyracksDataException, IndexException {
throw new UnsupportedOperationException("tuple modify not supported in LSM-Disk-Only-RTree");
}
@@ -457,7 +444,7 @@
// Not supported
@Override
- public ILSMComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
throw new UnsupportedOperationException("flush not supported in LSM-Disk-Only-RTree");
}
@@ -465,7 +452,7 @@
@Override
public void getOperationalComponents(ILSMIndexOperationContext ctx) {
List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
- List<ILSMComponent> immutableComponents;
+ List<ILSMDiskComponent> immutableComponents;
// Identify current list in case of a merge
if (version == 0) {
immutableComponents = diskComponents;
@@ -527,7 +514,7 @@
// The bulk loader used for both initial loading and transaction
// modifications
public class LSMTwoPCRTreeBulkLoader implements IIndexBulkLoader, ITwoPCIndexBulkLoader {
- private final ILSMComponent component;
+ private final ILSMDiskComponent component;
private final IIndexBulkLoader rtreeBulkLoader;
private final BTreeBulkLoader btreeBulkLoader;
private final IIndexBulkLoader builder;
@@ -563,8 +550,8 @@
btreeBulkLoader = (BTreeBulkLoader) ((LSMRTreeDiskComponent) component).getBTree()
.createBulkLoader(fillFactor, verifyInput, numElementsHint, false);
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
- BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
- bloomFilterFalsePositiveRate);
+ BloomFilterSpecification bloomFilterSpec =
+ BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate);
builder = ((LSMRTreeDiskComponent) component).getBloomFilter().createBuilder(numElementsHint,
bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
}
@@ -657,6 +644,20 @@
}
}
+
+ // This method is used to create a target for a bulk modify operation. This
+ // component must then eventually be either committed or deleted
+ private ILSMDiskComponent createTransactionTarget() throws HyracksDataException, IndexException {
+ LSMComponentFileReferences componentFileRefs;
+ try {
+ componentFileRefs = fileManager.getNewTransactionFileReference();
+ } catch (IOException e) {
+ throw new HyracksDataException("Failed to create transaction components", e);
+ }
+ return createDiskComponent(componentFactory, componentFileRefs.getInsertIndexFileReference(),
+ componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(),
+ true);
+ }
}
@Override
@@ -674,7 +675,7 @@
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
ITreeIndexCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer, buddyBTreeFields);
LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
- ILSMIndexAccessorInternal accessor = new LSMRTreeAccessor(lsmHarness, rctx);
+ ILSMIndexAccessor accessor = new LSMRTreeAccessor(lsmHarness, rctx);
// create the merge operation.
LSMRTreeMergeOperation mergeOp = new LSMRTreeMergeOperation(accessor, mergingComponents, cursor,
relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(),
@@ -695,7 +696,7 @@
}
@Override
- public ILSMIndexAccessorInternal createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion)
+ public ILSMIndexAccessor createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion)
throws HyracksDataException {
return new LSMRTreeAccessor(lsmHarness, createOpContext(searchCallback, targetIndexVersion));
}
@@ -710,7 +711,7 @@
// The accessor for disk only indexes don't use modification callback and
// always carry the target index version with them
@Override
- public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
+ public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
return new LSMRTreeAccessor(lsmHarness, createOpContext(searchCallback, version));
}
@@ -721,12 +722,12 @@
}
@Override
- public List<ILSMComponent> getFirstComponentList() {
+ public List<ILSMDiskComponent> getFirstComponentList() {
return diskComponents;
}
@Override
- public List<ILSMComponent> getSecondComponentList() {
+ public List<ILSMDiskComponent> getSecondComponentList() {
return secondDiskComponents;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
index 358a42a..dcd2c67 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
@@ -28,6 +28,7 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
@@ -37,8 +38,8 @@
private MultiComparator bTreeCmp;
private MultiComparator rTreeCmp;
public final List<ILSMComponent> componentHolder;
- private final List<ILSMComponent> componentsToBeMerged;
- private final List<ILSMComponent> componentsToBeReplicated;
+ private final List<ILSMDiskComponent> componentsToBeMerged;
+ private final List<ILSMDiskComponent> componentsToBeReplicated;
public final ISearchOperationCallback searchCallback;
private final int targetIndexVersion;
public ISearchPredicate searchPredicate;
@@ -49,16 +50,16 @@
int targetIndexVersion, ILSMHarness lsmHarness, int[] comparatorFields,
IBinaryComparatorFactory[] linearizerArray, ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory) {
- this.componentHolder = new LinkedList<ILSMComponent>();
- this.componentsToBeMerged = new LinkedList<ILSMComponent>();
- this.componentsToBeReplicated = new LinkedList<ILSMComponent>();
+ this.componentHolder = new LinkedList<>();
+ this.componentsToBeMerged = new LinkedList<>();
+ this.componentsToBeReplicated = new LinkedList<>();
this.searchCallback = searchCallback;
this.targetIndexVersion = targetIndexVersion;
this.bTreeCmp = MultiComparator.create(btreeCmpFactories);
this.rTreeCmp = MultiComparator.create(rtreeCmpFactories);
- initialState = new LSMRTreeCursorInitialState(rtreeLeafFrameFactory, rtreeInteriorFrameFactory,
- btreeLeafFrameFactory, bTreeCmp, lsmHarness, comparatorFields, linearizerArray, searchCallback,
- componentHolder);
+ initialState =
+ new LSMRTreeCursorInitialState(rtreeLeafFrameFactory, rtreeInteriorFrameFactory, btreeLeafFrameFactory,
+ bTreeCmp, lsmHarness, comparatorFields, linearizerArray, searchCallback, componentHolder);
}
@Override
@@ -110,7 +111,7 @@
}
@Override
- public List<ILSMComponent> getComponentsToBeMerged() {
+ public List<ILSMDiskComponent> getComponentsToBeMerged() {
return componentsToBeMerged;
}
@@ -129,7 +130,7 @@
}
@Override
- public List<ILSMComponent> getComponentsToBeReplicated() {
+ public List<ILSMDiskComponent> getComponentsToBeReplicated() {
return componentsToBeReplicated;
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index bfaef38..e62ccdc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -54,6 +54,7 @@
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.am.common.tuples.DualTupleReference;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
@@ -61,7 +62,7 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -81,25 +82,26 @@
protected int[] buddyBTreeFields;
public LSMRTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- ITreeIndexFrameFactory rtreeInteriorFrameFactory,
- ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory,
- ITreeIndexFrameFactory btreeLeafFrameFactory, ILSMIndexFileManager fileNameManager,
- TreeIndexFactory<RTree> diskRTreeFactory, TreeIndexFactory<BTree> diskBTreeFactory,
- BloomFilterFactory bloomFilterFactory, ILSMComponentFilterFactory filterFactory,
- ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
- double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount,
- IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
- ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, int[] rtreeFields, int[] buddyBTreeFields, int[] filterFields,
- boolean durable, boolean isPointMBR) throws HyracksDataException {
+ ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
+ ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
+ ILSMIndexFileManager fileNameManager, TreeIndexFactory<RTree> diskRTreeFactory,
+ TreeIndexFactory<BTree> diskBTreeFactory, BloomFilterFactory bloomFilterFactory,
+ ILSMComponentFilterFactory filterFactory, ILSMComponentFilterFrameFactory filterFrameFactory,
+ LSMComponentFilterManager filterManager, double bloomFilterFalsePositiveRate,
+ IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories,
+ IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
+ int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ int[] rtreeFields, int[] buddyBTreeFields, int[] filterFields, boolean durable, boolean isPointMBR)
+ throws HyracksDataException {
super(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
- btreeInteriorFrameFactory,
- btreeLeafFrameFactory, fileNameManager,
- new LSMRTreeDiskComponentFactory(diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, filterFactory),
+ btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager,
+ new LSMRTreeDiskComponentFactory(diskRTreeFactory, diskBTreeFactory, bloomFilterFactory,
+ filterFactory),
diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields,
linearizerArray, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback,
- filterFactory, filterFrameFactory, filterManager, rtreeFields, filterFields, durable, isPointMBR);
+ filterFactory, filterFrameFactory, filterManager, rtreeFields, filterFields, durable, isPointMBR,
+ diskRTreeFactory.getBufferCache());
this.buddyBTreeFields = buddyBTreeFields;
}
@@ -107,35 +109,34 @@
* For External indexes with no memory components
*/
public LSMRTree(IIOManager ioManager, ITreeIndexFrameFactory rtreeInteriorFrameFactory,
- ITreeIndexFrameFactory rtreeLeafFrameFactory,
- ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
- ILSMIndexFileManager fileNameManager, TreeIndexFactory<RTree> diskRTreeFactory,
- TreeIndexFactory<BTree> diskBTreeFactory, BloomFilterFactory bloomFilterFactory,
- double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount,
- IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
- ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields, boolean durable, boolean isPointMBR) {
+ ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory,
+ ITreeIndexFrameFactory btreeLeafFrameFactory, ILSMIndexFileManager fileNameManager,
+ TreeIndexFactory<RTree> diskRTreeFactory, TreeIndexFactory<BTree> diskBTreeFactory,
+ BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
+ IFileMapProvider diskFileMapProvider, IBinaryComparatorFactory[] rtreeCmpFactories,
+ IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
+ int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ int[] buddyBTreeFields, boolean durable, boolean isPointMBR) {
super(ioManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory,
- btreeLeafFrameFactory,
- fileNameManager,
+ btreeLeafFrameFactory, fileNameManager,
new LSMRTreeDiskComponentFactory(diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, null),
- diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields,
+ diskFileMapProvider, rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields,
linearizerArray, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback,
- durable, isPointMBR);
+ durable, isPointMBR, diskRTreeFactory.getBufferCache());
this.buddyBTreeFields = buddyBTreeFields;
}
/**
* Opens LSMRTree, cleaning up invalid files from base dir, and registering
* all valid files as on-disk RTrees and BTrees.
- *
+ *
* @throws HyracksDataException
*/
@Override
public synchronized void activate() throws HyracksDataException {
super.activate();
- List<ILSMComponent> immutableComponents = diskComponents;
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
List<LSMComponentFileReferences> validFileReferences;
try {
validFileReferences = fileManager.cleanupAndGetValidFiles();
@@ -146,10 +147,10 @@
for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
LSMRTreeDiskComponent component;
try {
- component = createDiskComponent(componentFactory,
- lsmComonentFileReference.getInsertIndexFileReference(),
- lsmComonentFileReference.getDeleteIndexFileReference(),
- lsmComonentFileReference.getBloomFilterFileReference(), false);
+ component =
+ createDiskComponent(componentFactory, lsmComonentFileReference.getInsertIndexFileReference(),
+ lsmComonentFileReference.getDeleteIndexFileReference(),
+ lsmComonentFileReference.getBloomFilterFileReference(), false);
} catch (IndexException e) {
throw new HyracksDataException(e);
}
@@ -161,8 +162,8 @@
@Override
public synchronized void deactivate(boolean flushOnExit) throws HyracksDataException {
super.deactivate(flushOnExit);
- List<ILSMComponent> immutableComponents = diskComponents;
- for (ILSMComponent c : immutableComponents) {
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
+ for (ILSMDiskComponent c : immutableComponents) {
LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
RTree rtree = component.getRTree();
BTree btree = component.getBTree();
@@ -182,8 +183,8 @@
@Override
public synchronized void destroy() throws HyracksDataException {
super.destroy();
- List<ILSMComponent> immutableComponents = diskComponents;
- for (ILSMComponent c : immutableComponents) {
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
+ for (ILSMDiskComponent c : immutableComponents) {
LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
component.getBTree().destroy();
component.getBloomFilter().destroy();
@@ -195,8 +196,8 @@
@Override
public synchronized void clear() throws HyracksDataException {
super.clear();
- List<ILSMComponent> immutableComponents = diskComponents;
- for (ILSMComponent c : immutableComponents) {
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
+ for (ILSMDiskComponent c : immutableComponents) {
LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
component.getBTree().deactivate();
component.getBloomFilter().deactivate();
@@ -213,8 +214,8 @@
throws HyracksDataException {
ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- ILSMIndexOperationContext rctx = createOpContext(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
+ ILSMIndexOperationContext rctx =
+ createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
rctx.setOperation(IndexOperation.FLUSH);
rctx.getComponentHolder().addAll(ctx.getComponentHolder());
LSMRTreeAccessor accessor = new LSMRTreeAccessor(lsmHarness, rctx);
@@ -224,7 +225,7 @@
}
@Override
- public ILSMComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
LSMRTreeFlushOperation flushOp = (LSMRTreeFlushOperation) operation;
LSMRTreeMemoryComponent flushingComponent = (LSMRTreeMemoryComponent) flushOp.getFlushingComponent();
// Renaming order is critical because we use assume ordering when we
@@ -297,8 +298,8 @@
}
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numBTreeTuples);
- BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
- bloomFilterFalsePositiveRate);
+ BloomFilterSpecification bloomFilterSpec =
+ BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate);
IIndexCursor btreeScanCursor = memBTreeAccessor.createSearchCursor(false);
memBTreeAccessor.search(btreeScanCursor, btreeNullPredicate);
@@ -325,32 +326,32 @@
List<ITupleReference> filterTuples = new ArrayList<>();
filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
- filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilterInfo(component.getLSMComponentFilter(), component.getRTree());
+ filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
+ filterManager.writeFilter(component.getLSMComponentFilter(), component.getRTree());
}
-
+ // Note. If we change the filter to write to metadata object, we don't need the if block above
+ flushingComponent.getMetadata().copy(component.getMetadata());
bTreeBulkloader.end();
-
return component;
}
@Override
public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException, IndexException {
- ILSMIndexOperationContext rctx = createOpContext(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
+ ILSMIndexOperationContext rctx =
+ createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
rctx.setOperation(IndexOperation.MERGE);
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
ITreeIndexCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer, buddyBTreeFields);
LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
- ILSMIndexAccessorInternal accessor = new LSMRTreeAccessor(lsmHarness, rctx);
+ ILSMIndexAccessor accessor = new LSMRTreeAccessor(lsmHarness, rctx);
ioScheduler.scheduleOperation(new LSMRTreeMergeOperation(accessor, mergingComponents, cursor,
relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(),
relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir()));
}
@Override
- public ILSMComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation;
ITreeIndexCursor cursor = mergeOp.getCursor();
ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
@@ -379,8 +380,8 @@
}
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
- BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
- bloomFilterFalsePositiveRate);
+ BloomFilterSpecification bloomFilterSpec =
+ BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate);
IIndexBulkLoader builder = mergedComponent.getBloomFilter().createBuilder(numElements,
bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
@@ -403,8 +404,8 @@
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
}
- filterManager.updateFilterInfo(mergedComponent.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilterInfo(mergedComponent.getLSMComponentFilter(), mergedComponent.getRTree());
+ filterManager.updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
+ filterManager.writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getRTree());
}
btreeBulkLoader.end();
@@ -424,7 +425,7 @@
}
@Override
- public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
+ public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
return new LSMRTreeAccessor(lsmHarness, createOpContext(modificationCallback, searchCallback));
}
@@ -469,10 +470,11 @@
}
}
- protected ILSMComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
+ protected ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
return createDiskComponent(componentFactory, componentFileRefs.getInsertIndexFileReference(),
- componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true);
+ componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(),
+ true);
}
@Override
@@ -487,7 +489,8 @@
// This function is modified for R-Trees without antimatter tuples to allow buddy B-Tree to have only primary keys
@Override
- public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
+ public void modify(IIndexOperationContext ictx, ITupleReference tuple)
+ throws HyracksDataException, IndexException {
LSMRTreeOpContext ctx = (LSMRTreeOpContext) ictx;
if (ctx.getOperation() == IndexOperation.PHYSICALDELETE) {
throw new UnsupportedOperationException("Physical delete not supported in the LSM-RTree");
@@ -523,7 +526,7 @@
}
public class LSMRTreeBulkLoader implements IIndexBulkLoader {
- private final ILSMComponent component;
+ private final ILSMDiskComponent component;
private final IIndexBulkLoader bulkLoader;
private final IIndexBulkLoader buddyBTreeBulkloader;
private boolean cleanedUpArtifacts = false;
@@ -590,7 +593,7 @@
if (!cleanedUpArtifacts) {
if (component.getLSMComponentFilter() != null) {
- filterManager.writeFilterInfo(component.getLSMComponentFilter(),
+ filterManager.writeFilter(component.getLSMComponentFilter(),
((LSMRTreeDiskComponent) component).getRTree());
}
@@ -630,7 +633,7 @@
}
@Override
- public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException {
+ public void markAsValid(ILSMDiskComponent lsmComponent) throws HyracksDataException {
LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) lsmComponent;
markAsValidInternal(component.getBTree().getBufferCache(), component.getBloomFilter());
markAsValidInternal((component).getBTree());
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java
index 4e6952b..982f89b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java
@@ -21,17 +21,18 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import org.apache.hyracks.storage.am.btree.impls.BTree;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
-import org.apache.hyracks.storage.am.lsm.common.impls.AbstractDiskLSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponent;
import org.apache.hyracks.storage.am.rtree.impls.RTree;
-public class LSMRTreeDiskComponent extends AbstractDiskLSMComponent {
+public class LSMRTreeDiskComponent extends AbstractLSMDiskComponent {
private final RTree rtree;
private final BTree btree;
private final BloomFilter bloomFilter;
public LSMRTreeDiskComponent(RTree rtree, BTree btree, BloomFilter bloomFilter, ILSMComponentFilter filter) {
- super(filter);
+ super((IMetadataPageManager) rtree.getPageManager(), filter);
this.rtree = rtree;
this.btree = btree;
this.bloomFilter = bloomFilter;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentFactory.java
index a102ce5..2e457d9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentFactory.java
@@ -22,16 +22,13 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import org.apache.hyracks.storage.am.btree.impls.BTree;
-import org.apache.hyracks.storage.am.common.api.IndexException;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.am.rtree.impls.RTree;
-import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-public class LSMRTreeDiskComponentFactory implements ILSMComponentFactory {
+public class LSMRTreeDiskComponentFactory implements ILSMDiskComponentFactory {
private final TreeIndexFactory<RTree> rtreeFactory;
private final TreeIndexFactory<BTree> btreeFactory;
private final BloomFilterFactory bloomFilterFactory;
@@ -46,16 +43,10 @@
}
@Override
- public ILSMComponent createLSMComponentInstance(LSMComponentFileReferences cfr) throws IndexException,
- HyracksDataException {
+ public LSMRTreeDiskComponent createComponent(LSMComponentFileReferences cfr) throws HyracksDataException {
return new LSMRTreeDiskComponent(rtreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()),
btreeFactory.createIndexInstance(cfr.getDeleteIndexFileReference()),
bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()),
- filterFactory == null ? null : filterFactory.createLSMComponentFilter());
- }
-
- @Override
- public IBufferCache getBufferCache() {
- return rtreeFactory.getBufferCache();
+ filterFactory == null ? null : filterFactory.createFilter());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
index 31455ae..013821d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
@@ -29,11 +29,11 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
public class LSMRTreeFlushOperation implements ILSMIOOperation, Comparable<LSMRTreeFlushOperation> {
- private final ILSMIndexAccessorInternal accessor;
+ private final ILSMIndexAccessor accessor;
private final ILSMComponent flushingComponent;
private final FileReference rtreeFlushTarget;
private final FileReference btreeFlushTarget;
@@ -41,7 +41,7 @@
private final ILSMIOOperationCallback callback;
private final String indexIdentifier;
- public LSMRTreeFlushOperation(ILSMIndexAccessorInternal accessor, ILSMComponent flushingComponent,
+ public LSMRTreeFlushOperation(ILSMIndexAccessor accessor, ILSMComponent flushingComponent,
FileReference rtreeFlushTarget, FileReference btreeFlushTarget, FileReference bloomFilterFlushTarget,
ILSMIOOperationCallback callback, String indexIdentifier) {
this.accessor = accessor;
@@ -60,7 +60,7 @@
@Override
public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ Set<IODeviceHandle> devs = new HashSet<>();
devs.add(rtreeFlushTarget.getDeviceHandle());
if (btreeFlushTarget != null) {
devs.add(btreeFlushTarget.getDeviceHandle());
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMemoryComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMemoryComponent.java
index 339059f..957e2a9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMemoryComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMemoryComponent.java
@@ -23,10 +23,10 @@
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
-import org.apache.hyracks.storage.am.lsm.common.impls.AbstractMemoryLSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMMemoryComponent;
import org.apache.hyracks.storage.am.rtree.impls.RTree;
-public class LSMRTreeMemoryComponent extends AbstractMemoryLSMComponent {
+public class LSMRTreeMemoryComponent extends AbstractLSMMemoryComponent {
private final RTree rtree;
private final BTree btree;
@@ -47,7 +47,7 @@
}
@Override
- protected void reset() throws HyracksDataException {
+ public void reset() throws HyracksDataException {
super.reset();
rtree.deactivate();
rtree.destroy();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java
index 12b47b4..6a12ab2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java
@@ -30,10 +30,10 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
public class LSMRTreeMergeOperation implements ILSMIOOperation {
- private final ILSMIndexAccessorInternal accessor;
+ private final ILSMIndexAccessor accessor;
private final List<ILSMComponent> mergingComponents;
private final ITreeIndexCursor cursor;
private final FileReference rtreeMergeTarget;
@@ -43,7 +43,7 @@
private final String indexIdentifier;
private boolean keepDeletedTuples;
- public LSMRTreeMergeOperation(ILSMIndexAccessorInternal accessor, List<ILSMComponent> mergingComponents,
+ public LSMRTreeMergeOperation(ILSMIndexAccessor accessor, List<ILSMComponent> mergingComponents,
ITreeIndexCursor cursor, FileReference rtreeMergeTarget, FileReference btreeMergeTarget,
FileReference bloomFilterMergeTarget, ILSMIOOperationCallback callback, String indexIdentifier) {
this.accessor = accessor;
@@ -58,7 +58,7 @@
@Override
public Set<IODeviceHandle> getReadDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ Set<IODeviceHandle> devs = new HashSet<>();
for (ILSMComponent o : mergingComponents) {
LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) o;
devs.add(component.getRTree().getFileReference().getDeviceHandle());
@@ -72,7 +72,7 @@
@Override
public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ Set<IODeviceHandle> devs = new HashSet<>();
devs.add(rtreeMergeTarget.getDeviceHandle());
if (btreeMergeTarget != null) {
devs.add(btreeMergeTarget.getDeviceHandle());
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
index 62f572f..433d99d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
@@ -33,8 +33,10 @@
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
import org.apache.hyracks.storage.am.rtree.impls.RTree;
import org.apache.hyracks.storage.am.rtree.impls.RTreeOpContext;
@@ -53,8 +55,8 @@
private IndexOperation op;
public final List<ILSMComponent> componentHolder;
- private final List<ILSMComponent> componentsToBeMerged;
- private final List<ILSMComponent> componentsToBeReplicated;
+ private final List<ILSMDiskComponent> componentsToBeMerged;
+ private final List<ILSMDiskComponent> componentsToBeReplicated;
private IModificationOperationCallback modificationCallback;
private ISearchOperationCallback searchCallback;
public final PermutingTupleReference indexTuple;
@@ -63,12 +65,11 @@
public ISearchPredicate searchPredicate;
public LSMRTreeCursorInitialState searchInitialState;
- public LSMRTreeOpContext(List<ILSMComponent> mutableComponents, ITreeIndexFrameFactory rtreeLeafFrameFactory,
+ public LSMRTreeOpContext(List<ILSMMemoryComponent> mutableComponents, ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
- ITreeIndexFrameFactory btreeInteriorFrameFactory, IBinaryComparatorFactory[] rtreeCmpFactories,
- IBinaryComparatorFactory[] btreeCmpFactories, IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback, int[] rtreeFields, int[] filterFields, ILSMHarness lsmHarness,
- int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray) {
+ IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback,
+ int[] rtreeFields, int[] filterFields, ILSMHarness lsmHarness, int[] comparatorFields,
+ IBinaryComparatorFactory[] linearizerArray) {
mutableRTreeAccessors = new RTree.RTreeAccessor[mutableComponents.size()];
mutableBTreeAccessors = new BTree.BTreeAccessor[mutableComponents.size()];
rtreeOpContexts = new RTreeOpContext[mutableComponents.size()];
@@ -90,9 +91,9 @@
assert mutableComponents.size() > 0;
currentRTreeOpContext = rtreeOpContexts[0];
currentBTreeOpContext = btreeOpContexts[0];
- this.componentHolder = new LinkedList<ILSMComponent>();
- this.componentsToBeMerged = new LinkedList<ILSMComponent>();
- this.componentsToBeReplicated = new LinkedList<ILSMComponent>();
+ this.componentHolder = new LinkedList<>();
+ this.componentsToBeMerged = new LinkedList<>();
+ this.componentsToBeReplicated = new LinkedList<>();
this.modificationCallback = modificationCallback;
this.searchCallback = searchCallback;
@@ -161,7 +162,7 @@
}
@Override
- public List<ILSMComponent> getComponentsToBeMerged() {
+ public List<ILSMDiskComponent> getComponentsToBeMerged() {
return componentsToBeMerged;
}
@@ -176,7 +177,7 @@
}
@Override
- public List<ILSMComponent> getComponentsToBeReplicated() {
+ public List<ILSMDiskComponent> getComponentsToBeReplicated() {
return componentsToBeReplicated;
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index 6b67494..179865a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -44,15 +44,16 @@
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
@@ -73,7 +74,7 @@
// On-disk components.
// For creating RTree's used in bulk load. Different from diskRTreeFactory
// because it should have a different tuple writer in it's leaf frames.
- private final ILSMComponentFactory bulkLoaComponentFactory;
+ private final ILSMDiskComponentFactory bulkLoaComponentFactory;
public LSMRTreeWithAntiMatterTuples(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
@@ -87,20 +88,20 @@
ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
int[] rtreeFields, int[] filterFields, boolean durable, boolean isPointMBR) throws HyracksDataException {
super(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
- btreeInteriorFrameFactory,
- btreeLeafFrameFactory, fileManager,
+ btreeInteriorFrameFactory, btreeLeafFrameFactory, fileManager,
new LSMRTreeWithAntiMatterTuplesDiskComponentFactory(diskRTreeFactory, filterFactory),
diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields,
linearizerArray, 0, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFactory,
- filterFrameFactory, filterManager, rtreeFields, filterFields, durable, isPointMBR);
- bulkLoaComponentFactory = new LSMRTreeWithAntiMatterTuplesDiskComponentFactory(bulkLoadRTreeFactory,
- filterFactory);
+ filterFrameFactory, filterManager, rtreeFields, filterFields, durable, isPointMBR,
+ diskRTreeFactory.getBufferCache());
+ bulkLoaComponentFactory =
+ new LSMRTreeWithAntiMatterTuplesDiskComponentFactory(bulkLoadRTreeFactory, filterFactory);
}
@Override
public synchronized void activate() throws HyracksDataException {
super.activate();
- List<ILSMComponent> immutableComponents = diskComponents;
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
immutableComponents.clear();
List<LSMComponentFileReferences> validFileReferences;
try {
@@ -124,8 +125,8 @@
@Override
public synchronized void deactivate(boolean flushOnExit) throws HyracksDataException {
super.deactivate(flushOnExit);
- List<ILSMComponent> immutableComponents = diskComponents;
- for (ILSMComponent c : immutableComponents) {
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
+ for (ILSMDiskComponent c : immutableComponents) {
RTree rtree = ((LSMRTreeDiskComponent) c).getRTree();
rtree.deactivateCloseHandle();
}
@@ -140,8 +141,8 @@
@Override
public synchronized void destroy() throws HyracksDataException {
super.destroy();
- List<ILSMComponent> immutableComponents = diskComponents;
- for (ILSMComponent c : immutableComponents) {
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
+ for (ILSMDiskComponent c : immutableComponents) {
RTree rtree = ((LSMRTreeDiskComponent) c).getRTree();
rtree.destroy();
}
@@ -151,8 +152,8 @@
@Override
public synchronized void clear() throws HyracksDataException {
super.clear();
- List<ILSMComponent> immutableComponents = diskComponents;
- for (ILSMComponent c : immutableComponents) {
+ List<ILSMDiskComponent> immutableComponents = diskComponents;
+ for (ILSMDiskComponent c : immutableComponents) {
RTree rtree = ((LSMRTreeDiskComponent) c).getRTree();
rtree.deactivate();
rtree.destroy();
@@ -168,13 +169,13 @@
LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference();
opCtx.setOperation(IndexOperation.FLUSH);
opCtx.getComponentHolder().add(flushingComponent);
- ILSMIndexAccessorInternal accessor = new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness, opCtx);
+ ILSMIndexAccessor accessor = new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness, opCtx);
ioScheduler.scheduleOperation(new LSMRTreeFlushOperation(accessor, flushingComponent,
relFlushFileRefs.getInsertIndexFileReference(), null, null, callback, fileManager.getBaseDir()));
}
@Override
- public ILSMComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException {
LSMRTreeFlushOperation flushOp = (LSMRTreeFlushOperation) operation;
// Renaming order is critical because we use assume ordering when we
// read the file names when we open the tree.
@@ -185,8 +186,8 @@
RTreeSearchCursor rtreeScanCursor = (RTreeSearchCursor) memRTreeAccessor.createSearchCursor(false);
SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
- LSMRTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getRTreeFlushTarget(), null,
- null, true);
+ LSMRTreeDiskComponent component =
+ createDiskComponent(componentFactory, flushOp.getRTreeFlushTarget(), null, null, true);
RTree diskRTree = component.getRTree();
// scan the memory BTree
@@ -256,9 +257,10 @@
List<ITupleReference> filterTuples = new ArrayList<>();
filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
- filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilterInfo(component.getLSMComponentFilter(), component.getRTree());
+ filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
+ filterManager.writeFilter(component.getLSMComponentFilter(), component.getRTree());
}
+ flushingComponent.getMetadata().copy(component.getMetadata());
rTreeBulkloader.end();
return component;
@@ -277,13 +279,13 @@
}
ITreeIndexCursor cursor = new LSMRTreeWithAntiMatterTuplesSearchCursor(rctx, returnDeletedTuples);
LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
- ILSMIndexAccessorInternal accessor = new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness, rctx);
+ ILSMIndexAccessor accessor = new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness, rctx);
ioScheduler.scheduleOperation(new LSMRTreeMergeOperation(accessor, mergingComponents, cursor,
relMergeFileRefs.getInsertIndexFileReference(), null, null, callback, fileManager.getBaseDir()));
}
@Override
- public ILSMComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
+ public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException {
LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation;
ITreeIndexCursor cursor = mergeOp.getCursor();
ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
@@ -292,8 +294,8 @@
search(opCtx, cursor, rtreeSearchPred);
// Bulk load the tuples from all on-disk RTrees into the new RTree.
- LSMRTreeDiskComponent component = createDiskComponent(componentFactory, mergeOp.getRTreeMergeTarget(), null,
- null, true);
+ LSMRTreeDiskComponent component =
+ createDiskComponent(componentFactory, mergeOp.getRTreeMergeTarget(), null, null, true);
RTree mergedRTree = component.getRTree();
IIndexBulkLoader bulkloader = mergedRTree.createBulkLoader(1.0f, false, 0L, false);
try {
@@ -311,8 +313,8 @@
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
}
- filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilterInfo(component.getLSMComponentFilter(), component.getRTree());
+ filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
+ filterManager.writeFilter(component.getLSMComponentFilter(), component.getRTree());
}
bulkloader.end();
@@ -320,7 +322,7 @@
}
@Override
- public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback,
+ public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
return new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness,
createOpContext(modificationCallback, searchCallback));
@@ -353,14 +355,8 @@
}
}
- private ILSMComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
- LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference();
- return createDiskComponent(bulkLoaComponentFactory, relFlushFileRefs.getInsertIndexFileReference(), null, null,
- true);
- }
-
public class LSMRTreeWithAntiMatterTuplesBulkLoader implements IIndexBulkLoader {
- private final ILSMComponent component;
+ private final ILSMDiskComponent component;
private final IIndexBulkLoader bulkLoader;
private boolean cleanedUpArtifacts = false;
private boolean isEmptyComponent = true;
@@ -426,7 +422,7 @@
if (!cleanedUpArtifacts) {
if (component.getLSMComponentFilter() != null) {
- filterManager.writeFilterInfo(component.getLSMComponentFilter(),
+ filterManager.writeFilter(component.getLSMComponentFilter(),
((LSMRTreeDiskComponent) component).getRTree());
}
bulkLoader.end();
@@ -455,10 +451,16 @@
}
}
+ private ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException, IndexException {
+ LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference();
+ return createDiskComponent(bulkLoaComponentFactory, relFlushFileRefs.getInsertIndexFileReference(), null,
+ null, true);
+ }
+
}
@Override
- public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException {
+ public void markAsValid(ILSMDiskComponent lsmComponent) throws HyracksDataException {
RTree rtree = ((LSMRTreeDiskComponent) lsmComponent).getRTree();
markAsValidInternal(rtree);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentFactory.java
index 1e96fc6..f1af5a1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentFactory.java
@@ -20,16 +20,13 @@
package org.apache.hyracks.storage.am.lsm.rtree.impls;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.common.api.IndexException;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.am.rtree.impls.RTree;
-import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-public class LSMRTreeWithAntiMatterTuplesDiskComponentFactory implements ILSMComponentFactory {
+public class LSMRTreeWithAntiMatterTuplesDiskComponentFactory implements ILSMDiskComponentFactory {
private final TreeIndexFactory<RTree> rtreeFactory;
private final ILSMComponentFilterFactory filterFactory;
@@ -40,14 +37,8 @@
}
@Override
- public ILSMComponent createLSMComponentInstance(LSMComponentFileReferences cfr) throws IndexException,
- HyracksDataException {
+ public LSMRTreeDiskComponent createComponent(LSMComponentFileReferences cfr) throws HyracksDataException {
return new LSMRTreeDiskComponent(rtreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()), null,
- null, filterFactory == null ? null : filterFactory.createLSMComponentFilter());
- }
-
- @Override
- public IBufferCache getBufferCache() {
- return rtreeFactory.getBufferCache();
+ null, filterFactory == null ? null : filterFactory.createFilter());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
index 91fa813..8901c86 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
@@ -24,7 +24,6 @@
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.am.rtree.impls.RTree;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
@@ -44,7 +43,7 @@
}
@Override
- public RTree createIndexInstance(FileReference file) throws IndexException {
+ public RTree createIndexInstance(FileReference file) {
return new RTree(bufferCache, fileMapProvider, freePageManagerFactory.createPageManager(bufferCache),
interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, file, isPointMBR);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
index 90273f4..a3fc0e2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
@@ -24,53 +24,53 @@
public interface IBufferCache {
- long INVALID_DPID = -1l;
+ long INVALID_DPID = -1L;
int INVALID_PAGEID = -1;
int RESERVED_HEADER_BYTES = 8;
- public void createFile(FileReference fileRef) throws HyracksDataException;
+ void createFile(FileReference fileRef) throws HyracksDataException;
- public void openFile(int fileId) throws HyracksDataException;
+ void openFile(int fileId) throws HyracksDataException;
- public void closeFile(int fileId) throws HyracksDataException;
+ void closeFile(int fileId) throws HyracksDataException;
- public void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException;
+ void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException;
- public ICachedPage tryPin(long dpid) throws HyracksDataException;
+ ICachedPage tryPin(long dpid) throws HyracksDataException;
- public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException;
+ ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException;
- public void unpin(ICachedPage page) throws HyracksDataException;
+ void unpin(ICachedPage page) throws HyracksDataException;
- public void flushDirtyPage(ICachedPage page) throws HyracksDataException;
+ void flushDirtyPage(ICachedPage page) throws HyracksDataException;
- public void adviseWontNeed(ICachedPage page);
+ void adviseWontNeed(ICachedPage page);
- public ICachedPage confiscatePage(long dpid) throws HyracksDataException;
+ ICachedPage confiscatePage(long dpid) throws HyracksDataException;
- public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId) throws HyracksDataException;
+ ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId) throws HyracksDataException;
- public void returnPage(ICachedPage page);
+ void returnPage(ICachedPage page);
- public void returnPage(ICachedPage page, boolean reinsert);
+ void returnPage(ICachedPage page, boolean reinsert);
- public void force(int fileId, boolean metadata) throws HyracksDataException;
+ void force(int fileId, boolean metadata) throws HyracksDataException;
int getPageSize();
int getPageSizeWithHeader();
- public int getNumPages();
+ int getNumPages();
- public int getNumPagesOfFile(int fileId) throws HyracksDataException;
+ int getNumPagesOfFile(int fileId) throws HyracksDataException;
- public int getFileReferenceCount(int fileId);
+ int getFileReferenceCount(int fileId);
- public void close() throws HyracksDataException;
+ void close() throws HyracksDataException;
- public IFIFOPageQueue createFIFOQueue();
+ IFIFOPageQueue createFIFOQueue();
- public void finishQueue();
+ void finishQueue();
void setPageDiskId(ICachedPage page, long dpid);
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeExamplesTest.java
index dd4dbbb..6c6161f 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeExamplesTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeExamplesTest.java
@@ -27,7 +27,6 @@
import org.apache.hyracks.storage.am.btree.util.BTreeUtils;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrameFactory;
-import org.apache.hyracks.storage.am.common.api.TreeIndexException;
import org.apache.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import org.apache.hyracks.storage.am.common.freepage.LinkedMetaDataPageManager;
import org.junit.After;
@@ -49,10 +48,10 @@
@Override
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
int[] bloomFilterKeyFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
- int[] btreeFields, int[] filterFields) throws TreeIndexException {
+ int[] btreeFields, int[] filterFields) throws HyracksDataException {
ITreeIndexMetadataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- LinkedMetaDataPageManager freePageManager = new LinkedMetaDataPageManager(harness.getBufferCache(),
- metaFrameFactory);
+ LinkedMetaDataPageManager freePageManager =
+ new LinkedMetaDataPageManager(harness.getBufferCache(), metaFrameFactory);
return BTreeUtils.createBTree(harness.getBufferCache(), harness.getFileMapProvider(), typeTraits, cmpFactories,
BTreeLeafFrameType.REGULAR_NSM, harness.getFileReference(), freePageManager);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java
index dcf7f09..fdc5e95 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java
@@ -32,7 +32,6 @@
import org.apache.hyracks.storage.am.common.TestOperationSelector.TestOperation;
import org.apache.hyracks.storage.am.common.TestWorkloadConf;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
-import org.apache.hyracks.storage.am.common.api.TreeIndexException;
import org.apache.hyracks.storage.am.common.datagen.ProbabilityHelper;
public class BTreeMultiThreadTest extends OrderedIndexMultiThreadTest {
@@ -52,10 +51,10 @@
@Override
protected ITreeIndex createIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
- int[] bloomFilterKeyFields) throws TreeIndexException {
+ int[] bloomFilterKeyFields) throws HyracksDataException {
return BTreeUtils.createBTree(harness.getBufferCache(), harness.getFileMapProvider(), typeTraits, cmpFactories,
- BTreeLeafFrameType.REGULAR_NSM, harness.getFileReference(), harness.getPageManagerFactory()
- .createPageManager(harness.getBufferCache()));
+ BTreeLeafFrameType.REGULAR_NSM, harness.getFileReference(),
+ harness.getPageManagerFactory().createPageManager(harness.getBufferCache()));
}
@Override
@@ -69,20 +68,20 @@
// Insert only workload.
TestOperation[] insertOnlyOps = new TestOperation[] { TestOperation.INSERT };
- workloadConfs.add(new TestWorkloadConf(insertOnlyOps, ProbabilityHelper
- .getUniformProbDist(insertOnlyOps.length)));
+ workloadConfs
+ .add(new TestWorkloadConf(insertOnlyOps, ProbabilityHelper.getUniformProbDist(insertOnlyOps.length)));
// Inserts mixed with point searches and scans.
TestOperation[] insertSearchOnlyOps = new TestOperation[] { TestOperation.INSERT, TestOperation.POINT_SEARCH,
TestOperation.SCAN, TestOperation.DISKORDER_SCAN };
- workloadConfs.add(new TestWorkloadConf(insertSearchOnlyOps, ProbabilityHelper
- .getUniformProbDist(insertSearchOnlyOps.length)));
+ workloadConfs.add(new TestWorkloadConf(insertSearchOnlyOps,
+ ProbabilityHelper.getUniformProbDist(insertSearchOnlyOps.length)));
// Inserts, updates, deletes, and upserts.
TestOperation[] insertDeleteUpdateUpsertOps = new TestOperation[] { TestOperation.INSERT, TestOperation.DELETE,
TestOperation.UPDATE, TestOperation.UPSERT };
- workloadConfs.add(new TestWorkloadConf(insertDeleteUpdateUpsertOps, ProbabilityHelper
- .getUniformProbDist(insertDeleteUpdateUpsertOps.length)));
+ workloadConfs.add(new TestWorkloadConf(insertDeleteUpdateUpsertOps,
+ ProbabilityHelper.getUniformProbDist(insertDeleteUpdateUpsertOps.length)));
// All operations mixed.
TestOperation[] allOps = new TestOperation[] { TestOperation.INSERT, TestOperation.DELETE,
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyTreeFactory.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyTreeFactory.java
index e1810e3..dafd6b9 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyTreeFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyTreeFactory.java
@@ -21,7 +21,6 @@
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
-import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
public class DummyTreeFactory extends TreeIndexFactory<ITreeIndex> {
@@ -31,7 +30,7 @@
}
@Override
- public ITreeIndex createIndexInstance(FileReference file) throws IndexException {
+ public ITreeIndex createIndexInstance(FileReference file) {
return null;
}