ASTERIXDB-1058: Lazy LSM memory components allocation
Change-Id: I476e756f8d71260ea614c8c072fc9503053866c9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/405
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <imaxon@apache.org>
Reviewed-by: Young-Seok Kim <kisskys@gmail.com>
diff --git a/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/RuntimeContext.java b/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/RuntimeContext.java
index 6fde446..b1216d5 100644
--- a/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/RuntimeContext.java
+++ b/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/RuntimeContext.java
@@ -63,7 +63,7 @@
ILocalResourceRepositoryFactory localResourceRepositoryFactory = new TransientLocalResourceRepositoryFactory();
localResourceRepository = localResourceRepositoryFactory.createRepository();
resourceIdFactory = (new ResourceIdFactoryProvider(localResourceRepository)).createResourceIdFactory();
- lcManager = new IndexLifecycleManager(localResourceRepository);
+ lcManager = new IndexLifecycleManager();
}
public void close() throws HyracksDataException {
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexLifecycleManager.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexLifecycleManager.java
index 41eb5d5..05395a8 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexLifecycleManager.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexLifecycleManager.java
@@ -34,6 +34,4 @@
public IIndex getIndex(String resourceName) throws HyracksDataException;
public void unregister(String resourceName) throws HyracksDataException;
-
- public IIndex getIndex(int datasetID, long resourceID) throws HyracksDataException;
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java
index 00ae8c2..c884cbb 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java
@@ -25,29 +25,26 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.lifecycle.ILifeCycleComponent;
import org.apache.hyracks.storage.am.common.api.IIndex;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManager;
-import org.apache.hyracks.storage.common.file.ILocalResourceRepository;
-import org.apache.hyracks.storage.common.file.LocalResource;
public class IndexLifecycleManager implements IIndexLifecycleManager, ILifeCycleComponent {
private static final long DEFAULT_MEMORY_BUDGET = 1024 * 1024 * 100; // 100 megabytes
- private final Map<Long, IndexInfo> indexInfos;
+ private final Map<String, IndexInfo> indexInfos;
private final long memoryBudget;
- private final ILocalResourceRepository localResourcesRepository;
private long memoryUsed;
- public IndexLifecycleManager(ILocalResourceRepository localResourcesRepository) {
- this(localResourcesRepository, DEFAULT_MEMORY_BUDGET);
+ public IndexLifecycleManager() {
+ this(DEFAULT_MEMORY_BUDGET);
}
- public IndexLifecycleManager(ILocalResourceRepository localResourcesRepository, long memoryBudget) {
- this.localResourcesRepository = localResourcesRepository;
- this.indexInfos = new HashMap<Long, IndexInfo>();
+ public IndexLifecycleManager(long memoryBudget) {
+ this.indexInfos = new HashMap<>();
this.memoryBudget = memoryBudget;
this.memoryUsed = 0;
}
@@ -64,9 +61,14 @@
}
info.index.deactivate();
- memoryUsed -= info.index.getMemoryAllocationSize();
+ //find resource name and deallocate its memory
+ for (Entry<String, IndexInfo> entry : indexInfos.entrySet()) {
+ if (entry.getValue() == info) {
+ deallocateMemory(entry.getKey());
+ break;
+ }
+ }
info.isOpen = false;
-
return true;
}
@@ -75,12 +77,14 @@
private int referenceCount;
private long lastAccess;
private boolean isOpen;
+ private boolean memoryAllocated;
public IndexInfo(IIndex index) {
this.index = index;
this.lastAccess = -1;
this.referenceCount = 0;
this.isOpen = false;
+ this.memoryAllocated = false;
}
public void touch() {
@@ -125,7 +129,6 @@
}
}
}
-
}
public String toString() {
@@ -170,9 +173,9 @@
String headerFormat = "%-20s %-10s %-20s %-20s %-20s\n";
String rowFormat = "%-20d %-10b %-20d %-20s %-20s\n";
- sb.append(String.format(headerFormat, "ResourceID", "Open", "Reference Count", "Last Access", "Index Name"));
+ sb.append(String.format(headerFormat, "ResourceName", "Open", "Reference Count", "Last Access", "Index Name"));
IndexInfo ii;
- for (Map.Entry<Long, IndexInfo> entry : indexInfos.entrySet()) {
+ for (Map.Entry<String, IndexInfo> entry : indexInfos.entrySet()) {
ii = entry.getValue();
sb.append(String.format(rowFormat, entry.getKey(), ii.isOpen, ii.referenceCount, ii.lastAccess, ii.index));
}
@@ -181,76 +184,86 @@
@Override
public void register(String resourceName, IIndex index) throws HyracksDataException {
- long resourceID = getResourceID(resourceName);
- if (indexInfos.containsKey(resourceID)) {
- throw new HyracksDataException("Index with resource ID " + resourceID + " already exists.");
+ if (indexInfos.containsKey(resourceName)) {
+ throw new HyracksDataException("Index with resource name " + resourceName + " already exists.");
}
-
- indexInfos.put(resourceID, new IndexInfo(index));
+ indexInfos.put(resourceName, new IndexInfo(index));
}
@Override
public void open(String resourceName) throws HyracksDataException {
- long resourceID = getResourceID(resourceName);
- IndexInfo info = indexInfos.get(resourceID);
+ IndexInfo info = indexInfos.get(resourceName);
if (info == null) {
- throw new HyracksDataException("Failed to open index with resource ID " + resourceID
+ throw new HyracksDataException("Failed to open index with resource name " + resourceName
+ " since it does not exist.");
}
if (!info.isOpen) {
- long inMemorySize = info.index.getMemoryAllocationSize();
- while (memoryUsed + inMemorySize > memoryBudget) {
- if (!evictCandidateIndex()) {
- throw new HyracksDataException("Cannot activate index since memory budget would be exceeded.");
- }
- }
+ allocateMemory(resourceName);
info.index.activate();
info.isOpen = true;
- memoryUsed += inMemorySize;
}
info.touch();
}
@Override
public void close(String resourceName) throws HyracksDataException {
- long resourceID = getResourceID(resourceName);
- indexInfos.get(resourceID).untouch();
+ indexInfos.get(resourceName).untouch();
}
@Override
public IIndex getIndex(String resourceName) throws HyracksDataException {
- long resourceID = getResourceID(resourceName);
- IndexInfo info = indexInfos.get(resourceID);
+ IndexInfo info = indexInfos.get(resourceName);
return info == null ? null : info.index;
}
@Override
public void unregister(String resourceName) throws HyracksDataException {
- long resourceID = getResourceID(resourceName);
- IndexInfo info = indexInfos.remove(resourceID);
+ IndexInfo info = indexInfos.get(resourceName);
if (info == null) {
- throw new HyracksDataException("Index with resource ID " + resourceID + " does not exist.");
+ throw new HyracksDataException("Index with resource name " + resourceName + " does not exist.");
}
if (info.referenceCount != 0) {
- indexInfos.put(resourceID, info);
+ indexInfos.put(resourceName, info);
throw new HyracksDataException("Cannot remove index while it is open.");
}
if (info.isOpen) {
info.index.deactivate();
- memoryUsed -= info.index.getMemoryAllocationSize();
+ deallocateMemory(resourceName);
+ }
+ indexInfos.remove(resourceName);
+ }
+
+ private void allocateMemory(String resourceName) throws HyracksDataException {
+ IndexInfo info = indexInfos.get(resourceName);
+ if (info == null) {
+ throw new HyracksDataException("Failed to allocate memory for index with resource ID " + resourceName
+ + " since it does not exist.");
+ }
+ if (!info.memoryAllocated) {
+ long inMemorySize = info.index.getMemoryAllocationSize();
+ while (memoryUsed + inMemorySize > memoryBudget) {
+ if (!evictCandidateIndex()) {
+ throw new HyracksDataException(
+ "Cannot allocate memory for index since memory budget would be exceeded.");
+ }
+ }
+ memoryUsed += inMemorySize;
+ info.memoryAllocated = true;
}
}
- private long getResourceID(String resourceName) throws HyracksDataException {
- LocalResource lr = localResourcesRepository.getResourceByName(resourceName);
- return lr == null ? -1 : lr.getResourceId();
- }
-
- @Override
- public IIndex getIndex(int datasetID, long resourceID) throws HyracksDataException {
- throw new UnsupportedOperationException();
+ private void deallocateMemory(String resourceName) throws HyracksDataException {
+ IndexInfo info = indexInfos.get(resourceName);
+ if (info == null) {
+ throw new HyracksDataException("Failed to deallocate memory for index with resource name " + resourceName
+ + " since it does not exist.");
+ }
+ if (info.isOpen && info.memoryAllocated) {
+ memoryUsed -= info.index.getMemoryAllocationSize();
+ info.memoryAllocated = false;
+ }
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index 343f690..bf0040a 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -898,4 +898,9 @@
return files;
}
+
+ @Override
+ public void allocateMemoryComponents() throws HyracksDataException {
+ //do nothing since external index never use memory components
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 663d2f7..822b320 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -113,6 +113,9 @@
super(virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
filterManager, filterFields, durable);
+ this.insertLeafFrameFactory = insertLeafFrameFactory;
+ this.deleteLeafFrameFactory = deleteLeafFrameFactory;
+ this.cmpFactories = cmpFactories;
int i = 0;
for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
LSMBTreeMemoryComponent mutableComponent = new LSMBTreeMemoryComponent(new BTree(virtualBufferCache,
@@ -124,10 +127,6 @@
memoryComponents.add(mutableComponent);
++i;
}
-
- this.insertLeafFrameFactory = insertLeafFrameFactory;
- this.deleteLeafFrameFactory = deleteLeafFrameFactory;
- this.cmpFactories = cmpFactories;
componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, filterFactory);
bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory,
filterFactory);
@@ -170,12 +169,6 @@
if (isActivated) {
throw new HyracksDataException("Failed to activate the index since it is already activated.");
}
- for (ILSMComponent c : memoryComponents) {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).open();
- mutableComponent.getBTree().create();
- mutableComponent.getBTree().activate();
- }
List<ILSMComponent> immutableComponents = diskComponents;
immutableComponents.clear();
List<LSMComponentFileReferences> validFileReferences;
@@ -223,12 +216,7 @@
btree.deactivate();
bloomFilter.deactivate();
}
- for (ILSMComponent c : memoryComponents) {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- mutableComponent.getBTree().deactivate();
- mutableComponent.getBTree().destroy();
- ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).close();
- }
+ deallocateMemoryComponents();
isActivated = false;
}
@@ -262,12 +250,8 @@
throw new HyracksDataException("Failed to clear the index since it is not activated.");
}
+ clearMemoryComponents();
List<ILSMComponent> immutableComponents = diskComponents;
- for (ILSMComponent c : memoryComponents) {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- mutableComponent.getBTree().clear();
- mutableComponent.reset();
- }
for (ILSMComponent c : immutableComponents) {
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
component.getBloomFilter().deactivate();
@@ -284,7 +268,6 @@
List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
int cmc = currentMutableComponentId.get();
ctx.setCurrentMutableComponentId(cmc);
- int numMutableComponents = memoryComponents.size();
operationalComponents.clear();
switch (ctx.getOperation()) {
case UPDATE:
@@ -295,29 +278,13 @@
operationalComponents.add(memoryComponents.get(cmc));
break;
case INSERT:
- for (int i = 0; i < numMutableComponents - 1; i++) {
- ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- if (mutableComponent.isReadable()) {
- // Make sure newest components are added first
- operationalComponents.add(0, mutableComponent);
- }
- }
- // The current mutable component is always added
- operationalComponents.add(0, memoryComponents.get(cmc));
+ addOperationalMutableComponents(operationalComponents);
operationalComponents.addAll(immutableComponents);
break;
case SEARCH:
- for (int i = 0; i < numMutableComponents - 1; i++) {
- ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- if (mutableComponent.isReadable()) {
- // Make sure newest components are added first
- operationalComponents.add(0, mutableComponent);
- }
+ if (memoryComponentsAllocated) {
+ addOperationalMutableComponents(operationalComponents);
}
- // The current mutable component is always added
- operationalComponents.add(0, memoryComponents.get(cmc));
if (filterManager != null) {
for (ILSMComponent c : immutableComponents) {
if (c.getLSMComponentFilter().satisfy(
@@ -834,10 +801,7 @@
@Override
public void validate() throws HyracksDataException {
- for (ILSMComponent c : memoryComponents) {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- mutableComponent.getBTree().validate();
- }
+ validateMemoryComponents();
List<ILSMComponent> immutableComponents = diskComponents;
for (ILSMComponent c : immutableComponents) {
BTree btree = ((LSMBTreeDiskComponent) c).getBTree();
@@ -857,13 +821,75 @@
@Override
public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
-
Set<String> files = new HashSet<String>();
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) lsmComponent;
files.add(component.getBTree().getFileReference().toString());
files.add(component.getBloomFilter().getFileReference().toString());
-
+
return files;
}
-}
+
+ @Override
+ public synchronized void allocateMemoryComponents() throws HyracksDataException {
+ if (!isActivated) {
+ throw new HyracksDataException("Failed to allocate memory components since the index is not active");
+ }
+ if (memoryComponentsAllocated) {
+ return;
+ }
+ for (ILSMComponent c : memoryComponents) {
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+ ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).open();
+ mutableComponent.getBTree().create();
+ mutableComponent.getBTree().activate();
+ }
+ memoryComponentsAllocated = true;
+ }
+
+ private void addOperationalMutableComponents(List<ILSMComponent> operationalComponents) {
+ int cmc = currentMutableComponentId.get();
+ int numMutableComponents = memoryComponents.size();
+ for (int i = 0; i < numMutableComponents - 1; i++) {
+ ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+ if (mutableComponent.isReadable()) {
+ // Make sure newest components are added first
+ operationalComponents.add(0, mutableComponent);
+ }
+ }
+ // The current mutable component is always added
+ operationalComponents.add(0, memoryComponents.get(cmc));
+ }
+
+ private synchronized void clearMemoryComponents() throws HyracksDataException {
+ if (memoryComponentsAllocated) {
+ for (ILSMComponent c : memoryComponents) {
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+ mutableComponent.getBTree().clear();
+ mutableComponent.reset();
+ }
+ }
+ }
+
+ private synchronized void validateMemoryComponents() throws HyracksDataException {
+ if (memoryComponentsAllocated) {
+ for (ILSMComponent c : memoryComponents) {
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+ mutableComponent.getBTree().validate();
+ }
+ }
+ }
+
+ private synchronized void deallocateMemoryComponents() throws HyracksDataException {
+ if (memoryComponentsAllocated) {
+ for (ILSMComponent c : memoryComponents) {
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+ mutableComponent.getBTree().deactivate();
+ mutableComponent.getBTree().destroy();
+ ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).close();
+ }
+ memoryComponentsAllocated = false;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
index aa410f3..08be21d 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
@@ -88,4 +88,11 @@
public void scheduleReplication(ILSMIndexOperationContext ctx, List<ILSMComponent> lsmComponents, boolean bulkload,
ReplicationOperation operation, LSMOperationType opType) throws HyracksDataException;
+ /**
+ * Allocates the memory components of an LSM index in the buffer cache.
+ * @throws HyracksDataException
+ */
+ public void allocateMemoryComponents() throws HyracksDataException;
+
+ public boolean isMemoryComponentsAllocated();
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index b3d21be..adfadaa 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -76,6 +76,7 @@
protected boolean isActivated;
protected final AtomicBoolean[] flushRequests;
+ protected boolean memoryComponentsAllocated = false;
public AbstractLSMIndex(List<IVirtualBufferCache> virtualBufferCaches, IBufferCache diskBufferCache,
ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider,
@@ -334,4 +335,10 @@
throw new HyracksDataException(e);
}
}
+
+ public abstract void allocateMemoryComponents() throws HyracksDataException;
+
+ public boolean isMemoryComponentsAllocated() {
+ return memoryComponentsAllocated;
+ }
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index ced47de..54500a6 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -327,8 +327,11 @@
return modify(ctx, tryOperation, tuple, opType);
}
- protected boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple,
+ private boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple,
LSMOperationType opType) throws HyracksDataException, IndexException {
+ if (!lsmIndex.isMemoryComponentsAllocated()) {
+ lsmIndex.allocateMemoryComponents();
+ }
boolean failedOperation = false;
if (!getAndEnterComponents(ctx, opType, tryOperation)) {
return false;
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 28b96db..d7a2611 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -168,15 +168,6 @@
try {
List<ILSMComponent> immutableComponents = diskComponents;
- for (ILSMComponent c : memoryComponents) {
- LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
- ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).open();
- mutableComponent.getInvIndex().create();
- mutableComponent.getInvIndex().activate();
- mutableComponent.getDeletedKeysBTree().create();
- mutableComponent.getDeletedKeysBTree().activate();
- }
-
immutableComponents.clear();
List<LSMComponentFileReferences> validFileReferences = fileManager.cleanupAndGetValidFiles();
for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
@@ -192,7 +183,6 @@
immutableComponents.add(component);
}
isActivated = true;
- // TODO: Maybe we can make activate throw an index exception?
} catch (IndexException e) {
throw new HyracksDataException(e);
}
@@ -204,12 +194,7 @@
throw new HyracksDataException("Failed to clear the index since it is not activated.");
}
- for (ILSMComponent c : memoryComponents) {
- LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
- mutableComponent.getInvIndex().clear();
- mutableComponent.getDeletedKeysBTree().clear();
- mutableComponent.reset();
- }
+ clearMemoryComponents();
List<ILSMComponent> immutableComponents = diskComponents;
for (ILSMComponent c : immutableComponents) {
LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
@@ -228,8 +213,6 @@
if (!isActivated) {
throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
}
-
- isActivated = false;
if (flushOnExit) {
BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
@@ -248,14 +231,8 @@
component.getInvIndex().deactivate();
component.getDeletedKeysBTree().deactivate();
}
- for (ILSMComponent c : memoryComponents) {
- LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
- mutableComponent.getInvIndex().deactivate();
- mutableComponent.getDeletedKeysBTree().deactivate();
- mutableComponent.getInvIndex().destroy();
- mutableComponent.getDeletedKeysBTree().destroy();
- ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).close();
- }
+ deallocateMemoryComponents();
+ isActivated = false;
}
@Override
@@ -290,7 +267,6 @@
List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
int cmc = currentMutableComponentId.get();
ctx.setCurrentMutableComponentId(cmc);
- int numMutableComponents = memoryComponents.size();
operationalComponents.clear();
switch (ctx.getOperation()) {
case FLUSH:
@@ -299,17 +275,9 @@
operationalComponents.add(memoryComponents.get(cmc));
break;
case SEARCH:
- for (int i = 0; i < numMutableComponents - 1; i++) {
- ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
- LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
- if (mutableComponent.isReadable()) {
- // Make sure newest components are added first
- operationalComponents.add(0, mutableComponent);
- }
+ if (memoryComponentsAllocated) {
+ addOperationalMutableComponents(operationalComponents);
}
- // The current mutable component is always added
- operationalComponents.add(0, memoryComponents.get(cmc));
-
if (filterManager != null) {
for (ILSMComponent c : immutableComponents) {
if (c.getLSMComponentFilter().satisfy(
@@ -322,7 +290,6 @@
} else {
operationalComponents.addAll(immutableComponents);
}
-
break;
case MERGE:
operationalComponents.addAll(ctx.getComponentsToBeMerged());
@@ -406,7 +373,6 @@
throws HyracksDataException, IndexException {
List<ILSMComponent> operationalComponents = ictx.getComponentHolder();
int numComponents = operationalComponents.size();
- assert numComponents > 0;
boolean includeMutableComponent = false;
ArrayList<IIndexAccessor> indexAccessors = new ArrayList<IIndexAccessor>(numComponents);
ArrayList<IIndexAccessor> deletedKeysBTreeAccessors = new ArrayList<IIndexAccessor>(numComponents);
@@ -916,11 +882,7 @@
@Override
public void validate() throws HyracksDataException {
- for (ILSMComponent c : memoryComponents) {
- LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
- mutableComponent.getInvIndex().validate();
- mutableComponent.getDeletedKeysBTree().validate();
- }
+ validateMemoryComponents();
List<ILSMComponent> immutableComponents = diskComponents;
for (ILSMComponent c : immutableComponents) {
LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
@@ -958,4 +920,73 @@
return files;
}
-}
+
+ @Override
+ public synchronized void allocateMemoryComponents() throws HyracksDataException {
+ if (!isActivated) {
+ throw new HyracksDataException("Failed to allocate memory components since the index is not active.");
+ }
+ if (memoryComponentsAllocated) {
+ return;
+ }
+ for (ILSMComponent c : memoryComponents) {
+ LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+ ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).open();
+ mutableComponent.getInvIndex().create();
+ mutableComponent.getInvIndex().activate();
+ mutableComponent.getDeletedKeysBTree().create();
+ mutableComponent.getDeletedKeysBTree().activate();
+ }
+ memoryComponentsAllocated = true;
+ }
+
+ private void addOperationalMutableComponents(List<ILSMComponent> operationalComponents) {
+ int cmc = currentMutableComponentId.get();
+ int numMutableComponents = memoryComponents.size();
+ for (int i = 0; i < numMutableComponents - 1; i++) {
+ ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
+ LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+ if (mutableComponent.isReadable()) {
+ // Make sure newest components are added first
+ operationalComponents.add(0, mutableComponent);
+ }
+ }
+ // The current mutable component is always added
+ operationalComponents.add(0, memoryComponents.get(cmc));
+ }
+
+ private synchronized void clearMemoryComponents() throws HyracksDataException {
+ if (memoryComponentsAllocated) {
+ for (ILSMComponent c : memoryComponents) {
+ LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+ mutableComponent.getInvIndex().clear();
+ mutableComponent.getDeletedKeysBTree().clear();
+ mutableComponent.reset();
+ }
+ }
+ }
+
+ private synchronized void validateMemoryComponents() throws HyracksDataException {
+ if (memoryComponentsAllocated) {
+ for (ILSMComponent c : memoryComponents) {
+ LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+ mutableComponent.getInvIndex().validate();
+ mutableComponent.getDeletedKeysBTree().validate();
+ }
+ }
+ }
+
+ private synchronized void deallocateMemoryComponents() throws HyracksDataException {
+ if (memoryComponentsAllocated) {
+ for (ILSMComponent c : memoryComponents) {
+ LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+ mutableComponent.getInvIndex().deactivate();
+ mutableComponent.getDeletedKeysBTree().deactivate();
+ mutableComponent.getInvIndex().destroy();
+ mutableComponent.getDeletedKeysBTree().destroy();
+ ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).close();
+ }
+ memoryComponentsAllocated = false;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index 533dfac..7ec6a6e 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -169,15 +169,6 @@
if (isActivated) {
throw new HyracksDataException("Failed to activate the index since it is already activated.");
}
-
- for (ILSMComponent c : memoryComponents) {
- LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
- ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).open();
- mutableComponent.getRTree().create();
- mutableComponent.getBTree().create();
- mutableComponent.getRTree().activate();
- mutableComponent.getBTree().activate();
- }
}
@Override
@@ -196,15 +187,7 @@
throw new HyracksDataException(e);
}
}
-
- for (ILSMComponent c : memoryComponents) {
- LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
- mutableComponent.getRTree().deactivate();
- mutableComponent.getBTree().deactivate();
- mutableComponent.getRTree().destroy();
- mutableComponent.getBTree().destroy();
- ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).close();
- }
+ deallocateMemoryComponents();
}
@Override
@@ -219,13 +202,7 @@
if (!isActivated) {
throw new HyracksDataException("Failed to clear the index since it is not activated.");
}
-
- for (ILSMComponent c : memoryComponents) {
- LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
- mutableComponent.getRTree().clear();
- mutableComponent.getBTree().clear();
- mutableComponent.reset();
- }
+ clearMemoryComponents();
}
@Override
@@ -234,7 +211,6 @@
List<ILSMComponent> immutableComponents = diskComponents;
int cmc = currentMutableComponentId.get();
ctx.setCurrentMutableComponentId(cmc);
- int numMutableComponents = memoryComponents.size();
operationalComponents.clear();
switch (ctx.getOperation()) {
case INSERT:
@@ -243,17 +219,9 @@
operationalComponents.add(memoryComponents.get(cmc));
break;
case SEARCH:
- for (int i = 0; i < numMutableComponents - 1; i++) {
- ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
- LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
- if (mutableComponent.isReadable()) {
- // Make sure newest components are added first
- operationalComponents.add(0, mutableComponent);
- }
+ if (memoryComponentsAllocated) {
+ addOperationalMutableComponents(operationalComponents);
}
- // The current mutable component is always added
- operationalComponents.add(0, memoryComponents.get(cmc));
-
if (filterManager != null) {
for (ILSMComponent c : immutableComponents) {
if (c.getLSMComponentFilter().satisfy(
@@ -266,7 +234,6 @@
} else {
operationalComponents.addAll(immutableComponents);
}
-
break;
case MERGE:
operationalComponents.addAll(ctx.getComponentsToBeMerged());
@@ -449,4 +416,63 @@
public String toString() {
return "LSMRTree [" + fileManager.getBaseDir() + "]";
}
-}
+
+ @Override
+ public synchronized void allocateMemoryComponents() throws HyracksDataException {
+ if (!isActivated) {
+ throw new HyracksDataException("Failed to allocate memory components since the index is not active.");
+ }
+ if (memoryComponentsAllocated) {
+ return;
+ }
+ for (ILSMComponent c : memoryComponents) {
+ LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+ ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).open();
+ mutableComponent.getRTree().create();
+ mutableComponent.getBTree().create();
+ mutableComponent.getRTree().activate();
+ mutableComponent.getBTree().activate();
+ }
+ memoryComponentsAllocated = true;
+ }
+
+ private void addOperationalMutableComponents(List<ILSMComponent> operationalComponents) {
+ int cmc = currentMutableComponentId.get();
+ int numMutableComponents = memoryComponents.size();
+ for (int i = 0; i < numMutableComponents - 1; i++) {
+ ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
+ LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+ if (mutableComponent.isReadable()) {
+ // Make sure newest components are added first
+ operationalComponents.add(0, mutableComponent);
+ }
+ }
+ // The current mutable component is always added
+ operationalComponents.add(0, memoryComponents.get(cmc));
+ }
+
+ private synchronized void clearMemoryComponents() throws HyracksDataException {
+ if (memoryComponentsAllocated) {
+ for (ILSMComponent c : memoryComponents) {
+ LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+ mutableComponent.getRTree().clear();
+ mutableComponent.getBTree().clear();
+ mutableComponent.reset();
+ }
+ }
+ }
+
+ private synchronized void deallocateMemoryComponents() throws HyracksDataException {
+ if (memoryComponentsAllocated) {
+ for (ILSMComponent c : memoryComponents) {
+ LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+ mutableComponent.getRTree().deactivate();
+ mutableComponent.getBTree().deactivate();
+ mutableComponent.getRTree().destroy();
+ mutableComponent.getBTree().destroy();
+ ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).close();
+ }
+ memoryComponentsAllocated = false;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java b/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java
index 671c479..d987501 100644
--- a/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java
+++ b/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java
@@ -75,7 +75,7 @@
public synchronized static IIndexLifecycleManager getIndexLifecycleManager(IHyracksTaskContext ctx) {
if (lcManager == null) {
- lcManager = new IndexLifecycleManager(getLocalResourceRepository(ctx));
+ lcManager = new IndexLifecycleManager();
}
return lcManager;
}