added datasetlifecyclemanager to properly account for memory throughout index lifecycle; adjusted default memory budget to be slightly lower
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
index baceb57..c9dedfb 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
@@ -15,6 +15,7 @@
import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixFileMapManager;
import edu.uci.ics.asterix.common.context.ConstantMergePolicy;
+import edu.uci.ics.asterix.common.context.DatasetLifecycleManager;
import edu.uci.ics.asterix.common.exceptions.ACIDException;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.common.transactions.IAsterixAppRuntimeContextProvider;
@@ -31,14 +32,11 @@
import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.storage.am.common.api.IIndex;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManager;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexLifecycleManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.MultitenantVirtualBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.SynchronousScheduler;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.BufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ClockPageReplacementStrategy;
import edu.uci.ics.hyracks.storage.common.buffercache.DelayPageCleanerPolicy;
@@ -63,7 +61,7 @@
private AsterixStorageProperties storageProperties;
private AsterixTransactionProperties txnProperties;
- private IIndexLifecycleManager indexLifecycleManager;
+ private DatasetLifecycleManager indexLifecycleManager;
private Map<Integer, IVirtualBufferCache> datasetVirtualBufferCaches;
private IFileMapManager fileMapManager;
private IBufferCache bufferCache;
@@ -104,8 +102,6 @@
storageProperties.getBufferCachePageSize(), storageProperties.getBufferCacheNumPages(),
storageProperties.getBufferCacheMaxOpenFiles());
- indexLifecycleManager = new IndexLifecycleManager(storageProperties.getMemoryComponentGlobalBudget());
-
lsmIOScheduler = SynchronousScheduler.INSTANCE;
mergePolicy = new ConstantMergePolicy(storageProperties.getLSMIndexMergeThreshold(), this);
lsmBTreeOpTrackerFactory = new IndexOperationTrackerFactory(LSMBTreeIOOperationCallbackFactory.INSTANCE);
@@ -118,7 +114,7 @@
localResourceRepository = (PersistentLocalResourceRepository) persistentLocalResourceRepositoryFactory
.createRepository();
resourceIdFactory = (new ResourceIdFactoryProvider(localResourceRepository)).createResourceIdFactory();
-
+ indexLifecycleManager = new DatasetLifecycleManager(storageProperties, localResourceRepository);
IAsterixAppRuntimeContextProvider asterixAppRuntimeContextProvider = new AsterixAppRuntimeContextProviderForRecovery(
this);
txnSubsystem = new TransactionSubsystem(ncApplicationContext.getNodeId(), asterixAppRuntimeContextProvider);
@@ -217,15 +213,8 @@
return externalProperties;
}
+ @Override
public IVirtualBufferCache getVirtualBufferCache(int datasetID) {
- synchronized (datasetVirtualBufferCaches) {
- IVirtualBufferCache vbc = datasetVirtualBufferCaches.get(datasetID);
- if (vbc == null) {
- vbc = new MultitenantVirtualBufferCache(new VirtualBufferCache(new HeapBufferAllocator(),
- storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()));
- datasetVirtualBufferCaches.put(datasetID, vbc);
- }
- return vbc;
- }
+ return indexLifecycleManager.getVirtualBufferCache(datasetID);
}
}
\ No newline at end of file
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
index 8c73a63..c3dae04 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
@@ -8,6 +8,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
import edu.uci.ics.hyracks.storage.common.file.ILocalResourceRepository;
@@ -87,6 +88,17 @@
}
@Override
+ public IVirtualBufferCache getVirtualBufferCache(int datasetID) {
+ return asterixAppRuntimeContext.getVirtualBufferCache(datasetID);
+ }
+
+ @Override
+ public ILSMIOOperationCallbackProvider getNoOpIOOperationCallbackProvider() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
public ILSMIOOperationCallbackProvider getLSMBTreeIOOperationCallbackProvider() {
// TODO Auto-generated method stub
return null;
@@ -103,11 +115,4 @@
// TODO Auto-generated method stub
return null;
}
-
- @Override
- public ILSMIOOperationCallbackProvider getNoOpIOOperationCallbackProvider() {
- // TODO Auto-generated method stub
- return null;
- }
-
}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
index 33aaca6..7344b42 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
@@ -22,6 +22,7 @@
import java.util.logging.Logger;
import edu.uci.ics.asterix.api.common.Job;
+import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
import edu.uci.ics.asterix.common.config.GlobalConfig;
@@ -41,7 +42,6 @@
import edu.uci.ics.asterix.om.types.ARecordType;
import edu.uci.ics.asterix.om.types.IAType;
import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
-import edu.uci.ics.asterix.transaction.management.resource.ILocalResourceMetadata;
import edu.uci.ics.asterix.transaction.management.resource.LSMBTreeLocalResourceMetadata;
import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
import edu.uci.ics.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
@@ -170,8 +170,7 @@
AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(typeTraits,
- comparatorFactories, blooFilterKeyFields, true, storageProperties.getMemoryComponentPageSize(),
- storageProperties.getMemoryComponentNumPages());
+ comparatorFactories, blooFilterKeyFields, dataset.getDatasetId());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMBTreeResource);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java
index 2deebad..a3442a5 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java
@@ -1,10 +1,10 @@
package edu.uci.ics.asterix.file;
+import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixVirtualBufferCacheProvider;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
-import edu.uci.ics.asterix.transaction.management.resource.ILocalResourceMetadata;
import edu.uci.ics.asterix.transaction.management.resource.LSMBTreeLocalResourceMetadata;
import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
import edu.uci.ics.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
@@ -40,8 +40,8 @@
AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(
- secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields, false,
- storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
+ secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields,
+ dataset.getDatasetId());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMBTreeResource);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
index a43e300..cf0e25f 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
@@ -2,6 +2,7 @@
import java.util.List;
+import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
@@ -12,7 +13,6 @@
import edu.uci.ics.asterix.om.types.IAType;
import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
import edu.uci.ics.asterix.runtime.formats.FormatUtils;
-import edu.uci.ics.asterix.transaction.management.resource.ILocalResourceMetadata;
import edu.uci.ics.asterix.transaction.management.resource.LSMInvertedIndexLocalResourceMetadata;
import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
import edu.uci.ics.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
@@ -153,12 +153,10 @@
public JobSpecification buildCreationJobSpec() throws AsterixException, AlgebricksException {
JobSpecification spec = new JobSpecification();
- AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMInvertedIndexLocalResourceMetadata(invListsTypeTraits,
- primaryComparatorFactories, tokenTypeTraits, tokenComparatorFactories, tokenizerFactory,
- storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages(),
- isPartitioned);
+ primaryComparatorFactories, tokenTypeTraits, tokenComparatorFactories, tokenizerFactory, isPartitioned,
+ dataset.getDatasetId());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMInvertedIndexResource);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
index cc6240e..b7350ca 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
@@ -2,6 +2,7 @@
import java.util.List;
+import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixVirtualBufferCacheProvider;
@@ -15,7 +16,6 @@
import edu.uci.ics.asterix.om.types.ATypeTag;
import edu.uci.ics.asterix.om.types.IAType;
import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
-import edu.uci.ics.asterix.transaction.management.resource.ILocalResourceMetadata;
import edu.uci.ics.asterix.transaction.management.resource.LSMRTreeLocalResourceMetadata;
import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
import edu.uci.ics.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
@@ -65,8 +65,7 @@
ILocalResourceMetadata localResourceMetadata = new LSMRTreeLocalResourceMetadata(
secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, primaryComparatorFactories,
valueProviderFactories, RTreePolicyType.RTREE, AqlMetadataProvider.proposeLinearizer(keyType,
- secondaryComparatorFactories.length), storageProperties.getMemoryComponentPageSize(),
- storageProperties.getMemoryComponentNumPages());
+ secondaryComparatorFactories.length), dataset.getDatasetId());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMRTreeResource);
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/ILocalResourceMetadata.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/ILocalResourceMetadata.java
similarity index 86%
rename from asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/ILocalResourceMetadata.java
rename to asterix-common/src/main/java/edu/uci/ics/asterix/common/api/ILocalResourceMetadata.java
index 3b32dc2..9fdfe5f 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/ILocalResourceMetadata.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/ILocalResourceMetadata.java
@@ -1,4 +1,4 @@
-package edu.uci.ics.asterix.transaction.management.resource;
+package edu.uci.ics.asterix.common.api;
import java.io.Serializable;
@@ -10,5 +10,6 @@
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
int partition) throws HyracksDataException;
-
+
+ public int getDatasetID();
}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixStorageProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixStorageProperties.java
index 9e9ef44..7fa552e 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixStorageProperties.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixStorageProperties.java
@@ -15,10 +15,10 @@
private static final int STORAGE_MEMORYCOMPONENT_PAGESIZE_DEFAULT = (32 << 10); // 32KB
private static final String STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY = "storage.memorycomponent.numpages";
- private static final int STORAGE_MEMORYCOMPONENT_NUMPAGES_DEFAULT = 2048; // ... so 64MB components
+ private static final int STORAGE_MEMORYCOMPONENT_NUMPAGES_DEFAULT = 1024; // ... so 32MB components
private static final String STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY = "storage.memorycomponent.globalbudget";
- private static final long STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT = 1073741824; // 1GB
+ private static final long STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT = 536870192; // 512MB
private static final String STORAGE_LSM_MERGETHRESHOLD_KEY = "storage.lsm.mergethreshold";
private static int STORAGE_LSM_MERGETHRESHOLD_DEFAULT = 3;
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/DatasetLifecycleManager.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/DatasetLifecycleManager.java
new file mode 100644
index 0000000..975f320
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/DatasetLifecycleManager.java
@@ -0,0 +1,289 @@
+package edu.uci.ics.asterix.common.context;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.MultitenantVirtualBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
+import edu.uci.ics.hyracks.storage.common.file.ILocalResourceRepository;
+import edu.uci.ics.hyracks.storage.common.file.LocalResource;
+
+public class DatasetLifecycleManager implements IIndexLifecycleManager {
+ private final AsterixStorageProperties storageProperties;
+ private final Map<Integer, MultitenantVirtualBufferCache> datasetVirtualBufferCaches;
+ private final Map<Integer, DatasetInfo> datasetInfos;
+ private final ILocalResourceRepository resourceRepository;
+ private final long capacity;
+ private long used;
+
+ public DatasetLifecycleManager(AsterixStorageProperties storageProperties,
+ ILocalResourceRepository resourceRepository) {
+ this.storageProperties = storageProperties;
+ this.resourceRepository = resourceRepository;
+ datasetVirtualBufferCaches = new HashMap<Integer, MultitenantVirtualBufferCache>();
+ datasetInfos = new HashMap<Integer, DatasetInfo>();
+ capacity = storageProperties.getMemoryComponentGlobalBudget();
+ used = 0;
+ }
+
+ @Override
+ public synchronized IIndex getIndex(long resourceID) throws HyracksDataException {
+ DatasetInfo dsInfo = datasetInfos.get(getDIDfromRID(resourceID));
+ if (dsInfo == null) {
+ return null;
+ }
+ IndexInfo iInfo = dsInfo.indexes.get(resourceID);
+ if (iInfo == null) {
+ return null;
+ }
+ return iInfo.index;
+ }
+
+ @Override
+ public synchronized void register(long resourceID, IIndex index) throws HyracksDataException {
+ int did = getDIDfromRID(resourceID);
+ DatasetInfo dsInfo = datasetInfos.get(did);
+ if (dsInfo == null) {
+ dsInfo = new DatasetInfo(did);
+ } else if (dsInfo.indexes.containsKey(resourceID)) {
+ throw new HyracksDataException("Index with resource ID " + resourceID + " already exists.");
+ }
+ datasetInfos.put(did, dsInfo);
+ dsInfo.indexes.put(resourceID, new IndexInfo(index));
+ }
+
+ private int getDIDfromRID(long resourceID) throws HyracksDataException {
+ LocalResource lr = resourceRepository.getResourceById(resourceID);
+ if (lr == null) {
+ return -1;
+ }
+ return ((ILocalResourceMetadata) lr.getResourceObject()).getDatasetID();
+ }
+
+ @Override
+ public synchronized void unregister(long resourceID) throws HyracksDataException {
+ int did = getDIDfromRID(resourceID);
+ DatasetInfo dsInfo = datasetInfos.get(did);
+ IndexInfo iInfo = dsInfo.indexes.remove(resourceID);
+ if (dsInfo == null || iInfo == null) {
+ throw new HyracksDataException("Index with resource ID " + resourceID + " does not exist.");
+ }
+
+ if (iInfo.referenceCount != 0) {
+ dsInfo.indexes.put(resourceID, iInfo);
+ throw new HyracksDataException("Cannot remove index while it is open.");
+ }
+
+ if (iInfo.isOpen) {
+ iInfo.index.deactivate();
+ }
+
+ if (dsInfo.referenceCount == 0) {
+ if (dsInfo.isOpen) {
+ for (IndexInfo i : dsInfo.indexes.values()) {
+ i.index.deactivate();
+ }
+ }
+ datasetInfos.remove(did);
+ IVirtualBufferCache vbc = getVirtualBufferCache(did);
+ assert vbc != null;
+ used -= (vbc.getNumPages() * vbc.getPageSize());
+ }
+ }
+
+ @Override
+ public synchronized void open(long resourceID) throws HyracksDataException {
+ int did = getDIDfromRID(resourceID);
+ DatasetInfo dsInfo = datasetInfos.get(did);
+ if (dsInfo == null) {
+ throw new HyracksDataException("Failed to open index with resource ID " + resourceID
+ + " since it does not exist.");
+ }
+
+ IndexInfo iInfo = dsInfo.indexes.get(resourceID);
+ if (iInfo == null) {
+ throw new HyracksDataException("Failed to open index with resource ID " + resourceID
+ + " since it does not exist.");
+ }
+
+ if (!dsInfo.isOpen) {
+ IVirtualBufferCache vbc = getVirtualBufferCache(did);
+ assert vbc != null;
+ long additionalSize = vbc.getNumPages() * vbc.getPageSize();
+ while (used + additionalSize > capacity) {
+ if (!evictCandidateDataset()) {
+ throw new HyracksDataException("Cannot activate index since memory budget would be exceeded.");
+ }
+ }
+ used += additionalSize;
+ }
+
+ dsInfo.isOpen = true;
+ dsInfo.touch();
+ if (!iInfo.isOpen) {
+ iInfo.index.activate();
+ iInfo.isOpen = true;
+ }
+ iInfo.touch();
+ }
+
+ private boolean evictCandidateDataset() throws HyracksDataException {
+ // Why min()? As a heuristic for eviction, we will take an open index (an index consuming memory)
+ // that is not being used (refcount == 0) and has been least recently used. The sort order defined
+ // for IndexInfo maintains this. See IndexInfo.compareTo().
+ DatasetInfo dsInfo = Collections.min(datasetInfos.values());
+ if (dsInfo.referenceCount == 0 && dsInfo.isOpen) {
+ for (IndexInfo iInfo : dsInfo.indexes.values()) {
+ if (iInfo.isOpen) {
+ iInfo.index.deactivate();
+ iInfo.isOpen = false;
+ }
+ assert iInfo.referenceCount == 0;
+ }
+
+ IVirtualBufferCache vbc = getVirtualBufferCache(dsInfo.datasetID);
+ used -= vbc.getNumPages() * vbc.getPageSize();
+ dsInfo.isOpen = false;
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public synchronized void close(long resourceID) throws HyracksDataException {
+ int did = getDIDfromRID(resourceID);
+ DatasetInfo dsInfo = datasetInfos.get(did);
+ if (dsInfo == null) {
+ throw new HyracksDataException("No index found with resourceID " + resourceID);
+ }
+ IndexInfo iInfo = dsInfo.indexes.get(resourceID);
+ if (iInfo == null) {
+ throw new HyracksDataException("No index found with resourceID " + resourceID);
+ }
+ iInfo.untouch();
+ dsInfo.untouch();
+ }
+
+ @Override
+ public synchronized List<IIndex> getOpenIndexes() {
+ List<IIndex> openIndexes = new ArrayList<IIndex>();
+ for (DatasetInfo dsInfo : datasetInfos.values()) {
+ for (IndexInfo iInfo : dsInfo.indexes.values()) {
+ if (iInfo.isOpen) {
+ openIndexes.add(iInfo.index);
+ }
+ }
+ }
+ return openIndexes;
+ }
+
+ public IVirtualBufferCache getVirtualBufferCache(int datasetID) {
+ synchronized (datasetVirtualBufferCaches) {
+ MultitenantVirtualBufferCache vbc = datasetVirtualBufferCaches.get(datasetID);
+ if (vbc == null) {
+ vbc = new MultitenantVirtualBufferCache(new VirtualBufferCache(new HeapBufferAllocator(),
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()));
+ datasetVirtualBufferCaches.put(datasetID, vbc);
+ }
+ return vbc;
+ }
+ }
+
+ private static abstract class Info {
+ protected int referenceCount;
+ protected boolean isOpen;
+
+ public Info() {
+ referenceCount = 0;
+ isOpen = false;
+ }
+
+ public void touch() {
+ ++referenceCount;
+ }
+
+ public void untouch() {
+ --referenceCount;
+ }
+ }
+
+ private static class IndexInfo extends Info {
+ private IIndex index;
+
+ public IndexInfo(IIndex index) {
+ this.index = index;
+ }
+ }
+
+ private static class DatasetInfo extends Info implements Comparable<DatasetInfo> {
+ private final Map<Long, IndexInfo> indexes;
+ private final int datasetID;
+ private long lastAccess;
+
+ public DatasetInfo(int datasetID) {
+ this.indexes = new HashMap<Long, IndexInfo>();
+ this.lastAccess = -1;
+ this.datasetID = datasetID;
+ }
+
+ public void touch() {
+ super.touch();
+ lastAccess = System.currentTimeMillis();
+ }
+
+ public void untouch() {
+ super.untouch();
+ lastAccess = System.currentTimeMillis();
+ }
+
+ @Override
+ public int compareTo(DatasetInfo i) {
+ // sort by (isOpen, referenceCount, lastAccess) ascending, where true < false
+ //
+ // Example sort order:
+ // -------------------
+ // (F, 0, 70) <-- largest
+ // (F, 0, 60)
+ // (T, 10, 80)
+ // (T, 10, 70)
+ // (T, 9, 90)
+ // (T, 0, 100) <-- smallest
+ if (isOpen && !i.isOpen) {
+ return -1;
+ } else if (!isOpen && i.isOpen) {
+ return 1;
+ } else {
+ if (referenceCount < i.referenceCount) {
+ return -1;
+ } else if (referenceCount > i.referenceCount) {
+ return 1;
+ } else {
+ if (lastAccess < i.lastAccess) {
+ return -1;
+ } else if (lastAccess > i.lastAccess) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+ }
+
+ }
+
+ public String toString() {
+ return "DatasetID: " + datasetID + ", isOpen: " + isOpen + ", refCount: " + referenceCount
+ + ", lastAccess: " + lastAccess + "}";
+ }
+ }
+
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
index f762b16..1b91b0c 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
@@ -6,6 +6,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
import edu.uci.ics.hyracks.storage.common.file.ILocalResourceRepository;
@@ -46,4 +47,6 @@
public ResourceIdFactory getResourceIdFactory();
public IIOManager getIOManager();
+
+ public IVirtualBufferCache getVirtualBufferCache(int datasetID);
}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/transactions/ILoggerRepository.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/transactions/ILoggerRepository.java
index 036d66c..323b9ce 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/transactions/ILoggerRepository.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/transactions/ILoggerRepository.java
@@ -1,6 +1,8 @@
package edu.uci.ics.asterix.common.transactions;
+import edu.uci.ics.asterix.common.exceptions.ACIDException;
+
public interface ILoggerRepository {
- public ILogger getIndexLogger(long resourceId, byte resourceType);
+ public ILogger getIndexLogger(long resourceId, byte resourceType) throws ACIDException;
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
index 4eaf6ef..9963ec7 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
@@ -974,8 +974,8 @@
public void initializeDatasetIdFactory(JobId jobId) throws MetadataException, RemoteException {
int mostRecentDatasetId = MetadataPrimaryIndexes.FIRST_AVAILABLE_USER_DATASET_ID;
long resourceID = MetadataPrimaryIndexes.DATASET_DATASET.getResourceID();
- IIndex indexInstance = indexLifecycleManager.getIndex(resourceID);
try {
+ IIndex indexInstance = indexLifecycleManager.getIndex(resourceID);
indexLifecycleManager.open(resourceID);
IIndexAccessor indexAccessor = indexInstance.createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
index 76b4add..d833a13 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -27,8 +27,8 @@
import java.util.logging.Logger;
import edu.uci.ics.asterix.common.api.IAsterixAppRuntimeContext;
+import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
-import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
@@ -57,7 +57,6 @@
import edu.uci.ics.asterix.om.types.BuiltinType;
import edu.uci.ics.asterix.om.types.IAType;
import edu.uci.ics.asterix.runtime.formats.NonTaggedDataFormat;
-import edu.uci.ics.asterix.transaction.management.resource.ILocalResourceMetadata;
import edu.uci.ics.asterix.transaction.management.resource.LSMBTreeLocalResourceMetadata;
import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
import edu.uci.ics.asterix.transaction.management.service.logging.IndexResourceManager;
@@ -340,17 +339,14 @@
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER);
lsmBtree.create();
resourceID = runtimeContext.getResourceIdFactory().createId();
- indexLifecycleManager.register(resourceID, lsmBtree);
-
- AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(typeTraits,
- comparatorFactories, bloomFilterKeyFields, index.isPrimaryIndex(),
- storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
+ comparatorFactories, bloomFilterKeyFields, index.getDatasetId().getId());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMBTreeResource);
ILocalResourceFactory localResourceFactory = localResourceFactoryProvider.getLocalResourceFactory();
localResourceRepository.insert(localResourceFactory.createLocalResource(resourceID, file.getFile()
.getPath(), 0));
+ indexLifecycleManager.register(resourceID, lsmBtree);
} else {
resourceID = localResourceRepository.getResourceByName(file.getFile().getPath()).getResourceId();
lsmBtree = (LSMBTree) indexLifecycleManager.getIndex(resourceID);
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/PrimaryIndexModificationOperationCallback.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/PrimaryIndexModificationOperationCallback.java
index 15ccde4..9097570 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/PrimaryIndexModificationOperationCallback.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/PrimaryIndexModificationOperationCallback.java
@@ -41,9 +41,9 @@
protected final IndexOperation indexOp;
protected final ITransactionSubsystem txnSubsystem;
- public PrimaryIndexModificationOperationCallback(int datasetId, int[] primaryKeyFields,
- ITransactionContext txnCtx, ILockManager lockManager,
- ITransactionSubsystem txnSubsystem, long resourceId, byte resourceType, IndexOperation indexOp) {
+ public PrimaryIndexModificationOperationCallback(int datasetId, int[] primaryKeyFields, ITransactionContext txnCtx,
+ ILockManager lockManager, ITransactionSubsystem txnSubsystem, long resourceId, byte resourceType,
+ IndexOperation indexOp) {
super(datasetId, primaryKeyFields, txnCtx, lockManager);
this.resourceId = resourceId;
this.resourceType = resourceType;
@@ -63,19 +63,19 @@
@Override
public void found(ITupleReference before, ITupleReference after) throws HyracksDataException {
- ILogger logger = txnSubsystem.getTreeLoggerRepository().getIndexLogger(resourceId, resourceType);
- int pkHash = computePrimaryKeyHashValue(after, primaryKeyFields);
- LSMBTreeTupleReference lsmBTreeTuple = (LSMBTreeTupleReference) before;
- IndexOperation oldOp = IndexOperation.INSERT;
- if (before == null) {
- oldOp = IndexOperation.NOOP;
- }
- if (lsmBTreeTuple != null && lsmBTreeTuple.isAntimatter()) {
- oldOp = IndexOperation.DELETE;
- }
try {
- ((IndexLogger)logger).generateLogRecord(txnSubsystem, txnCtx, datasetId.getId(), pkHash, resourceId, indexOp, after,
- oldOp, before);
+ ILogger logger = txnSubsystem.getTreeLoggerRepository().getIndexLogger(resourceId, resourceType);
+ int pkHash = computePrimaryKeyHashValue(after, primaryKeyFields);
+ LSMBTreeTupleReference lsmBTreeTuple = (LSMBTreeTupleReference) before;
+ IndexOperation oldOp = IndexOperation.INSERT;
+ if (before == null) {
+ oldOp = IndexOperation.NOOP;
+ }
+ if (lsmBTreeTuple != null && lsmBTreeTuple.isAntimatter()) {
+ oldOp = IndexOperation.DELETE;
+ }
+ ((IndexLogger) logger).generateLogRecord(txnSubsystem, txnCtx, datasetId.getId(), pkHash, resourceId,
+ indexOp, after, oldOp, before);
} catch (ACIDException e) {
throw new HyracksDataException(e);
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/SecondaryIndexModificationOperationCallback.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/SecondaryIndexModificationOperationCallback.java
index 0e66b32..2287ed7 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/SecondaryIndexModificationOperationCallback.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/SecondaryIndexModificationOperationCallback.java
@@ -61,9 +61,10 @@
@Override
public void found(ITupleReference before, ITupleReference after) throws HyracksDataException {
- ILogger logger = txnSubsystem.getTreeLoggerRepository().getIndexLogger(resourceId, resourceType);
- int pkHash = computePrimaryKeyHashValue(after, primaryKeyFields);
+
try {
+ ILogger logger = txnSubsystem.getTreeLoggerRepository().getIndexLogger(resourceId, resourceType);
+ int pkHash = computePrimaryKeyHashValue(after, primaryKeyFields);
IndexOperation effectiveOldOp;
if (resourceType == ResourceType.LSM_BTREE) {
LSMBTreeTupleReference lsmBTreeTuple = (LSMBTreeTupleReference) before;
@@ -77,8 +78,8 @@
} else {
effectiveOldOp = oldOp;
}
- ((IndexLogger)logger).generateLogRecord(txnSubsystem, txnCtx, datasetId.getId(), pkHash, resourceId, indexOp, after,
- effectiveOldOp, before);
+ ((IndexLogger) logger).generateLogRecord(txnSubsystem, txnCtx, datasetId.getId(), pkHash, resourceId,
+ indexOp, after, effectiveOldOp, before);
} catch (ACIDException e) {
throw new HyracksDataException(e);
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/AbstractLSMLocalResourceMetadata.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/AbstractLSMLocalResourceMetadata.java
new file mode 100644
index 0000000..a95b677
--- /dev/null
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/AbstractLSMLocalResourceMetadata.java
@@ -0,0 +1,18 @@
+package edu.uci.ics.asterix.transaction.management.resource;
+
+import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
+
+public abstract class AbstractLSMLocalResourceMetadata implements ILocalResourceMetadata {
+
+ private static final long serialVersionUID = 1L;
+
+ protected final int datasetID;
+
+ public AbstractLSMLocalResourceMetadata(int datasetID) {
+ this.datasetID = datasetID;
+ }
+
+ public int getDatasetID() {
+ return datasetID;
+ }
+}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
index dd5d592..39fb695 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
@@ -10,34 +10,28 @@
import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
-import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
-public class LSMBTreeLocalResourceMetadata implements ILocalResourceMetadata {
+public class LSMBTreeLocalResourceMetadata extends AbstractLSMLocalResourceMetadata {
private static final long serialVersionUID = 1L;
private final ITypeTraits[] typeTraits;
private final IBinaryComparatorFactory[] cmpFactories;
private final int[] bloomFilterKeyFields;
- private final int memPageSize;
- private final int memNumPages;
public LSMBTreeLocalResourceMetadata(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
- int[] bloomFilterKeyFields, boolean isPrimary, int memPageSize, int memNumPages) {
+ int[] bloomFilterKeyFields, int datasetID) {
+ super(datasetID);
this.typeTraits = typeTraits;
this.cmpFactories = cmpFactories;
this.bloomFilterKeyFields = bloomFilterKeyFields;
- this.memPageSize = memPageSize;
- this.memNumPages = memNumPages;
}
@Override
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
int partition) {
FileReference file = new FileReference(new File(filePath));
- IVirtualBufferCache virtualBufferCache = new VirtualBufferCache(new HeapBufferAllocator(), memPageSize,
- memNumPages);
+ IVirtualBufferCache virtualBufferCache = runtimeContextProvider.getVirtualBufferCache(datasetID);
LSMBTree lsmBTree = LSMBTreeUtils.createLSMTree(virtualBufferCache, runtimeContextProvider.getIOManager(),
file, runtimeContextProvider.getBufferCache(), runtimeContextProvider.getFileMapManager(), typeTraits,
cmpFactories, bloomFilterKeyFields, runtimeContextProvider.getBloomFilterFalsePositiveRate(),
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
index aecbc76..743f36f 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
@@ -7,12 +7,10 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexUtils;
-import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
-public class LSMInvertedIndexLocalResourceMetadata implements ILocalResourceMetadata {
+public class LSMInvertedIndexLocalResourceMetadata extends AbstractLSMLocalResourceMetadata {
private static final long serialVersionUID = 1L;
@@ -21,29 +19,25 @@
private final ITypeTraits[] tokenTypeTraits;
private final IBinaryComparatorFactory[] tokenCmpFactories;
private final IBinaryTokenizerFactory tokenizerFactory;
- private final int memPageSize;
- private final int memNumPages;
private final boolean isPartitioned;
public LSMInvertedIndexLocalResourceMetadata(ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
- IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, int memPageSize,
- int memNumPages, boolean isPartitioned) {
+ IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
+ boolean isPartitioned, int datasetID) {
+ super(datasetID);
this.invListTypeTraits = invListTypeTraits;
this.invListCmpFactories = invListCmpFactories;
this.tokenTypeTraits = tokenTypeTraits;
this.tokenCmpFactories = tokenCmpFactories;
this.tokenizerFactory = tokenizerFactory;
- this.memPageSize = memPageSize;
- this.memNumPages = memNumPages;
this.isPartitioned = isPartitioned;
}
@Override
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
int partition) throws HyracksDataException {
- IVirtualBufferCache virtualBufferCache = new VirtualBufferCache(new HeapBufferAllocator(), memPageSize,
- memNumPages);
+ IVirtualBufferCache virtualBufferCache = runtimeContextProvider.getVirtualBufferCache(datasetID);
try {
if (isPartitioned) {
return InvertedIndexUtils.createPartitionedLSMInvertedIndex(virtualBufferCache,
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
index ea8628f..093ad13 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
@@ -12,12 +12,10 @@
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.utils.LSMRTreeUtils;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
-import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
-public class LSMRTreeLocalResourceMetadata implements ILocalResourceMetadata {
+public class LSMRTreeLocalResourceMetadata extends AbstractLSMLocalResourceMetadata {
private static final long serialVersionUID = 1L;
@@ -27,29 +25,24 @@
private final IPrimitiveValueProviderFactory[] valueProviderFactories;
private final RTreePolicyType rtreePolicyType;
private final ILinearizeComparatorFactory linearizeCmpFactory;
- private final int memPageSize;
- private final int memNumPages;
public LSMRTreeLocalResourceMetadata(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory, int memPageSize,
- int memNumPages) {
+ RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory, int datasetID) {
+ super(datasetID);
this.typeTraits = typeTraits;
this.rtreeCmpFactories = rtreeCmpFactories;
this.btreeCmpFactories = btreeCmpFactories;
this.valueProviderFactories = valueProviderFactories;
this.rtreePolicyType = rtreePolicyType;
this.linearizeCmpFactory = linearizeCmpFactory;
- this.memPageSize = memPageSize;
- this.memNumPages = memNumPages;
}
@Override
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
int partition) throws HyracksDataException {
FileReference file = new FileReference(new File(filePath));
- IVirtualBufferCache virtualBufferCache = new VirtualBufferCache(new HeapBufferAllocator(), memPageSize,
- memNumPages);
+ IVirtualBufferCache virtualBufferCache = runtimeContextProvider.getVirtualBufferCache(datasetID);
try {
return LSMRTreeUtils.createLSMTree(virtualBufferCache, runtimeContextProvider.getIOManager(), file,
runtimeContextProvider.getBufferCache(), runtimeContextProvider.getFileMapManager(), typeTraits,
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceFactory.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceFactory.java
index ed0f79f..707e02e 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceFactory.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceFactory.java
@@ -1,5 +1,6 @@
package edu.uci.ics.asterix.transaction.management.resource;
+import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
import edu.uci.ics.hyracks.storage.common.file.ILocalResourceFactory;
import edu.uci.ics.hyracks.storage.common.file.LocalResource;
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceFactoryProvider.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceFactoryProvider.java
index 4157c32..6afb9a9 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceFactoryProvider.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceFactoryProvider.java
@@ -1,5 +1,6 @@
package edu.uci.ics.asterix.transaction.management.resource;
+import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
import edu.uci.ics.hyracks.storage.common.file.ILocalResourceFactory;
import edu.uci.ics.hyracks.storage.common.file.ILocalResourceFactoryProvider;
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/IndexLoggerRepository.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/IndexLoggerRepository.java
index c51bb11..bd7d159 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/IndexLoggerRepository.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/IndexLoggerRepository.java
@@ -17,10 +17,12 @@
import java.util.HashMap;
import java.util.Map;
+import edu.uci.ics.asterix.common.exceptions.ACIDException;
import edu.uci.ics.asterix.common.transactions.ILogger;
import edu.uci.ics.asterix.common.transactions.ILoggerRepository;
import edu.uci.ics.asterix.common.transactions.ITransactionSubsystem;
import edu.uci.ics.asterix.common.transactions.MutableResourceId;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.common.api.IIndex;
public class IndexLoggerRepository implements ILoggerRepository {
@@ -35,13 +37,18 @@
}
@Override
- public synchronized ILogger getIndexLogger(long resourceId, byte resourceType) {
+ public synchronized ILogger getIndexLogger(long resourceId, byte resourceType) throws ACIDException {
mutableResourceId.setId(resourceId);
ILogger logger = loggers.get(mutableResourceId);
if (logger == null) {
MutableResourceId newMutableResourceId = new MutableResourceId(resourceId);
- IIndex index = (IIndex) txnSubsystem.getAsterixAppRuntimeContextProvider().getIndexLifecycleManager()
- .getIndex(resourceId);
+ IIndex index;
+ try {
+ index = (IIndex) txnSubsystem.getAsterixAppRuntimeContextProvider().getIndexLifecycleManager()
+ .getIndex(resourceId);
+ } catch (HyracksDataException e) {
+ throw new ACIDException(e);
+ }
logger = new IndexLogger(resourceId, resourceType, index);
loggers.put(newMutableResourceId, logger);
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/IndexResourceManager.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/IndexResourceManager.java
index ce3401c..8506df7 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/IndexResourceManager.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/IndexResourceManager.java
@@ -19,6 +19,7 @@
import edu.uci.ics.asterix.common.transactions.IResourceManager;
import edu.uci.ics.asterix.common.transactions.ITransactionSubsystem;
import edu.uci.ics.asterix.common.transactions.LogicalLogLocator;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.common.api.IIndex;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
@@ -45,11 +46,13 @@
long resourceId = logRecordHelper.getResourceId(logLocator);
int offset = logRecordHelper.getLogContentBeginPos(logLocator);
- //TODO
- //replace TransactionResourceRepository with IndexLifeCycleManager
- // look up the repository to obtain the resource object
- IIndex index = (IIndex) txnSubsystem.getAsterixAppRuntimeContextProvider().getIndexLifecycleManager()
- .getIndex(resourceId);
+ IIndex index;
+ try {
+ index = (IIndex) txnSubsystem.getAsterixAppRuntimeContextProvider().getIndexLifecycleManager()
+ .getIndex(resourceId);
+ } catch (HyracksDataException e1) {
+ throw new ACIDException("Cannot undo: unable to find index");
+ }
/* field count */
int fieldCount = logLocator.getBuffer().readInt(offset);
@@ -115,8 +118,13 @@
long resourceId = logRecordHelper.getResourceId(logLocator);
int offset = logRecordHelper.getLogContentBeginPos(logLocator);
- IIndex index = (IIndex) txnSubsystem.getAsterixAppRuntimeContextProvider().getIndexLifecycleManager()
- .getIndex(resourceId);
+ IIndex index;
+ try {
+ index = (IIndex) txnSubsystem.getAsterixAppRuntimeContextProvider().getIndexLifecycleManager()
+ .getIndex(resourceId);
+ } catch (HyracksDataException e1) {
+ throw new ACIDException("Cannot redo: unable to find index");
+ }
/* field count */
int fieldCount = logLocator.getBuffer().readInt(offset);
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/recovery/RecoveryManager.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/recovery/RecoveryManager.java
index 50d4625..91356a3 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/recovery/RecoveryManager.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/recovery/RecoveryManager.java
@@ -34,6 +34,7 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
import edu.uci.ics.asterix.common.exceptions.ACIDException;
import edu.uci.ics.asterix.common.transactions.IAsterixAppRuntimeContextProvider;
import edu.uci.ics.asterix.common.transactions.IBuffer;
@@ -48,7 +49,6 @@
import edu.uci.ics.asterix.common.transactions.LogicalLogLocator;
import edu.uci.ics.asterix.common.transactions.PhysicalLogLocator;
import edu.uci.ics.asterix.transaction.management.opcallbacks.IndexOperationTracker;
-import edu.uci.ics.asterix.transaction.management.resource.ILocalResourceMetadata;
import edu.uci.ics.asterix.transaction.management.service.logging.IndexResourceManager;
import edu.uci.ics.asterix.transaction.management.service.logging.LogManager;
import edu.uci.ics.asterix.transaction.management.service.logging.LogType;