[ASTERIXDB-1945][STO] Cleanup Buffer Cache API
- user model changes: no
- storage format changes: no
- interface changes: yes
INcApplicationContext
- removed IFileMapProvider getFileMapManager();
to hide FileMapManager from other components;
IStorageManager
- IFileMapProvider getFileMapProvider(INCServiceContext ctx);
to hide FileMapManager from other components;
IFileHandle
- added FileReference getFileReference();
to avoid unnecessary casts;
IIOManager
- public void deleteWorkspaceFiles() throws
HyracksDataException;
added throws;
ILSMIndexFileManager
- void createDirs() throws HyracksDataException;
added throws;
IInvertedIndex
- added void purge() throws HyracksDataException;
a. InvertedIndexes don't implement the ITreeIndex interface.
b. when we deactivate a disk component, we need to purge it so
the buffer cache doesn't go through each page.
c. this need to be revisited, ASTERIXDB-1944
IFileMapManager
- int registerFile(FileReference fileRef) throws
HyracksDataException;
return value added for future reference of the index file
inside BufferCache or VirtualBufferCache;
- FileReference unregisterFile(int fileId) throws
HyracksDataException;
return value added for future refernece of the file;
IBufferCache
- int createFile(FileReference fileRef) throws
HyracksDataException;
return value added for future reference of the index file
inside BufferCache or VirtualBufferCache;
- void deleteFile(int fileId) throws HyracksDataException;
remove the dirty page flag since there's no dirty page;
- int openFile(FileReference fileRef) throws
HyracksDataException;
return value added for future reference of the index file
inside BufferCache or VirtualBufferCache;
- added void deleteFile(FileReference file) throws
HyracksDataException;
we used to have this public methods in both BufferCache
and VirtualBufferCache. Now we lifted it into the interface.
AbstractLSMIndex
- removed protected abstract void
destroyMemoryComponent(ILSMMemoryComponent c)
throws HyracksDataException;
It is because turned out when we deactivate, we actually
destroy them. However, because of the not well defined API,
double destroy was okay and so we used to do double destroy.
Details:
This change fixes the buffer cache to follow the API such that:
1. createFile creates the file.
2. deleteFile deletes the file.
3. openFile opens the file.
4. closeFile closes the file.
5. creates existing file is not allowed.
6. deletes deleted file is not allowed.
7. open non existing file is not allowed.
In addition, we hide the file map from all other components.
Change-Id: I0a973c2adb2e7fdcbbf18c7b888af3de5f0acc74
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1843
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
index bc1e146..37e1c49 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
@@ -82,8 +82,9 @@
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ <version>${project.version}</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
index 54d64af..18d9a67 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
@@ -34,28 +34,28 @@
* merge).
*/
public interface ILSMIndexFileManager {
- public void createDirs();
+ void createDirs() throws HyracksDataException;
- public void deleteDirs() throws HyracksDataException;
+ void deleteDirs() throws HyracksDataException;
- public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException;
+ LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException;
- public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName)
+ LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName)
throws HyracksDataException;
- public String getBaseDir();
+ String getBaseDir();
// Deletes invalid files, and returns list of valid files from baseDir.
// The returned valid files are correctly sorted (based on the recency of data).
- public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException;
+ List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException;
- public Comparator<String> getFileNameComparator();
+ Comparator<String> getFileNameComparator();
/**
* @return delete existing transaction disk component file reference
* @throws HyracksDataException
*/
- public void deleteTransactionFiles() throws HyracksDataException;
+ void deleteTransactionFiles() throws HyracksDataException;
/**
* Rename files of a transaction removing the transaction prefix and return the component file reference in order to be committed
@@ -63,18 +63,18 @@
* @return the renamed component file references
* @throws HyracksDataException
*/
- public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException;
+ LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException;
/**
* Recover transaction files without returning them
*
* @throws HyracksDataException
*/
- public void recoverTransaction() throws HyracksDataException;
+ void recoverTransaction() throws HyracksDataException;
/**
* @return a reference to the transaction disk component file reference
* @throws IOException
*/
- public LSMComponentFileReferences getNewTransactionFileReference() throws IOException;
+ LSMComponentFileReferences getNewTransactionFileReference() throws IOException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java
index 090ca68..72d5423 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java
@@ -23,6 +23,6 @@
public class AbortRecoverLSMIndexFileManager extends AbstractLSMIndexFileManager {
public AbortRecoverLSMIndexFileManager(IIOManager ioManager, FileReference file) {
- super(ioManager, null, file, null);
+ super(ioManager, file, null);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index 2be2184..c5bf2ca 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -63,7 +63,6 @@
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.file.IFileMapProvider;
public abstract class AbstractLSMIndex implements ILSMIndex {
protected final ILSMHarness lsmHarness;
@@ -75,11 +74,9 @@
protected final List<ILSMMemoryComponent> memoryComponents;
protected final List<IVirtualBufferCache> virtualBufferCaches;
protected AtomicInteger currentMutableComponentId;
-
// On-disk components.
protected final IBufferCache diskBufferCache;
protected final ILSMIndexFileManager fileManager;
- protected final IFileMapProvider diskFileMapProvider;
// components with lower indexes are newer than components with higher index
protected final List<ILSMDiskComponent> diskComponents;
protected final List<ILSMDiskComponent> inactiveDiskComponents;
@@ -95,16 +92,14 @@
protected boolean memoryComponentsAllocated = false;
public AbstractLSMIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider,
- double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
- ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
- int[] filterFields, boolean durable, IComponentFilterHelper filterHelper,
- int[] treeFields) {
+ IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, double bloomFilterFalsePositiveRate,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFrameFactory filterFrameFactory,
+ LSMComponentFilterManager filterManager, int[] filterFields, boolean durable,
+ IComponentFilterHelper filterHelper, int[] treeFields) {
this.ioManager = ioManager;
this.virtualBufferCaches = virtualBufferCaches;
this.diskBufferCache = diskBufferCache;
- this.diskFileMapProvider = diskFileMapProvider;
this.fileManager = fileManager;
this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
this.ioScheduler = ioScheduler;
@@ -130,12 +125,10 @@
// The constructor used by external indexes
public AbstractLSMIndex(IIOManager ioManager, IBufferCache diskBufferCache, ILSMIndexFileManager fileManager,
- IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
- ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
- boolean durable) {
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean durable) {
this.ioManager = ioManager;
this.diskBufferCache = diskBufferCache;
- this.diskFileMapProvider = diskFileMapProvider;
this.fileManager = fileManager;
this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
this.ioScheduler = ioScheduler;
@@ -162,7 +155,6 @@
if (isActive) {
throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_ACTIVE_INDEX);
}
- fileManager.deleteDirs();
fileManager.createDirs();
diskComponents.clear();
}
@@ -238,16 +230,9 @@
throw HyracksDataException.create(ErrorCode.CANNOT_DESTROY_ACTIVE_INDEX);
}
destroyDiskComponents();
- destroyMemoryComponents();
fileManager.deleteDirs();
}
- protected void destroyMemoryComponents() throws HyracksDataException {
- for (ILSMMemoryComponent c : memoryComponents) {
- destroyMemoryComponent(c);
- }
- }
-
protected void destroyDiskComponents() throws HyracksDataException {
List<ILSMDiskComponent> immutableComponents = diskComponents;
for (ILSMDiskComponent c : immutableComponents) {
@@ -653,8 +638,6 @@
protected abstract void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException;
- protected abstract void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException;
-
protected abstract void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException;
protected abstract void clearMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
index fca010f..ead9db4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
@@ -33,11 +33,11 @@
import java.util.HashSet;
import java.util.List;
-import org.apache.commons.io.FileUtils;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.util.IoUtil;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
@@ -45,49 +45,63 @@
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
-import org.apache.hyracks.storage.common.file.IFileMapProvider;
public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManager {
- public static final String SPLIT_STRING = "_";
- protected static final String BLOOM_FILTER_STRING = "f";
- protected static final String TRANSACTION_PREFIX = ".T";
-
public enum TreeIndexState {
INVALID,
VERSION_MISMATCH,
VALID
}
+ /**
+ * Split different parts of the file name
+ */
+ public static final String DELIMITER = "_";
+ /**
+ * Indicates a B tree
+ */
+ public static final String BTREE_SUFFIX = "b";
+ /**
+ * Indicates an R tree
+ */
+ public static final String RTREE_SUFFIX = "r";
+ /**
+ * Indicates a bloom filter
+ */
+ public static final String BLOOM_FILTER_SUFFIX = "f";
+ /**
+ * Indicates a delete tree
+ */
+ public static final String DELETE_TREE_SUFFIX = "d";
+ /**
+ * Hides transaction components until they are either committed by removing this file or deleted along with the file
+ */
+ public static final String TXN_PREFIX = ".T";
+
+ protected static final FilenameFilter fileNameFilter = (dir, name) -> !name.startsWith(".");
+ protected static final FilenameFilter txnFileNameFilter = (dir, name) -> name.startsWith(TXN_PREFIX);
+ protected static FilenameFilter bloomFilterFilter =
+ (dir, name) -> !name.startsWith(".") && name.endsWith(BLOOM_FILTER_SUFFIX);
+ protected static final FilenameFilter dummyFilter = (dir, name) -> true;
+ protected static final Comparator<String> cmp = new FileNameComparator();
+
protected final IIOManager ioManager;
- protected final IFileMapProvider fileMapProvider;
// baseDir should reflect dataset name and partition name and be absolute
- protected String baseDir;
+ protected final String baseDir;
protected final Format formatter = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS");
- protected final Comparator<String> cmp = new FileNameComparator();
protected final Comparator<ComparableFileName> recencyCmp = new RecencyComparator();
protected final TreeIndexFactory<? extends ITreeIndex> treeFactory;
-
private String prevTimestamp = null;
- public AbstractLSMIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
+ public AbstractLSMIndexFileManager(IIOManager ioManager, FileReference file,
TreeIndexFactory<? extends ITreeIndex> treeFactory) {
this.ioManager = ioManager;
- this.baseDir = file.getFile().getAbsolutePath();
- if (!baseDir.endsWith(System.getProperty("file.separator"))) {
- baseDir += System.getProperty("file.separator");
- }
- this.fileMapProvider = fileMapProvider;
+ this.baseDir = file.getFile().getAbsolutePath().endsWith(File.separator) ? file.getFile().getAbsolutePath()
+ : file.getFile().getAbsolutePath() + File.separator;
this.treeFactory = treeFactory;
}
- private static FilenameFilter fileNameFilter = new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return !name.startsWith(".");
- }
- };
-
protected TreeIndexState isValidTreeIndex(ITreeIndex treeIndex) throws HyracksDataException {
IBufferCache bufferCache = treeIndex.getBufferCache();
treeIndex.activate();
@@ -148,7 +162,7 @@
if (!dir.canRead()) {
throw HyracksDataException.create(ErrorCode.CANNOT_READ_FILE, path);
} else if (!dir.exists()) {
- throw HyracksDataException.create(ErrorCode.FILE_DOES_NOT_EXISTS, path);
+ throw HyracksDataException.create(ErrorCode.FILE_DOES_NOT_EXIST, path);
} else if (!dir.isDirectory()) {
throw HyracksDataException.create(ErrorCode.FILE_IS_NOT_DIRECTORY, path);
}
@@ -162,44 +176,31 @@
ArrayList<ComparableFileName> tmpAllInvListsFiles = new ArrayList<>();
cleanupAndGetValidFilesInternal(filter, treeFactory, tmpAllInvListsFiles);
for (ComparableFileName cmpFileName : tmpAllInvListsFiles) {
- int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
+ int index = cmpFileName.fileName.lastIndexOf(DELIMITER);
String file = cmpFileName.fileName.substring(0, index);
if (groundTruth.contains(file)) {
validFiles.add(cmpFileName);
} else {
File invalidFile = new File(cmpFileName.fullPath);
- invalidFile.delete();
+ IoUtil.delete(invalidFile);
}
}
}
@Override
- public void createDirs() {
+ public void createDirs() throws HyracksDataException {
File f = new File(baseDir);
+ if (f.exists()) {
+ throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_EXISTING_INDEX);
+ }
f.mkdirs();
}
@Override
public void deleteDirs() throws HyracksDataException {
- File f = new File(baseDir);
- if (f.exists()) {
- delete(f);
- }
+ IoUtil.delete(new File(baseDir));
}
- private void delete(File f) throws HyracksDataException {
- if (!FileUtils.deleteQuietly(f)) {
- throw HyracksDataException.create(ErrorCode.UNIDENTIFIED_IO_ERROR_DELETING_DIR, f.getPath());
- }
- }
-
- protected static FilenameFilter bloomFilterFilter = new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return !name.startsWith(".") && name.endsWith(BLOOM_FILTER_STRING);
- }
- };
-
protected FileReference createFlushFile(String flushFileName) throws HyracksDataException {
return ioManager.resolveAbsolutePath(flushFileName);
}
@@ -212,17 +213,17 @@
public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException {
String ts = getCurrentTimestamp();
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseDir + ts + SPLIT_STRING + ts), null, null);
+ return new LSMComponentFileReferences(createFlushFile(baseDir + ts + DELIMITER + ts), null, null);
}
@Override
public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName)
throws HyracksDataException {
- String[] firstTimestampRange = firstFileName.split(SPLIT_STRING);
- String[] lastTimestampRange = lastFileName.split(SPLIT_STRING);
+ String[] firstTimestampRange = firstFileName.split(DELIMITER);
+ String[] lastTimestampRange = lastFileName.split(DELIMITER);
// Get the range of timestamps by taking the earliest and the latest timestamps
return new LSMComponentFileReferences(
- createMergeFile(baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1]), null, null);
+ createMergeFile(baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]), null, null);
}
@Override
@@ -292,7 +293,7 @@
* 2. Flushed files are sorted from newest to oldest (based on the timestamp
* string)
*/
- private class FileNameComparator implements Comparator<String> {
+ private static class FileNameComparator implements Comparator<String> {
@Override
public int compare(String a, String b) {
// Consciously ignoring locale.
@@ -307,7 +308,7 @@
@Override
public void recoverTransaction() throws HyracksDataException {
- String[] files = listDirFiles(baseDir, transactionFileNameFilter);
+ String[] files = listDirFiles(baseDir, txnFileNameFilter);
File dir = new File(baseDir);
try {
if (files.length == 0) {
@@ -334,7 +335,7 @@
this.fileRef = fileRef;
this.fullPath = fileRef.getFile().getAbsolutePath();
this.fileName = fileRef.getFile().getName();
- interval = fileName.split(SPLIT_STRING);
+ interval = fileName.split(DELIMITER);
}
@Override
@@ -361,7 +362,7 @@
// This function is used to delete transaction files for aborted transactions
@Override
public void deleteTransactionFiles() throws HyracksDataException {
- String[] files = listDirFiles(baseDir, transactionFileNameFilter);
+ String[] files = listDirFiles(baseDir, txnFileNameFilter);
if (files.length == 0) {
// Do nothing
} else if (files.length > 1) {
@@ -399,23 +400,9 @@
return null;
}
- protected static FilenameFilter transactionFileNameFilter = new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return name.startsWith(".T");
- }
- };
-
- protected static FilenameFilter dummyFilter = new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return true;
- }
- };
-
protected static FilenameFilter createTransactionFilter(String transactionFileName, final boolean inclusive) {
- final String timeStamp = transactionFileName
- .substring(transactionFileName.indexOf(TRANSACTION_PREFIX) + TRANSACTION_PREFIX.length());
+ final String timeStamp =
+ transactionFileName.substring(transactionFileName.indexOf(TXN_PREFIX) + TXN_PREFIX.length());
return new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
@@ -429,7 +416,7 @@
}
protected FilenameFilter getTransactionFileFilter(boolean inclusive) throws HyracksDataException {
- String[] files = listDirFiles(baseDir, transactionFileNameFilter);
+ String[] files = listDirFiles(baseDir, txnFileNameFilter);
if (files.length == 0) {
return dummyFilter;
} else {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
index 562ed5a..4eefcd8 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
@@ -26,21 +26,20 @@
import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class BTreeFactory extends TreeIndexFactory<BTree> {
- public BTreeFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider,
- IPageManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
- super(ioManager, bufferCache, fileMapProvider, freePageManagerFactory, interiorFrameFactory, leafFrameFactory,
- cmpFactories, fieldCount);
+ public BTreeFactory(IIOManager ioManager, IBufferCache bufferCache, IPageManagerFactory freePageManagerFactory,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
+ IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
+ super(ioManager, bufferCache, freePageManagerFactory, interiorFrameFactory, leafFrameFactory, cmpFactories,
+ fieldCount);
}
@Override
public BTree createIndexInstance(FileReference file) {
- return new BTree(bufferCache, fileMapProvider, freePageManagerFactory.createPageManager(bufferCache),
- interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, file);
+ return new BTree(bufferCache, freePageManagerFactory.createPageManager(bufferCache), interiorFrameFactory,
+ leafFrameFactory, cmpFactories, fieldCount, file);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java
index 55fc92c..7a05d6d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java
@@ -25,20 +25,16 @@
import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.file.IFileMapProvider;
public abstract class IndexFactory<T extends IIndex> {
protected final IIOManager ioManager;
protected final IBufferCache bufferCache;
- protected final IFileMapProvider fileMapProvider;
protected final IPageManagerFactory freePageManagerFactory;
- public IndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider,
- IPageManagerFactory freePageManagerFactory) {
+ public IndexFactory(IIOManager ioManager, IBufferCache bufferCache, IPageManagerFactory freePageManagerFactory) {
this.ioManager = ioManager;
this.bufferCache = bufferCache;
- this.fileMapProvider = fileMapProvider;
this.freePageManagerFactory = freePageManagerFactory;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
index 6878fcf..83e140c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
@@ -18,6 +18,8 @@
*/
package org.apache.hyracks.storage.am.lsm.common.impls;
+import java.util.HashMap;
+import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -29,6 +31,7 @@
import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
import org.apache.hyracks.storage.common.file.IFileMapManager;
+import org.apache.hyracks.util.JSONUtil;
public class MultitenantVirtualBufferCache implements IVirtualBufferCache {
private static final Logger LOGGER = Logger.getLogger(ExternalIndexHarness.class.getName());
@@ -42,8 +45,8 @@
}
@Override
- public void createFile(FileReference fileRef) throws HyracksDataException {
- vbc.createFile(fileRef);
+ public int createFile(FileReference fileRef) throws HyracksDataException {
+ return vbc.createFile(fileRef);
}
@Override
@@ -57,8 +60,8 @@
}
@Override
- public void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException {
- vbc.deleteFile(fileId, flushDirtyPages);
+ public void deleteFile(int fileId) throws HyracksDataException {
+ vbc.deleteFile(fileId);
}
@Override
@@ -91,6 +94,7 @@
return vbc.getPageSize();
}
+ @Override
public int getPageSizeWithHeader() {
return vbc.getPageSizeWithHeader();
}
@@ -133,7 +137,7 @@
@Override
public int getNumPagesOfFile(int fileId) throws HyracksDataException {
- throw new UnsupportedOperationException();
+ return vbc.getNumPagesOfFile(fileId);
}
@Override
@@ -205,4 +209,28 @@
throws HyracksDataException {
vbc.resizePage(page, multiplier, extraPageBlockHelper);
}
+
+ @Override
+ public String toString() {
+ return JSONUtil.fromMap(toMap());
+ }
+
+ private Map<String, Object> toMap() {
+ HashMap<String, Object> map = new HashMap<>();
+ map.put("class", getClass().getSimpleName());
+ map.put("vbc", vbc.toString());
+ map.put("openCount", openCount);
+ return map;
+ }
+
+ @Override
+ public int openFile(FileReference fileRef) throws HyracksDataException {
+ return vbc.openFile(fileRef);
+ }
+
+ @Override
+ public void deleteFile(FileReference file) throws HyracksDataException {
+ vbc.deleteFile(file);
+ }
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java
index 2a7bcca..b6e2dd4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java
@@ -25,7 +25,6 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.file.IFileMapProvider;
public abstract class TreeIndexFactory<T extends ITreeIndex> extends IndexFactory<T> {
@@ -34,10 +33,10 @@
protected final IBinaryComparatorFactory[] cmpFactories;
protected final int fieldCount;
- public TreeIndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider,
- IPageManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
- super(ioManager, bufferCache, fileMapProvider, freePageManagerFactory);
+ public TreeIndexFactory(IIOManager ioManager, IBufferCache bufferCache, IPageManagerFactory freePageManagerFactory,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
+ IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
+ super(ioManager, bufferCache, freePageManagerFactory);
this.interiorFrameFactory = interiorFrameFactory;
this.leafFrameFactory = leafFrameFactory;
this.cmpFactories = cmpFactories;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
index 27d879c..83e377a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
@@ -21,6 +21,8 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import java.util.logging.Level;
@@ -37,7 +39,8 @@
import org.apache.hyracks.storage.common.buffercache.VirtualPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
import org.apache.hyracks.storage.common.file.IFileMapManager;
-import org.apache.hyracks.storage.common.file.TransientFileMapManager;
+import org.apache.hyracks.storage.common.file.FileMapManager;
+import org.apache.hyracks.util.JSONUtil;
public class VirtualBufferCache implements IVirtualBufferCache {
private static final Logger LOGGER = Logger.getLogger(ExternalIndexHarness.class.getName());
@@ -59,7 +62,7 @@
public VirtualBufferCache(ICacheMemoryAllocator allocator, int pageSize, int numPages) {
this.allocator = allocator;
- this.fileMapManager = new TransientFileMapManager();
+ this.fileMapManager = new FileMapManager();
this.pageSize = pageSize;
this.numPages = 2 * (numPages / 2) + 1;
@@ -71,12 +74,19 @@
}
@Override
- public void createFile(FileReference fileRef) throws HyracksDataException {
+ public int createFile(FileReference fileRef) throws HyracksDataException {
+ synchronized (fileMapManager) {
+ return fileMapManager.registerFile(fileRef);
+ }
+ }
+
+ @Override
+ public int openFile(FileReference fileRef) throws HyracksDataException {
synchronized (fileMapManager) {
if (fileMapManager.isMapped(fileRef)) {
- throw new HyracksDataException("File " + fileRef + " is already mapped");
+ return fileMapManager.lookupFileId(fileRef);
}
- fileMapManager.registerFile(fileRef);
+ return fileMapManager.registerFile(fileRef);
}
}
@@ -89,14 +99,18 @@
}
@Override
- public void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException {
+ public void deleteFile(FileReference fileRef) throws HyracksDataException {
synchronized (fileMapManager) {
- if (!fileMapManager.isMapped(fileId)) {
- throw new HyracksDataException("File with id " + fileId + " is not mapped");
- }
+ int fileId = fileMapManager.lookupFileId(fileRef);
+ deleteFile(fileId);
+ }
+ }
+
+ @Override
+ public void deleteFile(int fileId) throws HyracksDataException {
+ synchronized (fileMapManager) {
fileMapManager.unregisterFile(fileId);
}
-
for (int i = 0; i < buckets.length; i++) {
final CacheBucket bucket = buckets[i];
bucket.bucketLock.lock();
@@ -185,8 +199,11 @@
}
if (!newPage) {
- throw new HyracksDataException("Page " + BufferedFileHandle.getPageId(dpid) + " does not exist in file "
- + fileMapManager.lookupFileName(BufferedFileHandle.getFileId(dpid)));
+ synchronized (fileMapManager) {
+ throw new HyracksDataException(
+ "Page " + BufferedFileHandle.getPageId(dpid) + " does not exist in file "
+ + fileMapManager.lookupFileName(BufferedFileHandle.getFileId(dpid)));
+ }
}
page = getOrAllocPage(dpid);
@@ -355,12 +372,7 @@
@Override
public int getNumPagesOfFile(int fileId) throws HyracksDataException {
- synchronized (fileMapManager) {
- if (fileMapManager.isMapped(fileId)) {
- return numPages;
- }
- }
- return 0;
+ return -1;
}
@Override
@@ -424,7 +436,21 @@
@Override
public void purgeHandle(int fileId) throws HyracksDataException {
-
+ deleteFile(fileId);
}
+ @Override
+ public String toString() {
+ return JSONUtil.fromMap(toMap());
+ }
+
+ private Map<String, Object> toMap() {
+ HashMap<String, Object> map = new HashMap<>();
+ map.put("class", getClass().getSimpleName());
+ map.put("allocator", allocator.toString());
+ map.put("pageSize", pageSize);
+ map.put("numPages", numPages);
+ map.put("open", open);
+ return map;
+ }
}