Revert "ASTERIXDB-1945 [STO] Cleanup Buffer Cache API"
This reverts commit ae3daf6ef3397e583637360dc460c6391e03dc29.
Change-Id: I5e4e23f43a68e82c38fb8d1d7f4c0d01985c3a10
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1842
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <imaxon@apache.org>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AppRuntimeContextProviderForRecovery.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AppRuntimeContextProviderForRecovery.java
index 1fea840..397b459 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AppRuntimeContextProviderForRecovery.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AppRuntimeContextProviderForRecovery.java
@@ -19,9 +19,9 @@
package org.apache.asterix.api.common;
import org.apache.asterix.app.nc.NCAppRuntimeContext;
-import org.apache.asterix.common.api.IDatasetLifecycleManager;
-import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.api.ThreadExecutor;
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.api.IDatasetLifecycleManager;
import org.apache.asterix.common.transactions.IAppRuntimeContextProvider;
import org.apache.asterix.common.transactions.ITransactionSubsystem;
import org.apache.hyracks.api.io.IIOManager;
@@ -29,6 +29,7 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.common.ILocalResourceRepository;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class AppRuntimeContextProviderForRecovery implements IAppRuntimeContextProvider {
@@ -44,6 +45,11 @@
}
@Override
+ public IFileMapProvider getFileMapManager() {
+ return asterixAppRuntimeContext.getFileMapManager();
+ }
+
+ @Override
public ITransactionSubsystem getTransactionSubsystem() {
return asterixAppRuntimeContext.getTransactionSubsystem();
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index 55b9adc..c23da6c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -47,6 +47,7 @@
import org.apache.asterix.common.config.StorageProperties;
import org.apache.asterix.common.config.TransactionProperties;
import org.apache.asterix.common.context.DatasetLifecycleManager;
+import org.apache.asterix.common.context.FileMapManager;
import org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
@@ -93,7 +94,8 @@
import org.apache.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
import org.apache.hyracks.storage.common.buffercache.IPageCleanerPolicy;
import org.apache.hyracks.storage.common.buffercache.IPageReplacementStrategy;
-import org.apache.hyracks.storage.common.file.FileMapManager;
+import org.apache.hyracks.storage.common.file.IFileMapManager;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.storage.common.file.ILocalResourceRepositoryFactory;
import org.apache.hyracks.storage.common.file.IResourceIdFactory;
@@ -115,6 +117,7 @@
private final NodeProperties nodeProperties;
private ThreadExecutor threadExecutor;
private IDatasetLifecycleManager datasetLifecycleManager;
+ private IFileMapManager fileMapManager;
private IBufferCache bufferCache;
private ITransactionSubsystem txnSubsystem;
private IMetadataNode metadataNodeStub;
@@ -165,6 +168,7 @@
public void initialize(boolean initialRun) throws IOException, ACIDException {
ioManager = getServiceContext().getIoManager();
threadExecutor = new ThreadExecutor(getServiceContext().getThreadFactory());
+ fileMapManager = new FileMapManager(ioManager);
ICacheMemoryAllocator allocator = new HeapBufferAllocator();
IPageCleanerPolicy pcp = new DelayPageCleanerPolicy(600000);
IPageReplacementStrategy prs = new ClockPageReplacementStrategy(allocator,
@@ -232,15 +236,16 @@
//initialize replication channel
replicationChannel = new ReplicationChannel(nodeId, replicationProperties, txnSubsystem.getLogManager(),
- replicaResourcesManager, replicationManager, getServiceContext(), asterixAppRuntimeContextProvider);
+ replicaResourcesManager, replicationManager, getServiceContext(),
+ asterixAppRuntimeContextProvider);
remoteRecoveryManager = new RemoteRecoveryManager(replicationManager, this, replicationProperties);
- bufferCache = new BufferCache(ioManager, prs, pcp, new FileMapManager(),
+ bufferCache = new BufferCache(ioManager, prs, pcp, fileMapManager,
storageProperties.getBufferCacheMaxOpenFiles(), getServiceContext().getThreadFactory(),
replicationManager);
} else {
- bufferCache = new BufferCache(ioManager, prs, pcp, new FileMapManager(),
+ bufferCache = new BufferCache(ioManager, prs, pcp, fileMapManager,
storageProperties.getBufferCacheMaxOpenFiles(), getServiceContext().getThreadFactory());
}
@@ -297,6 +302,11 @@
}
@Override
+ public IFileMapProvider getFileMapManager() {
+ return fileMapManager;
+ }
+
+ @Override
public ITransactionSubsystem getTransactionSubsystem() {
return txnSubsystem;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index 0f6b396..7c8e153 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -46,17 +46,16 @@
import org.apache.asterix.messaging.MessagingChannelInterfaceFactory;
import org.apache.asterix.messaging.NCMessageBroker;
import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
+import org.apache.commons.io.FileUtils;
import org.apache.hyracks.api.application.INCServiceContext;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.client.ClusterControllerInfo;
import org.apache.hyracks.api.client.HyracksConnection;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.config.IConfigManager;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.IFileDeviceResolver;
import org.apache.hyracks.api.job.resource.NodeCapacity;
import org.apache.hyracks.api.messages.IMessageBroker;
-import org.apache.hyracks.api.util.IoUtil;
import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.control.nc.BaseNCApplication;
import org.apache.hyracks.control.nc.NodeControllerService;
@@ -215,7 +214,7 @@
return new NodeCapacity(memorySize, maximumCoresForComputation);
}
- private void performLocalCleanUp() throws HyracksDataException {
+ private void performLocalCleanUp() {
//Delete working area files from failed jobs
runtimeContext.getIoManager().deleteWorkspaceFiles();
@@ -226,10 +225,7 @@
for (String ioDevice : ioDevices) {
String tempDatasetsDir =
ioDevice + storageDirName + File.separator + StoragePathUtil.TEMP_DATASETS_STORAGE_FOLDER;
- File tmpDsDir = new File(tempDatasetsDir);
- if (tmpDsDir.exists()) {
- IoUtil.delete(tmpDsDir);
- }
+ FileUtils.deleteQuietly(new File(tempDatasetsDir));
}
//TODO
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java
index d4b9a92..a4b994b 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java
@@ -38,6 +38,7 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.common.ILocalResourceRepository;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.storage.common.file.IResourceIdFactory;
public interface INcApplicationContext extends IApplicationContext {
@@ -58,6 +59,8 @@
IBufferCache getBufferCache();
+ IFileMapProvider getFileMapManager();
+
ILocalResourceRepository getLocalResourceRepository();
IDatasetLifecycleManager getDatasetLifecycleManager();
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/FileMapManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/FileMapManager.java
new file mode 100644
index 0000000..c4a294d
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/FileMapManager.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.context;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.storage.common.file.IFileMapManager;
+
+public class FileMapManager implements IFileMapManager {
+
+ private static final long serialVersionUID = 1L;
+ private final transient IIOManager ioManager;
+ private final Map<Integer, String> id2nameMap = new HashMap<>();
+ private final Map<String, Integer> name2IdMap = new HashMap<>();
+ private int idCounter = 0;
+
+ public FileMapManager(IIOManager ioManager) {
+ this.ioManager = ioManager;
+ }
+
+ @Override
+ public FileReference lookupFileName(int fileId) throws HyracksDataException {
+ String fName = id2nameMap.get(fileId);
+ if (fName == null) {
+ throw new HyracksDataException("No mapping found for id: " + fileId);
+ }
+ return ioManager.resolveAbsolutePath(fName);
+ }
+
+ @Override
+ public int lookupFileId(FileReference fileRef) throws HyracksDataException {
+ String fileName = fileRef.getFile().getAbsolutePath();
+ Integer fileId = name2IdMap.get(fileName);
+ if (fileId == null) {
+ throw new HyracksDataException("No mapping found for name: " + fileName);
+ }
+ return fileId;
+ }
+
+ @Override
+ public boolean isMapped(FileReference fileRef) {
+ String fileName = fileRef.getFile().getAbsolutePath();
+ return name2IdMap.containsKey(fileName);
+ }
+
+ @Override
+ public boolean isMapped(int fileId) {
+ return id2nameMap.containsKey(fileId);
+ }
+
+ @Override
+ public void unregisterFile(int fileId) throws HyracksDataException {
+ String fileName = id2nameMap.remove(fileId);
+ name2IdMap.remove(fileName);
+ }
+
+ @Override
+ public void registerFile(FileReference fileRef) throws HyracksDataException {
+ Integer fileId = idCounter++;
+ String fileName = fileRef.getFile().getAbsolutePath();
+ id2nameMap.put(fileId, fileName);
+ name2IdMap.put(fileName, fileId);
+ }
+
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java
index 8c91879..b9442dc 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java
@@ -34,7 +34,7 @@
@Override
public long getComponentFileLSNOffset(ILSMDiskComponent diskComponent, String diskComponentFilePath)
throws HyracksDataException {
- if (diskComponentFilePath.endsWith(LSMBTreeFileManager.BTREE_SUFFIX)) {
+ if (diskComponentFilePath.endsWith(LSMBTreeFileManager.BTREE_STRING)) {
LSMBTreeDiskComponent btreeComponent = (LSMBTreeDiskComponent) diskComponent;
IMetadataPageManager metadataPageManager =
(IMetadataPageManager) btreeComponent.getBTree().getPageManager();
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java
index 9f50f57..0e80397 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java
@@ -29,7 +29,7 @@
@Override
public long getComponentFileLSNOffset(ILSMDiskComponent diskComponent, String diskComponentFilePath)
throws HyracksDataException {
- if (diskComponentFilePath.endsWith(LSMBTreeWithBuddyFileManager.BTREE_SUFFIX)) {
+ if (diskComponentFilePath.endsWith(LSMBTreeWithBuddyFileManager.BTREE_STRING)) {
LSMBTreeWithBuddyDiskComponent btreeComponent = (LSMBTreeWithBuddyDiskComponent) diskComponent;
IMetadataPageManager metadataPageManager =
(IMetadataPageManager) btreeComponent.getBTree().getPageManager();
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java
index 1001af9..4908c08 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java
@@ -34,7 +34,7 @@
@Override
public long getComponentFileLSNOffset(ILSMDiskComponent diskComponent, String diskComponentFilePath)
throws HyracksDataException {
- if (diskComponentFilePath.endsWith(LSMRTreeFileManager.RTREE_SUFFIX)) {
+ if (diskComponentFilePath.endsWith(LSMRTreeFileManager.RTREE_STRING)) {
LSMRTreeDiskComponent rtreeComponent = (LSMRTreeDiskComponent) diskComponent;
IMetadataPageManager metadataPageManager =
(IMetadataPageManager) rtreeComponent.getRTree().getPageManager();
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IAppRuntimeContextProvider.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IAppRuntimeContextProvider.java
index 49f5457..f4b638f 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IAppRuntimeContextProvider.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IAppRuntimeContextProvider.java
@@ -18,34 +18,37 @@
*/
package org.apache.asterix.common.transactions;
-import org.apache.asterix.common.api.IDatasetLifecycleManager;
-import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.api.ThreadExecutor;
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.api.IDatasetLifecycleManager;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.common.ILocalResourceRepository;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public interface IAppRuntimeContextProvider {
- ThreadExecutor getThreadExecutor();
+ public ThreadExecutor getThreadExecutor();
- IBufferCache getBufferCache();
+ public IBufferCache getBufferCache();
- ITransactionSubsystem getTransactionSubsystem();
+ public IFileMapProvider getFileMapManager();
- IDatasetLifecycleManager getDatasetLifecycleManager();
+ public ITransactionSubsystem getTransactionSubsystem();
- double getBloomFilterFalsePositiveRate();
+ public IDatasetLifecycleManager getDatasetLifecycleManager();
- ILSMOperationTracker getLSMBTreeOperationTracker(int datasetID);
+ public double getBloomFilterFalsePositiveRate();
- ILSMIOOperationScheduler getLSMIOScheduler();
+ public ILSMOperationTracker getLSMBTreeOperationTracker(int datasetID);
- ILocalResourceRepository getLocalResourceRepository();
+ public ILSMIOOperationScheduler getLSMIOScheduler();
- IIOManager getIOManager();
+ public ILocalResourceRepository getLocalResourceRepository();
- INcApplicationContext getAppContext();
+ public IIOManager getIOManager();
+
+ public INcApplicationContext getAppContext();
}
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/storage/LSMComponentProperties.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/storage/LSMComponentProperties.java
index a8b15d2..8dbd98f 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/storage/LSMComponentProperties.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/storage/LSMComponentProperties.java
@@ -54,7 +54,6 @@
job.getLSMIndexOperationContext());
opType = job.getLSMOpType();
}
-
public LSMComponentProperties() {
}
@@ -133,7 +132,7 @@
componentId.append(File.separator);
componentId.append(idxName);
componentId.append(File.separator);
- componentId.append(fileName.substring(0, fileName.lastIndexOf(AbstractLSMIndexFileManager.DELIMITER)));
+ componentId.append(fileName.substring(0, fileName.lastIndexOf(AbstractLSMIndexFileManager.SPLIT_STRING)));
return componentId.toString();
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/RuntimeComponentsProvider.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/RuntimeComponentsProvider.java
index 177bed2..ebebfbd 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/RuntimeComponentsProvider.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/RuntimeComponentsProvider.java
@@ -26,6 +26,7 @@
import org.apache.hyracks.storage.common.ILocalResourceRepository;
import org.apache.hyracks.storage.common.IStorageManager;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.storage.common.file.IResourceIdFactory;
public class RuntimeComponentsProvider implements IStorageManager, ILSMIOOperationSchedulerProvider {
@@ -48,6 +49,11 @@
}
@Override
+ public IFileMapProvider getFileMapProvider(INCServiceContext ctx) {
+ return ((INcApplicationContext) ctx.getApplicationContext()).getFileMapManager();
+ }
+
+ @Override
public ILocalResourceRepository getLocalResourceRepository(INCServiceContext ctx) {
return ((INcApplicationContext) ctx.getApplicationContext()).getLocalResourceRepository();
}
diff --git a/asterixdb/asterix-transactions/src/test/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java b/asterixdb/asterix-transactions/src/test/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java
index a27f987..ba34414 100644
--- a/asterixdb/asterix-transactions/src/test/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java
+++ b/asterixdb/asterix-transactions/src/test/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java
@@ -22,9 +22,9 @@
import java.util.concurrent.Executors;
-import org.apache.asterix.common.api.IDatasetLifecycleManager;
-import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.api.ThreadExecutor;
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.api.IDatasetLifecycleManager;
import org.apache.asterix.common.transactions.IAppRuntimeContextProvider;
import org.apache.asterix.common.transactions.ITransactionSubsystem;
import org.apache.hyracks.api.io.IIOManager;
@@ -32,6 +32,7 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.common.ILocalResourceRepository;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
class TestRuntimeContextProvider implements IAppRuntimeContextProvider {
@@ -49,6 +50,11 @@
}
@Override
+ public IFileMapProvider getFileMapManager() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public ITransactionSubsystem getTransactionSubsystem() {
throw new UnsupportedOperationException();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/pom.xml b/hyracks-fullstack/hyracks/hyracks-api/pom.xml
index f5565fb..ddba2d8 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-api/pom.xml
@@ -102,9 +102,5 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
</dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
</dependencies>
</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
index 8a91547..8f36fcd 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
@@ -63,7 +63,7 @@
public static final int FILE_IS_NOT_DIRECTORY = 27;
public static final int CANNOT_READ_FILE = 28;
public static final int UNIDENTIFIED_IO_ERROR_READING_FILE = 29;
- public static final int FILE_DOES_NOT_EXIST = 30;
+ public static final int FILE_DOES_NOT_EXISTS = 30;
public static final int UNIDENTIFIED_IO_ERROR_DELETING_DIR = 31;
public static final int RESULT_NO_RECORD = 32;
public static final int DUPLICATE_KEY = 33;
@@ -100,22 +100,6 @@
public static final int FLUSH_ON_CLOSED_WRITER = 64;
public static final int FAIL_ON_FAILED_WRITER = 65;
public static final int MISSED_FAIL_CALL = 66;
- public static final int CANNOT_CREATE_FILE = 67;
- public static final int NO_MAPPING_FOR_FILE_ID = 68;
- public static final int NO_MAPPING_FOR_FILENAME = 69;
- public static final int CANNOT_GET_NUMBER_OF_ELEMENT_FROM_INACTIVE_FILTER = 70;
- public static final int CANNOT_CREATE_BLOOM_FILTER_BUILDER_FOR_INACTIVE_FILTER = 71;
- public static final int CANNOT_CREATE_BLOOM_FILTER_WITH_NUMBER_OF_PAGES = 72;
- public static final int CANNOT_ADD_TUPLES_TO_DUMMY_BLOOM_FILTER = 73;
- public static final int CANNOT_CREATE_ACTIVE_BLOOM_FILTER = 74;
- public static final int CANNOT_DEACTIVATE_INACTIVE_BLOOM_FILTER = 75;
- public static final int CANNOT_DESTROY_ACTIVE_BLOOM_FILTER = 76;
- public static final int CANNOT_PURGE_ACTIVE_INDEX = 77;
- public static final int CANNOT_PURGE_ACTIVE_BLOOM_FILTER = 78;
- public static final int CANNOT_BULK_LOAD_NON_EMPTY_TREE = 79;
- public static final int CANNOT_CREATE_EXISTING_INDEX = 80;
- public static final int FILE_ALREADY_MAPPED = 81;
- public static final int FILE_ALREADY_EXISTS = 82;
// Compilation error codes.
public static final int RULECOLLECTION_NOT_INSTANCE_OF_LIST = 10000;
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
index b48ba14..12a138a 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
@@ -56,12 +56,12 @@
if (!(o instanceof FileReference)) {
return false;
}
- return path.equals(((FileReference) o).path) && dev.equals(((FileReference) o).dev);
+ return file.equals(((FileReference) o).file);
}
@Override
public int hashCode() {
- return path.hashCode();
+ return file.hashCode();
}
/**
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IFileHandle.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IFileHandle.java
index 768ab87..a24c667 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IFileHandle.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IFileHandle.java
@@ -18,8 +18,9 @@
*/
package org.apache.hyracks.api.io;
-@FunctionalInterface
+/**
+ * IFileHandle is an interface that exists only for Java compilers to perform static typing
+ * when handling file handle objects. Users must not implement this interface.
+ */
public interface IFileHandle {
-
- FileReference getFileReference();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
index b0cc07a..50ef4f0 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
@@ -59,7 +59,7 @@
public long getSize(IFileHandle fileHandle);
- public void deleteWorkspaceFiles() throws HyracksDataException;
+ public void deleteWorkspaceFiles();
/**
* @param ioDeviceId
@@ -79,7 +79,6 @@
/**
* Gets a file reference from an absolute path
- *
* @deprecated
* use getFileRef(int ioDeviceId, String path) instead
* @param path
@@ -91,7 +90,6 @@
/**
* Create a workspace file with the given prefix
- *
* @param prefix
* @return A FileReference for the created workspace file
* @throws HyracksDataException
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java
deleted file mode 100644
index f4c5114..0000000
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.api.util;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hyracks.api.exceptions.ErrorCode;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-
-/**
- * This util class takes care of creation and deletion of files and directories
- * and throws the appropriate error in case of failure.
- */
-public class IoUtil {
-
- private IoUtil() {
- }
-
- /**
- * Delete a file
- *
- * @param fileRef
- * the file to be deleted
- * @throws HyracksDataException
- * if the file doesn't exist or if it couldn't be deleted
- */
- public static void delete(FileReference fileRef) throws HyracksDataException {
- delete(fileRef.getFile());
- }
-
- /**
- * Delete a file
- *
- * @param file
- * the file to be deleted
- * @throws HyracksDataException
- * if the file doesn't exist or if it couldn't be deleted
- */
- public static void delete(File file) throws HyracksDataException {
- try {
- if (file.isDirectory()) {
- FileUtils.deleteDirectory(file);
- } else {
- Files.delete(file.toPath());
- }
- } catch (IOException e) {
- throw HyracksDataException.create(ErrorCode.CANNOT_DELETE_FILE, e, file.getAbsolutePath());
- }
- }
-
- /**
- * Create a file on disk
- *
- * @param fileRef
- * the file to create
- * @throws HyracksDataException
- * if the file already exists or if it couldn't be created
- */
- public static void create(FileReference fileRef) throws HyracksDataException {
- if (fileRef.getFile().exists()) {
- throw HyracksDataException.create(ErrorCode.FILE_ALREADY_EXISTS, fileRef.getAbsolutePath());
- }
- fileRef.getFile().getParentFile().mkdirs();
- try {
- if (!fileRef.getFile().createNewFile()) {
- throw HyracksDataException.create(ErrorCode.FILE_ALREADY_EXISTS, fileRef.getAbsolutePath());
- }
- } catch (IOException e) {
- throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_FILE, e, fileRef.getAbsolutePath());
- }
- }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
index 9f983a7..4bf069c 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
@@ -46,7 +46,7 @@
27 = File %1$s is not a directory
28 = User doesn't have read permissions on the file %1$s
29 = Unidentified IO error occurred while reading the file %1$s
-30 = File %1$s doesn't exist
+30 = File %1$s doesn't exists
31 = Unidentified IO error occurred while deleting the dir %1$s
32 = No record for partition %1$s of result set %2$s
33 = Inserting duplicate keys into the primary storage
@@ -83,21 +83,5 @@
64 = Data pipeline protocol violation: flush() is called on a closed writer
65 = Data pipeline protocol violation: fail() is called twice on a writer
66 = Data pipeline protocol violation: fail() is not called by the upstream when there is a failure in the downstream
-67 = Cannot create the file: %1$s
-68 = No mapping found for file id %1$s
-69 = No mapping found for filename %1$s
-70 = Cannot get the number of elements from inactive bloom filter
-71 = Failed to create the bloom filter builder since it is not active
-72 = Cannot create a bloom filter with number of pages = %1$s
-73 = Cannot add elements to this filter since it is supposed to be empty (number of elements hint passed to the filter during construction was 0)
-74 = Failed to create the bloom filter since it is active
-75 = Failed to deactivate the bloom filter since it is inactive
-76 = Failed to destroy the bloom filter since it is active
-77 = Failed to purge the index since it is active
-78 = Failed to purge the bloom filter since it is active
-79 = Cannot bulk-load a non-empty tree
-80 = Cannot create index because it already exists
-81 = File %1$s is already mapped
-82 = Failed to create the file %1$s because it already exists
10000 = The given rule collection %1$s is not an instance of the List class.
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/FileHandle.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/FileHandle.java
index f71dcdf..594b2c3 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/FileHandle.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/FileHandle.java
@@ -22,8 +22,6 @@
import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
-import org.apache.hyracks.api.exceptions.ErrorCode;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IFileHandle;
import org.apache.hyracks.api.io.IIOManager;
@@ -39,23 +37,15 @@
this.fileRef = fileRef;
}
- /**
- * Open the file
- *
- * @param rwMode
- * @param syncMode
- * @throws IOException
- */
public void open(IIOManager.FileReadWriteMode rwMode, IIOManager.FileSyncMode syncMode) throws IOException {
String mode;
- if (!fileRef.getFile().exists()) {
- throw HyracksDataException.create(ErrorCode.FILE_DOES_NOT_EXIST, fileRef.getAbsolutePath());
- }
switch (rwMode) {
case READ_ONLY:
mode = "r";
break;
+
case READ_WRITE:
+ fileRef.getFile().getAbsoluteFile().getParentFile().mkdirs();
switch (syncMode) {
case METADATA_ASYNC_DATA_ASYNC:
mode = "rw";
@@ -88,7 +78,6 @@
raf = null;
}
- @Override
public FileReference getFileReference() {
return fileRef;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index 92831f4..722ff9e 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -38,7 +38,6 @@
import org.apache.hyracks.api.io.IIOFuture;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.io.IODeviceHandle;
-import org.apache.hyracks.api.util.IoUtil;
public class IOManager implements IIOManager {
/*
@@ -131,7 +130,8 @@
while (remaining > 0) {
int len = ((FileHandle) fHandle).getFileChannel().write(data, offset);
if (len < 0) {
- throw new HyracksDataException("Error writing to file: " + fHandle.getFileReference().toString());
+ throw new HyracksDataException(
+ "Error writing to file: " + ((FileHandle) fHandle).getFileReference().toString());
}
remaining -= len;
offset += len;
@@ -164,7 +164,8 @@
len = fileChannel.write(dataArray);
}
if (len < 0) {
- throw new HyracksDataException("Error writing to file: " + fHandle.getFileReference().toString());
+ throw new HyracksDataException(
+ "Error writing to file: " + ((FileHandle) fHandle).getFileReference().toString());
}
remaining -= len;
offset += len;
@@ -332,17 +333,17 @@
@Override
public long getSize(IFileHandle fileHandle) {
- return fileHandle.getFileReference().getFile().length();
+ return ((FileHandle) fileHandle).getFileReference().getFile().length();
}
@Override
- public void deleteWorkspaceFiles() throws HyracksDataException {
+ public void deleteWorkspaceFiles() {
for (IODeviceHandle ioDevice : workspaces) {
File workspaceFolder = new File(ioDevice.getMount(), ioDevice.getWorkspace());
if (workspaceFolder.exists() && workspaceFolder.isDirectory()) {
File[] workspaceFiles = workspaceFolder.listFiles(WORKSPACE_FILES_FILTER);
for (File workspaceFile : workspaceFiles) {
- IoUtil.delete(workspaceFile);
+ workspaceFile.delete();
}
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/BTreeHelperStorageManager.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/BTreeHelperStorageManager.java
index 3d33ca1..73ac7a8 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/BTreeHelperStorageManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/BTreeHelperStorageManager.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.storage.common.IResourceLifecycleManager;
import org.apache.hyracks.storage.common.IStorageManager;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.storage.common.file.ResourceIdFactory;
public class BTreeHelperStorageManager implements IStorageManager {
@@ -41,6 +42,11 @@
}
@Override
+ public IFileMapProvider getFileMapProvider(INCServiceContext ctx) {
+ return RuntimeContext.get(ctx).getFileMapManager();
+ }
+
+ @Override
public ILocalResourceRepository getLocalResourceRepository(INCServiceContext ctx) {
return RuntimeContext.get(ctx).getLocalResourceRepository();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/RuntimeContext.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/RuntimeContext.java
index 32f8418..a2175ee 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/RuntimeContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/org/apache/hyracks/examples/btree/helper/RuntimeContext.java
@@ -39,7 +39,7 @@
import org.apache.hyracks.storage.common.file.ILocalResourceRepositoryFactory;
import org.apache.hyracks.storage.common.file.ResourceIdFactory;
import org.apache.hyracks.storage.common.file.ResourceIdFactoryProvider;
-import org.apache.hyracks.storage.common.file.FileMapManager;
+import org.apache.hyracks.storage.common.file.TransientFileMapManager;
import org.apache.hyracks.storage.common.file.TransientLocalResourceRepositoryFactory;
public class RuntimeContext {
@@ -56,7 +56,7 @@
};
public RuntimeContext(INCServiceContext appCtx) throws HyracksDataException {
- fileMapManager = new FileMapManager();
+ fileMapManager = new TransientFileMapManager();
ICacheMemoryAllocator allocator = new HeapBufferAllocator();
IPageReplacementStrategy prs = new ClockPageReplacementStrategy(allocator, 32768, 50);
bufferCache = new BufferCache(appCtx.getIoManager(), prs, new DelayPageCleanerPolicy(1000), fileMapManager, 100,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java
index 47a9734..3b49d08 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java
@@ -21,7 +21,6 @@
import java.nio.ByteBuffer;
-import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
@@ -30,6 +29,7 @@
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class BloomFilter {
@@ -40,6 +40,7 @@
private static final int NUM_BITS_OFFSET = NUM_ELEMENTS_OFFSET + 8; // 12
private final IBufferCache bufferCache;
+ private final IFileMapProvider fileMapProvider;
private final FileReference file;
private final int[] keyFields;
private int fileId = -1;
@@ -50,11 +51,13 @@
private long numElements;
private long numBits;
private final int numBitsPerPage;
- private static final byte[] ZERO_BUFFER = new byte[131072]; // 128kb
- private static final long SEED = 0L;
+ private final static byte[] ZERO_BUFFER = new byte[131072]; // 128kb
+ private final static long SEED = 0L;
- public BloomFilter(IBufferCache bufferCache, FileReference file, int[] keyFields) throws HyracksDataException {
+ public BloomFilter(IBufferCache bufferCache, IFileMapProvider fileMapProvider, FileReference file, int[] keyFields)
+ throws HyracksDataException {
this.bufferCache = bufferCache;
+ this.fileMapProvider = fileMapProvider;
this.file = file;
this.keyFields = keyFields;
this.numBitsPerPage = bufferCache.getPageSize() * Byte.SIZE;
@@ -77,7 +80,7 @@
public long getNumElements() throws HyracksDataException {
if (!isActivated) {
- throw HyracksDataException.create(ErrorCode.CANNOT_GET_NUMBER_OF_ELEMENT_FROM_INACTIVE_FILTER);
+ throw new HyracksDataException("The bloom filter is not activated.");
}
return numElements;
}
@@ -91,8 +94,8 @@
long hash = Math.abs((hashes[0] + i * hashes[1]) % numBits);
// we increment the page id by one, since the metadata page id of the filter is 0.
- ICachedPage page =
- bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, (int) (hash / numBitsPerPage) + 1), false);
+ ICachedPage page = bufferCache.pin(
+ BufferedFileHandle.getDiskPageId(fileId, (int) (hash / numBitsPerPage) + 1), false);
page.acquireReadLatch();
try {
ByteBuffer buffer = page.getBuffer();
@@ -112,22 +115,40 @@
return true;
}
+ private void prepareFile() throws HyracksDataException {
+ boolean fileIsMapped = false;
+ synchronized (fileMapProvider) {
+ fileIsMapped = fileMapProvider.isMapped(file);
+ if (!fileIsMapped) {
+ bufferCache.createFile(file);
+ }
+ fileId = fileMapProvider.lookupFileId(file);
+ try {
+ // Also creates the file if it doesn't exist yet.
+ bufferCache.openFile(fileId);
+ } catch (HyracksDataException e) {
+ // Revert state of buffer cache since file failed to open.
+ if (!fileIsMapped) {
+ bufferCache.deleteFile(fileId, false);
+ }
+ throw e;
+ }
+ }
+ }
+
public synchronized void create() throws HyracksDataException {
if (isActivated) {
- throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_ACTIVE_BLOOM_FILTER);
+ throw new HyracksDataException("Failed to create the bloom filter since it is activated.");
}
- fileId = bufferCache.createFile(file);
+ prepareFile();
+ bufferCache.closeFile(fileId);
}
public synchronized void activate() throws HyracksDataException {
if (isActivated) {
return;
}
- if (fileId >= 0) {
- bufferCache.openFile(fileId);
- } else {
- fileId = bufferCache.openFile(file);
- }
+ prepareFile();
readBloomFilterMetaData();
isActivated = true;
}
@@ -155,26 +176,23 @@
public synchronized void deactivate() throws HyracksDataException {
if (!isActivated) {
- throw HyracksDataException.create(ErrorCode.CANNOT_DEACTIVATE_INACTIVE_BLOOM_FILTER);
+ return;
}
bufferCache.closeFile(fileId);
isActivated = false;
}
- public void purge() throws HyracksDataException {
- if (isActivated) {
- throw HyracksDataException.create(ErrorCode.CANNOT_PURGE_ACTIVE_BLOOM_FILTER);
- }
- bufferCache.purgeHandle(fileId);
- // after purging, the fileId has no mapping and no meaning
- fileId = -1;
- }
-
public synchronized void destroy() throws HyracksDataException {
if (isActivated) {
- throw HyracksDataException.create(ErrorCode.CANNOT_DESTROY_ACTIVE_BLOOM_FILTER);
+ throw new HyracksDataException("Failed to destroy the bloom filter since it is activated.");
}
- bufferCache.deleteFile(file);
+
+ file.delete();
+ if (fileId == -1) {
+ return;
+ }
+ bufferCache.deleteFile(fileId, false);
+ fileId = -1;
}
public IIndexBulkLoader createBuilder(long numElements, int numHashes, int numBitsPerElement)
@@ -194,7 +212,7 @@
public BloomFilterBuilder(long numElements, int numHashes, int numBitsPerElement) throws HyracksDataException {
if (!isActivated) {
- throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_BLOOM_FILTER_BUILDER_FOR_INACTIVE_FILTER);
+ throw new HyracksDataException("Failed to create the bloom filter builder since it is not activated.");
}
queue = bufferCache.createFIFOQueue();
this.numElements = numElements;
@@ -202,7 +220,7 @@
numBits = this.numElements * numBitsPerElement;
long tmp = (long) Math.ceil(numBits / (double) numBitsPerPage);
if (tmp > Integer.MAX_VALUE) {
- throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_BLOOM_FILTER_WITH_NUMBER_OF_PAGES, tmp);
+ throw new HyracksDataException("Cannot create a bloom filter with his huge number of pages.");
}
numPages = (int) tmp;
pages = new ICachedPage[numPages];
@@ -241,12 +259,13 @@
@Override
public void add(ITupleReference tuple) throws HyracksDataException {
if (numPages == 0) {
- throw HyracksDataException.create(ErrorCode.CANNOT_ADD_TUPLES_TO_DUMMY_BLOOM_FILTER);
+ throw new HyracksDataException(
+ "Cannot add elements to this filter since it is supposed to be empty (number of elements hint passed to the filter during construction was 0).");
}
MurmurHash128Bit.hash3_x64_128(tuple, keyFields, SEED, hashes);
for (int i = 0; i < numHashes; ++i) {
long hash = Math.abs((hashes[0] + i * hashes[1]) % numBits);
- ICachedPage page = pages[(int) (hash / numBitsPerPage)];
+ ICachedPage page = pages[((int) (hash / numBitsPerPage))];
ByteBuffer buffer = page.getBuffer();
int byteIndex = (int) (hash % numBitsPerPage) >> 3; // divide by 8
byte b = buffer.get(byteIndex);
@@ -278,8 +297,8 @@
bufferCache.returnPage(p, false);
}
}
- if (metaDataPage != null) {
- bufferCache.returnPage(metaDataPage, false);
+ if (metaDataPage != null ){
+ bufferCache.returnPage(metaDataPage,false);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilterFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilterFactory.java
index 8d01bb7..c3a1718 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilterFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilterFactory.java
@@ -22,18 +22,21 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class BloomFilterFactory {
private final IBufferCache bufferCache;
+ private final IFileMapProvider fileMapProvider;
private final int[] bloomFilterKeyFields;
- public BloomFilterFactory(IBufferCache bufferCache, int[] bloomFilterKeyFields) {
+ public BloomFilterFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider, int[] bloomFilterKeyFields) {
this.bufferCache = bufferCache;
+ this.fileMapProvider = fileMapProvider;
this.bloomFilterKeyFields = bloomFilterKeyFields;
}
public BloomFilter createBloomFiltertInstance(FileReference file) throws HyracksDataException {
- return new BloomFilter(bufferCache, file, bloomFilterKeyFields);
+ return new BloomFilter(bufferCache, fileMapProvider, file, bloomFilterKeyFields);
}
public int[] getBloomFilterKeyFields() {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeResource.java
index 24d2a37..90bc126 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeResource.java
@@ -55,8 +55,9 @@
IBufferCache bufferCache = storageManager.getBufferCache(ctx);
IIOManager ioManager = ctx.getIoManager();
FileReference resourceRef = ioManager.resolve(path);
- return BTreeUtils.createBTree(bufferCache, typeTraits, comparatorFactories, BTreeLeafFrameType.REGULAR_NSM,
- resourceRef, pageManagerFactory.createPageManager(bufferCache));
+ return BTreeUtils.createBTree(bufferCache, storageManager.getFileMapProvider(ctx), typeTraits,
+ comparatorFactories, BTreeLeafFrameType.REGULAR_NSM, resourceRef,
+ pageManagerFactory.createPageManager(bufferCache));
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
index ce35fbf..7f57524 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
@@ -63,6 +63,7 @@
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class BTree extends AbstractTreeIndex {
@@ -76,10 +77,11 @@
private final ReadWriteLock treeLatch;
private final int maxTupleSize;
- public BTree(IBufferCache bufferCache, IPageManager freePageManager, ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount,
- FileReference file) {
- super(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, file);
+ public BTree(IBufferCache bufferCache, IFileMapProvider fileMapProvider, IPageManager freePageManager,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
+ IBinaryComparatorFactory[] cmpFactories, int fieldCount, FileReference file) {
+ super(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
+ fieldCount, file);
this.treeLatch = new ReentrantReadWriteLock(true);
this.smoCounter = new AtomicInteger();
ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
@@ -94,11 +96,11 @@
RangePredicate diskOrderScanPred = new RangePredicate(null, null, true, true, ctx.getCmp(), ctx.getCmp());
int maxPageId = freePageManager.getMaxPageId(ctx.getMetaFrame());
int currentPageId = bulkloadLeafStart;
- ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), currentPageId), false);
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false);
page.acquireReadLatch();
try {
cursor.setBufferCache(bufferCache);
- cursor.setFileId(getFileId());
+ cursor.setFileId(fileId);
cursor.setCurrentPageId(currentPageId);
cursor.setMaxPageId(maxPageId);
ctx.getCursorInitialState().setPage(page);
@@ -127,7 +129,7 @@
}
private void validate(BTreeOpContext ctx, int pageId) throws HyracksDataException {
- ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), pageId), false);
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
ctx.getInteriorFrame().setPage(page);
PageValidationInfo currentPvi = ctx.getValidationInfos().peekFirst();
@@ -204,14 +206,14 @@
repeatOp = false;
}
cursor.setBufferCache(bufferCache);
- cursor.setFileId(getFileId());
+ cursor.setFileId(fileId);
}
private void unsetSmPages(BTreeOpContext ctx) throws HyracksDataException {
ICachedPage originalPage = ctx.getInteriorFrame().getPage();
for (int i = 0; i < ctx.getSmPages().size(); i++) {
int pageId = ctx.getSmPages().get(i);
- ICachedPage smPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), pageId), false);
+ ICachedPage smPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
smPage.acquireWriteLatch();
try {
ctx.getInteriorFrame().setPage(smPage);
@@ -236,11 +238,11 @@
private void createNewRoot(BTreeOpContext ctx) throws HyracksDataException {
// Make sure the root is always in the same page.
ICachedPage leftNode =
- bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), ctx.getSplitKey().getLeftPage()), false);
+ bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, ctx.getSplitKey().getLeftPage()), false);
leftNode.acquireWriteLatch();
try {
int newLeftId = freePageManager.takePage(ctx.getMetaFrame());
- ICachedPage newLeftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), newLeftId), true);
+ ICachedPage newLeftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, newLeftId), true);
newLeftNode.acquireWriteLatch();
try {
boolean largePage = false;
@@ -348,7 +350,7 @@
}
}
int rightPageId = freePageManager.takePage(ctx.getMetaFrame());
- ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), rightPageId), true);
+ ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true);
rightNode.acquireWriteLatch();
try {
IBTreeLeafFrame rightFrame = ctx.createLeafFrame();
@@ -464,8 +466,7 @@
switch (spaceStatus) {
case INSUFFICIENT_SPACE: {
int rightPageId = freePageManager.takePage(ctx.getMetaFrame());
- ICachedPage rightNode =
- bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), rightPageId), true);
+ ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true);
rightNode.acquireWriteLatch();
try {
IBTreeFrame rightFrame = ctx.createInteriorFrame();
@@ -544,7 +545,7 @@
}
private ICachedPage isConsistent(int pageId, BTreeOpContext ctx) throws Exception {
- ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), pageId), false);
+ ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
node.acquireReadLatch();
ctx.getInteriorFrame().setPage(node);
boolean isConsistent = ctx.getPageLsns().getLast() == ctx.getInteriorFrame().getPageLsn();
@@ -558,7 +559,7 @@
private void performOp(int pageId, ICachedPage parent, boolean parentIsReadLatched, BTreeOpContext ctx)
throws HyracksDataException {
- ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), pageId), false);
+ ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
ctx.getInteriorFrame().setPage(node);
// this check performs an unprotected read in the page
// the following could happen: TODO fill out
@@ -619,8 +620,8 @@
case UPDATE: {
// Is there a propagated split key?
if (ctx.getSplitKey().getBuffer() != null) {
- ICachedPage interiorNode = bufferCache
- .pin(BufferedFileHandle.getDiskPageId(getFileId(), pageId), false);
+ ICachedPage interiorNode =
+ bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
interiorNode.acquireWriteLatch();
try {
// Insert or update op. Both can cause split keys to propagate upwards.
@@ -768,7 +769,7 @@
public void printTree(int pageId, ICachedPage parent, boolean unpin, IBTreeLeafFrame leafFrame,
IBTreeInteriorFrame interiorFrame, byte treeHeight, ISerializerDeserializer[] keySerdes,
StringBuilder strBuilder, MultiComparator cmp) throws Exception {
- ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), pageId), false);
+ ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
node.acquireReadLatch();
try {
if (parent != null && unpin == true) {
@@ -1036,7 +1037,7 @@
splitKey.setRightPage(leafFrontier.pageId);
}
if (tupleSize > maxTupleSize) {
- final long dpid = BufferedFileHandle.getDiskPageId(getFileId(), leafFrontier.pageId);
+ final long dpid = BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId);
// calculate required number of pages.
int headerSize = Math.max(leafFrame.getPageHeaderSize(), interiorFrame.getPageHeaderSize());
final int multiplier =
@@ -1051,7 +1052,7 @@
leafFrame.initBuffer((byte) 0);
((IBTreeLeafFrame) leafFrame).setLargeFlag(true);
} else {
- final long dpid = BufferedFileHandle.getDiskPageId(getFileId(), leafFrontier.pageId);
+ final long dpid = BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId);
leafFrontier.page = bufferCache.confiscatePage(dpid);
leafFrame.setPage(leafFrontier.page);
leafFrame.initBuffer((byte) 0);
@@ -1115,7 +1116,7 @@
((IBTreeInteriorFrame) interiorFrame).deleteGreatest();
int finalPageId = freePageManager.takePage(metaFrame);
- bufferCache.setPageDiskId(frontier.page, BufferedFileHandle.getDiskPageId(getFileId(), finalPageId));
+ bufferCache.setPageDiskId(frontier.page, BufferedFileHandle.getDiskPageId(fileId, finalPageId));
pagesToWrite.add(frontier.page);
splitKey.setLeftPage(finalPageId);
@@ -1175,7 +1176,7 @@
}
private void setPageDpid(ICachedPage page, int pageId) {
- bufferCache.setPageDiskId(page, BufferedFileHandle.getDiskPageId(getFileId(), pageId));
+ bufferCache.setPageDiskId(page, BufferedFileHandle.getDiskPageId(fileId, pageId));
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
index 303777e..5cc9599 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
@@ -35,26 +35,29 @@
import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class BTreeUtils {
- public static BTree createBTree(IBufferCache bufferCache, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] cmpFactories, BTreeLeafFrameType leafType, FileReference file,
- IPageManager freePageManager) throws HyracksDataException {
+ public static BTree createBTree(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, BTreeLeafFrameType leafType,
+ FileReference file, IPageManager freePageManager) throws HyracksDataException {
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- return new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
- typeTraits.length, file);
+ BTree btree = new BTree(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory,
+ cmpFactories, typeTraits.length, file);
+ return btree;
}
- public static BTree createBTree(IBufferCache bufferCache, IPageManager freePageManager, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] cmpFactories, BTreeLeafFrameType leafType, FileReference file)
- throws HyracksDataException {
+ public static BTree createBTree(IBufferCache bufferCache, IPageManager freePageManager,
+ IFileMapProvider fileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
+ BTreeLeafFrameType leafType, FileReference file) throws HyracksDataException {
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- return new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
- typeTraits.length, file);
+ BTree btree = new BTree(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory,
+ cmpFactories, typeTraits.length, file);
+ return btree;
}
// Creates a new MultiComparator by constructing new IBinaryComparators.
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java
index 9f648c5..f1ae6b0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java
@@ -19,13 +19,10 @@
package org.apache.hyracks.storage.am.common.build;
import java.io.IOException;
-import java.util.logging.Level;
-import java.util.logging.Logger;
import org.apache.hyracks.api.application.INCServiceContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.util.IoUtil;
import org.apache.hyracks.storage.am.common.api.IIndexBuilder;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import org.apache.hyracks.storage.common.IIndex;
@@ -38,7 +35,6 @@
import org.apache.hyracks.storage.common.file.IResourceIdFactory;
public class IndexBuilder implements IIndexBuilder {
- private static final Logger LOGGER = Logger.getLogger(IndexBuilder.class.getName());
protected final INCServiceContext ctx;
protected final IStorageManager storageManager;
@@ -85,22 +81,8 @@
//The reason for this is to handle many cases such as:
//1. Crash while delete index is running (we don't do global cleanup on restart)
//2. Node leaves and then join with old data
- LOGGER.log(Level.WARNING,
- "Removing existing index on index create for the index: " + resourceRef.getRelativePath());
lcManager.unregister(resourceRef.getRelativePath());
- index.destroy();
} else {
- if (resourceRef.getFile().exists()) {
- // Index is not registered but the index file exists
- // This is another big problem that we need to disallow soon
- // We can only disallow this if we have a global cleanup after crash
- // on reboot
- LOGGER.log(Level.WARNING,
- "Deleting " + resourceRef.getRelativePath()
- + " on index create. The index is not registered"
- + " but the file exists in the filesystem");
- IoUtil.delete(resourceRef);
- }
index = resource.createInstance(ctx);
}
index.create();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
index cc3bfe7..c00cecb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
@@ -37,6 +37,7 @@
import org.apache.hyracks.storage.common.IStorageManager;
import org.apache.hyracks.storage.common.LocalResource;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class TreeIndexStatsOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable {
private final IHyracksTaskContext ctx;
@@ -68,11 +69,14 @@
try {
writer.open();
IBufferCache bufferCache = storageManager.getBufferCache(ctx.getJobletContext().getServiceContext());
+ IFileMapProvider fileMapProvider =
+ storageManager.getFileMapProvider(ctx.getJobletContext().getServiceContext());
LocalResource resource = treeIndexHelper.getResource();
IIOManager ioManager = ctx.getIoManager();
FileReference fileRef = ioManager.resolve(resource.getPath());
+ int indexFileId = fileMapProvider.lookupFileId(fileRef);
TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, treeIndex.getPageManager(),
- fileRef, treeIndex.getRootPageId());
+ indexFileId, treeIndex.getRootPageId());
TreeIndexStats stats = statsGatherer.gatherStats(treeIndex.getLeafFrameFactory().createFrame(),
treeIndex.getInteriorFrameFactory().createFrame(),
treeIndex.getPageManager().createMetadataFrame());
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
index 8ffd0eb..163a3d1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
@@ -40,6 +40,7 @@
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public abstract class AbstractTreeIndex implements ITreeIndex {
@@ -48,6 +49,7 @@
protected int rootPage = 1;
protected final IBufferCache bufferCache;
+ protected final IFileMapProvider fileMapProvider;
protected final IPageManager freePageManager;
protected final ITreeIndexFrameFactory interiorFrameFactory;
@@ -57,16 +59,21 @@
protected final int fieldCount;
protected FileReference file;
- private int fileId = -1;
+ protected int fileId = -1;
protected boolean isActive = false;
+ //hasEverBeenActivated is to stop the throwing of an exception of deactivating an index that
+ //was never activated or failed to activate in try/finally blocks, as there's no way to know if
+ //an index is activated or not from the outside.
+ protected boolean hasEverBeenActivated = false;
protected int bulkloadLeafStart = 0;
- public AbstractTreeIndex(IBufferCache bufferCache, IPageManager freePageManager,
+ public AbstractTreeIndex(IBufferCache bufferCache, IFileMapProvider fileMapProvider, IPageManager freePageManager,
ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
IBinaryComparatorFactory[] cmpFactories, int fieldCount, FileReference file) {
this.bufferCache = bufferCache;
+ this.fileMapProvider = fileMapProvider;
this.freePageManager = freePageManager;
this.interiorFrameFactory = interiorFrameFactory;
this.leafFrameFactory = leafFrameFactory;
@@ -80,8 +87,9 @@
if (isActive) {
throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_ACTIVE_INDEX);
}
- fileId = bufferCache.createFile(file);
- bufferCache.openFile(fileId);
+ synchronized (fileMapProvider) {
+ fileId = createAndOpen(bufferCache, fileMapProvider, file);
+ }
freePageManager.open(fileId);
freePageManager.init(interiorFrameFactory, leafFrameFactory);
setRootPage();
@@ -89,6 +97,27 @@
bufferCache.closeFile(fileId);
}
+ public static int createAndOpen(IBufferCache bufferCache, IFileMapProvider fileMapProvider, FileReference file)
+ throws HyracksDataException {
+ int fileId;
+ boolean fileIsMapped = fileMapProvider.isMapped(file);
+ if (!fileIsMapped) {
+ bufferCache.createFile(file);
+ }
+ fileId = fileMapProvider.lookupFileId(file);
+ try {
+ // Also creates the file if it doesn't exist yet.
+ bufferCache.openFile(fileId);
+ } catch (HyracksDataException e) {
+ // Revert state of buffer cache since file failed to open.
+ if (!fileIsMapped) {
+ bufferCache.deleteFile(fileId, false);
+ }
+ throw e;
+ }
+ return fileId;
+ }
+
private void setRootPage() throws HyracksDataException {
rootPage = freePageManager.getRootPageId();
bulkloadLeafStart = freePageManager.getBulkLoadLeaf();
@@ -99,35 +128,49 @@
if (isActive) {
throw HyracksDataException.create(ErrorCode.CANNOT_ACTIVATE_ACTIVE_INDEX);
}
- if (fileId >= 0) {
- bufferCache.openFile(fileId);
- } else {
- fileId = bufferCache.openFile(file);
+ boolean fileIsMapped = false;
+ synchronized (fileMapProvider) {
+ fileIsMapped = fileMapProvider.isMapped(file);
+ if (!fileIsMapped) {
+ bufferCache.createFile(file);
+ }
+ fileId = fileMapProvider.lookupFileId(file);
+ try {
+ // Also creates the file if it doesn't exist yet.
+ bufferCache.openFile(fileId);
+ } catch (HyracksDataException e) {
+ // Revert state of buffer cache since file failed to open.
+ if (!fileIsMapped) {
+ bufferCache.deleteFile(fileId, false);
+ }
+ throw e;
+ }
}
freePageManager.open(fileId);
setRootPage();
// TODO: Should probably have some way to check that the tree is physically consistent
// or that the file we just opened actually is a tree
isActive = true;
+ hasEverBeenActivated = true;
}
@Override
public synchronized void deactivate() throws HyracksDataException {
- if (!isActive) {
+ if (!isActive && hasEverBeenActivated) {
throw HyracksDataException.create(ErrorCode.CANNOT_DEACTIVATE_INACTIVE_INDEX);
}
- freePageManager.close();
- bufferCache.closeFile(fileId);
+ if (isActive) {
+ freePageManager.close();
+ bufferCache.closeFile(fileId);
+ }
+
isActive = false;
}
- public void purge() throws HyracksDataException {
- if (isActive) {
- throw HyracksDataException.create(ErrorCode.CANNOT_PURGE_ACTIVE_INDEX);
- }
+ public synchronized void deactivateCloseHandle() throws HyracksDataException {
+ deactivate();
bufferCache.purgeHandle(fileId);
- // after purging, the fileId has no mapping and no meaning
- fileId = -1;
+
}
@Override
@@ -135,7 +178,13 @@
if (isActive) {
throw HyracksDataException.create(ErrorCode.CANNOT_DESTROY_ACTIVE_INDEX);
}
- bufferCache.deleteFile(file);
+
+ if (fileId == -1) {
+ return;
+ }
+ bufferCache.deleteFile(fileId, false);
+ file.delete();
+ fileId = -1;
}
@Override
@@ -235,7 +284,7 @@
queue = bufferCache.createFIFOQueue();
if (!isEmptyTree(leafFrame)) {
- throw HyracksDataException.create(ErrorCode.CANNOT_BULK_LOAD_NON_EMPTY_TREE);
+ throw new HyracksDataException("Cannot bulk-load a non-empty tree.");
}
this.cmp = MultiComparator.create(cmpFactories);
@@ -263,6 +312,9 @@
pagesToWrite = new ArrayList<>();
}
+ @Override
+ public abstract void add(ITupleReference tuple) throws HyracksDataException;
+
protected void handleException() throws HyracksDataException {
// Unlatch and unpin pages that weren't in the queue to avoid leaking memory.
for (NodeFrontier nodeFrontier : nodeFrontiers) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
index 959f27a..2b47238 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
@@ -22,7 +22,7 @@
import java.util.Random;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.util.MathUtil;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
@@ -30,25 +30,25 @@
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
-import org.apache.hyracks.util.MathUtil;
public class TreeIndexBufferCacheWarmup {
private final IBufferCache bufferCache;
private final IMetadataPageManager freePageManager;
- private final FileReference fileRef;
+ private final int fileId;
private final ArrayList<IntArrayList> pagesByLevel = new ArrayList<>();
private final Random rnd = new Random();
- public TreeIndexBufferCacheWarmup(IBufferCache bufferCache, IMetadataPageManager freePageManager,
- FileReference fileRef) {
+ public TreeIndexBufferCacheWarmup(IBufferCache bufferCache,
+ IMetadataPageManager freePageManager, int fileId) {
this.bufferCache = bufferCache;
this.freePageManager = freePageManager;
- this.fileRef = fileRef;
+ this.fileId = fileId;
}
- public void warmup(ITreeIndexFrame frame, ITreeIndexMetadataFrame metaFrame, int[] warmupTreeLevels,
+ public void warmup(ITreeIndexFrame frame,
+ ITreeIndexMetadataFrame metaFrame, int[] warmupTreeLevels,
int[] warmupRepeats) throws HyracksDataException {
- int fileId = bufferCache.openFile(fileRef);
+ bufferCache.openFile(fileId);
// scan entire file to determine pages in each level
int maxPageId = freePageManager.getMaxPageId(metaFrame);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
index 5410282..7e14b4a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.common.util;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.storage.am.common.api.IPageManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
@@ -32,21 +31,25 @@
private final TreeIndexStats treeIndexStats = new TreeIndexStats();
private final IBufferCache bufferCache;
private final IPageManager freePageManager;
- private final FileReference fileRef;
+ private final int fileId;
private final int rootPage;
- public TreeIndexStatsGatherer(IBufferCache bufferCache, IPageManager freePageManager, FileReference fileRef,
- int rootPage) {
+ public TreeIndexStatsGatherer(IBufferCache bufferCache,
+ IPageManager freePageManager, int fileId, int rootPage) {
this.bufferCache = bufferCache;
this.freePageManager = freePageManager;
- this.fileRef = fileRef;
+ this.fileId = fileId;
this.rootPage = rootPage;
}
- public TreeIndexStats gatherStats(ITreeIndexFrame leafFrame, ITreeIndexFrame interiorFrame,
- ITreeIndexMetadataFrame metaFrame) throws HyracksDataException {
- int fileId = bufferCache.openFile(fileRef);
+ public TreeIndexStats gatherStats(ITreeIndexFrame leafFrame,
+ ITreeIndexFrame interiorFrame, ITreeIndexMetadataFrame metaFrame)
+ throws HyracksDataException {
+
+ bufferCache.openFile(fileId);
+
treeIndexStats.begin();
+
int maxPageId = freePageManager.getMaxPageId(metaFrame);
for (int pageId = 0; pageId <= maxPageId; pageId++) {
ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java
index f1e9cf1..6860b971 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java
@@ -57,9 +57,9 @@
public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException {
IIOManager ioManager = serviceCtx.getIoManager();
FileReference file = ioManager.resolve(path);
- return LSMBTreeUtil.createExternalBTree(ioManager, file, storageManager.getBufferCache(serviceCtx), typeTraits,
- cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate,
- mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
+ return LSMBTreeUtil.createExternalBTree(ioManager, file, storageManager.getBufferCache(serviceCtx),
+ storageManager.getFileMapProvider(serviceCtx), typeTraits, cmpFactories, bloomFilterKeyFields,
+ bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
opTrackerProvider.getOperationTracker(serviceCtx), ioSchedulerProvider.getIoScheduler(serviceCtx),
ioOpCallbackFactory.createIoOpCallback(), durable, metadataPageManagerFactory);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java
index 317a479..3819066 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java
@@ -61,7 +61,7 @@
IIOManager ioManager = serviceCtx.getIoManager();
FileReference file = ioManager.resolve(path);
return LSMBTreeUtil.createExternalBTreeWithBuddy(ioManager, file, storageManager.getBufferCache(serviceCtx),
- typeTraits, cmpFactories, bloomFilterFalsePositiveRate,
+ storageManager.getFileMapProvider(serviceCtx), typeTraits, cmpFactories, bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
opTrackerProvider.getOperationTracker(serviceCtx), ioSchedulerProvider.getIoScheduler(serviceCtx),
ioOpCallbackFactory.createIoOpCallback(), bloomFilterKeyFields, durable, metadataPageManagerFactory);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java
index f6e311c..6416f6b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java
@@ -70,9 +70,9 @@
IIOManager ioManager = serviceCtx.getIoManager();
FileReference file = ioManager.resolve(path);
List<IVirtualBufferCache> vbcs = vbcProvider.getVirtualBufferCaches(serviceCtx, file);
- return LSMBTreeUtil.createLSMTree(ioManager, vbcs, file, storageManager.getBufferCache(serviceCtx), typeTraits,
- cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate,
- mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
+ return LSMBTreeUtil.createLSMTree(ioManager, vbcs, file, storageManager.getBufferCache(serviceCtx),
+ storageManager.getFileMapProvider(serviceCtx), typeTraits, cmpFactories, bloomFilterKeyFields,
+ bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
opTrackerProvider.getOperationTracker(serviceCtx), ioSchedulerProvider.getIoScheduler(serviceCtx),
ioOpCallbackFactory.createIoOpCallback(), isPrimary, filterTypeTraits, filterCmpFactories, btreeFields,
filterFields, durable, metadataPageManagerFactory);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index c06edfd..cdde692 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -62,6 +62,7 @@
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
/**
* This is an lsm b-tree that does not have memory component and is modified
@@ -92,12 +93,13 @@
ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory,
ILSMIndexFileManager fileManager, TreeIndexFactory<BTree> diskBTreeFactory,
TreeIndexFactory<BTree> bulkLoadBTreeFactory, BloomFilterFactory bloomFilterFactory,
- double bloomFilterFalsePositiveRate, IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy,
- ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount,
+ IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
TreeIndexFactory<BTree> transactionBTreeFactory, boolean durable) {
super(ioManager, insertLeafFrameFactory, deleteLeafFrameFactory, fileManager, diskBTreeFactory,
- bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, cmpFactories, mergePolicy,
- opTracker, ioScheduler, ioOpCallback, false, durable);
+ bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, fieldCount,
+ cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, false, durable);
this.transactionComponentFactory =
new LSMBTreeDiskComponentFactory(transactionBTreeFactory, bloomFilterFactory, null);
this.secondDiskComponents = new LinkedList<>();
@@ -299,13 +301,21 @@
BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
cb.afterFinalize(LSMOperationType.FLUSH, null);
}
- for (ILSMDiskComponent c : diskComponents) {
- deactivateDiskComponent(c);
+ for (ILSMComponent c : diskComponents) {
+ LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+ BTree btree = component.getBTree();
+ BloomFilter bloomFilter = component.getBloomFilter();
+ btree.deactivateCloseHandle();
+ bloomFilter.deactivate();
}
- for (ILSMDiskComponent c : secondDiskComponents) {
- // Only deactivate non shared components
+ for (ILSMComponent c : secondDiskComponents) {
+ // Only deactivate non shared components (So components are not de-activated twice)
if (!diskComponents.contains(c)) {
- deactivateDiskComponent(c);
+ LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+ BTree btree = component.getBTree();
+ BloomFilter bloomFilter = component.getBloomFilter();
+ btree.deactivateCloseHandle();
+ bloomFilter.deactivate();
}
}
isActive = false;
@@ -347,13 +357,17 @@
if (isActive) {
throw new HyracksDataException("Failed to destroy the index since it is activated.");
}
- for (ILSMDiskComponent c : diskComponents) {
- destroyDiskComponent(c);
+ for (ILSMComponent c : diskComponents) {
+ LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+ component.getBTree().destroy();
+ component.getBloomFilter().destroy();
// Remove from second list to avoid destroying twice
secondDiskComponents.remove(c);
}
- for (ILSMDiskComponent c : secondDiskComponents) {
- destroyDiskComponent(c);
+ for (ILSMComponent c : secondDiskComponents) {
+ LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+ component.getBTree().destroy();
+ component.getBloomFilter().destroy();
}
diskComponents.clear();
secondDiskComponents.clear();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index 9959583..1c99b17 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -72,6 +72,7 @@
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeIndex, ITwoPCIndex {
@@ -99,12 +100,12 @@
IBufferCache diskBufferCache, ILSMIndexFileManager fileManager,
TreeIndexFactory<BTree> bulkLoadBTreeFactory, TreeIndexFactory<BTree> copyBtreeFactory,
TreeIndexFactory<BTree> buddyBtreeFactory, BloomFilterFactory bloomFilterFactory,
- double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
IBinaryComparatorFactory[] btreeCmpFactories, IBinaryComparatorFactory[] buddyBtreeCmpFactories,
int[] buddyBTreeFields, boolean durable) {
- super(ioManager, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
- ioScheduler, ioOpCallback, durable);
+ super(ioManager, diskBufferCache, fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy,
+ opTracker, ioScheduler, ioOpCallback, durable);
this.btreeCmpFactories = btreeCmpFactories;
this.buddyBtreeCmpFactories = buddyBtreeCmpFactories;
this.buddyBTreeFields = buddyBTreeFields;
@@ -120,7 +121,12 @@
@Override
public void create() throws HyracksDataException {
- super.create();
+ if (isActive) {
+ throw new HyracksDataException("Failed to create the index since it is activated.");
+ }
+ fileManager.deleteDirs();
+ fileManager.createDirs();
+ diskComponents.clear();
secondDiskComponents.clear();
}
@@ -767,12 +773,14 @@
BTree btree = component.getBTree();
BTree buddyBtree = component.getBuddyBTree();
BloomFilter bloomFilter = component.getBloomFilter();
- btree.deactivate();
- buddyBtree.deactivate();
+ btree.deactivateCloseHandle();
+ buddyBtree.deactivateCloseHandle();
bloomFilter.deactivate();
- btree.purge();
- buddyBtree.purge();
- bloomFilter.purge();
+ }
+
+ @Override
+ protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ //do nothing since external index never use memory components
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index a5c59d6..e830b3e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -76,6 +76,7 @@
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
@@ -102,11 +103,11 @@
TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory,
BloomFilterFactory bloomFilterFactory, IComponentFilterHelper filterHelper,
ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
- double bloomFilterFalsePositiveRate, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, int[] btreeFields, int[] filterFields,
- boolean durable) throws HyracksDataException {
- super(ioManager, virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager,
+ double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount,
+ IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck,
+ int[] btreeFields, int[] filterFields, boolean durable) throws HyracksDataException {
+ super(ioManager, virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
filterManager, filterFields, durable, filterHelper, btreeFields);
this.insertLeafFrameFactory = insertLeafFrameFactory;
@@ -115,7 +116,8 @@
int i = 0;
for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
LSMBTreeMemoryComponent mutableComponent = new LSMBTreeMemoryComponent(
- new BTree(virtualBufferCache, new VirtualFreePageManager(virtualBufferCache), interiorFrameFactory,
+ new BTree(virtualBufferCache, virtualBufferCache.getFileMapProvider(),
+ new VirtualFreePageManager(virtualBufferCache), interiorFrameFactory,
insertLeafFrameFactory, cmpFactories, fieldCount,
ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_" + i)),
virtualBufferCache, i == 0 ? true : false,
@@ -135,18 +137,19 @@
ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager,
TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory,
BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
- IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck,
- boolean durable) {
- super(ioManager, diskBTreeFactory.getBufferCache(), fileManager, bloomFilterFalsePositiveRate, mergePolicy,
- opTracker, ioScheduler, ioOpCallback, durable);
+ IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, boolean durable) {
+ super(ioManager, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
+ bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, durable);
this.insertLeafFrameFactory = insertLeafFrameFactory;
this.deleteLeafFrameFactory = deleteLeafFrameFactory;
this.cmpFactories = cmpFactories;
- this.needKeyDupCheck = needKeyDupCheck;
- this.hasBloomFilter = true;
componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, null);
bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory, null);
+ this.needKeyDupCheck = needKeyDupCheck;
+ //TODO remove BloomFilter from external dataset's secondary LSMBTree index
+ this.hasBloomFilter = true;
}
public boolean hasBloomFilter() {
@@ -180,6 +183,12 @@
}
@Override
+ protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+ mutableComponent.getBTree().destroy();
+ }
+
+ @Override
protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
if (hasBloomFilter) {
@@ -578,11 +587,9 @@
@Override
protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
- component.getBTree().deactivate();
- component.getBTree().purge();
+ component.getBTree().deactivateCloseHandle();
if (hasBloomFilter) {
component.getBloomFilter().deactivate();
- component.getBloomFilter().purge();
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
index 432a2b8..194eb3a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
@@ -37,15 +37,17 @@
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
+ public static final String BTREE_STRING = "b";
private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
private final boolean hasBloomFilter;
- public LSMBTreeFileManager(IIOManager ioManager, FileReference file,
+ public LSMBTreeFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
TreeIndexFactory<? extends ITreeIndex> btreeFactory, boolean hasBloomFilter) {
- super(ioManager, file, null);
+ super(ioManager, fileMapProvider, file, null);
this.btreeFactory = btreeFactory;
this.hasBloomFilter = hasBloomFilter;
}
@@ -53,28 +55,28 @@
@Override
public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException {
String ts = getCurrentTimestamp();
- String baseName = baseDir + ts + DELIMITER + ts;
+ String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), null,
- hasBloomFilter ? createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX) : null);
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), null,
+ hasBloomFilter ? createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING) : null);
}
@Override
public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName)
throws HyracksDataException {
- String[] firstTimestampRange = firstFileName.split(DELIMITER);
- String[] lastTimestampRange = lastFileName.split(DELIMITER);
+ String[] firstTimestampRange = firstFileName.split(SPLIT_STRING);
+ String[] lastTimestampRange = lastFileName.split(SPLIT_STRING);
- String baseName = baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1];
+ String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1];
// Get the range of timestamps by taking the earliest and the latest timestamps
- return new LSMComponentFileReferences(createMergeFile(baseName + DELIMITER + BTREE_SUFFIX), null,
- hasBloomFilter ? createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX) : null);
+ return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + BTREE_STRING), null,
+ hasBloomFilter ? createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING) : null);
}
private static FilenameFilter btreeFilter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
- return !name.startsWith(".") && name.endsWith(BTREE_SUFFIX);
+ return !name.startsWith(".") && name.endsWith(BTREE_STRING);
}
};
@@ -93,7 +95,7 @@
cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, btreeFilter), btreeFactory, allBTreeFiles);
HashSet<String> btreeFilesSet = new HashSet<>();
for (ComparableFileName cmpFileName : allBTreeFiles) {
- int index = cmpFileName.fileName.lastIndexOf(DELIMITER);
+ int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
}
@@ -198,19 +200,19 @@
public LSMComponentFileReferences getNewTransactionFileReference() throws IOException {
String ts = getCurrentTimestamp();
// Create transaction lock file
- Files.createFile(Paths.get(baseDir + TXN_PREFIX + ts));
+ Files.createFile(Paths.get(baseDir + TRANSACTION_PREFIX + ts));
- String baseName = baseDir + ts + DELIMITER + ts;
+ String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a transaction
- return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), null,
- createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), null,
+ createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING));
}
@Override
public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException {
FilenameFilter transactionFilter;
File dir = new File(baseDir);
- String[] files = dir.list(txnFileNameFilter);
+ String[] files = dir.list(transactionFileNameFilter);
if (files.length == 0) {
return null;
}
@@ -234,9 +236,9 @@
File bloomFilterFile = null;
for (String fileName : files) {
- if (fileName.endsWith(BTREE_SUFFIX)) {
+ if (fileName.endsWith(BTREE_STRING)) {
bTreeFile = new File(dir.getPath() + File.separator + fileName);
- } else if (fileName.endsWith(BLOOM_FILTER_SUFFIX)) {
+ } else if (fileName.endsWith(BLOOM_FILTER_STRING)) {
bloomFilterFile = new File(dir.getPath() + File.separator + fileName);
} else {
throw new HyracksDataException("unrecognized file found = " + fileName);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
index c866a45..0173c06 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
@@ -36,8 +36,11 @@
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager {
+ public static final String BUDDY_BTREE_STRING = "buddy";
+ public static final String BTREE_STRING = "b";
private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
private final TreeIndexFactory<? extends ITreeIndex> buddyBtreeFactory;
@@ -45,21 +48,21 @@
private static FilenameFilter btreeFilter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
- return !name.startsWith(".") && name.endsWith(BTREE_SUFFIX);
+ return !name.startsWith(".") && name.endsWith(BTREE_STRING);
}
};
private static FilenameFilter buddyBtreeFilter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
- return !name.startsWith(".") && name.endsWith(DELETE_TREE_SUFFIX);
+ return !name.startsWith(".") && name.endsWith(BUDDY_BTREE_STRING);
}
};
- public LSMBTreeWithBuddyFileManager(IIOManager ioManager, FileReference file,
+ public LSMBTreeWithBuddyFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
TreeIndexFactory<? extends ITreeIndex> btreeFactory,
TreeIndexFactory<? extends ITreeIndex> buddyBtreeFactory) {
- super(ioManager, file, null);
+ super(ioManager, fileMapProvider, file, null);
this.buddyBtreeFactory = buddyBtreeFactory;
this.btreeFactory = btreeFactory;
}
@@ -67,25 +70,25 @@
@Override
public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException {
String ts = getCurrentTimestamp();
- String baseName = baseDir + ts + DELIMITER + ts;
+ String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + DELETE_TREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING),
+ createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING),
+ createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING));
}
@Override
public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName)
throws HyracksDataException {
- String[] firstTimestampRange = firstFileName.split(DELIMITER);
- String[] lastTimestampRange = lastFileName.split(DELIMITER);
+ String[] firstTimestampRange = firstFileName.split(SPLIT_STRING);
+ String[] lastTimestampRange = lastFileName.split(SPLIT_STRING);
- String baseName = baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1];
+ String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1];
// Get the range of timestamps by taking the earliest and the latest
// timestamps
- return new LSMComponentFileReferences(createMergeFile(baseName + DELIMITER + BTREE_SUFFIX),
- createMergeFile(baseName + DELIMITER + DELETE_TREE_SUFFIX),
- createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + BTREE_STRING),
+ createMergeFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING),
+ createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING));
}
@Override
@@ -103,7 +106,7 @@
allBTreeFiles);
HashSet<String> btreeFilesSet = new HashSet<>();
for (ComparableFileName cmpFileName : allBTreeFiles) {
- int index = cmpFileName.fileName.lastIndexOf(DELIMITER);
+ int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
}
validateFiles(btreeFilesSet, allBuddyBTreeFiles, getCompoundFilter(buddyBtreeFilter, transactionFilefilter),
@@ -203,19 +206,19 @@
public LSMComponentFileReferences getNewTransactionFileReference() throws IOException {
String ts = getCurrentTimestamp();
// Create transaction lock file
- Files.createFile(Paths.get(baseDir + TXN_PREFIX + ts));
+ Files.createFile(Paths.get(baseDir + TRANSACTION_PREFIX + ts));
- String baseName = baseDir + ts + DELIMITER + ts;
- return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + DELETE_TREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ String baseName = baseDir + ts + SPLIT_STRING + ts;
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING),
+ createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING),
+ createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING));
}
@Override
public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException {
FilenameFilter transactionFilter;
File dir = new File(baseDir);
- String[] files = dir.list(txnFileNameFilter);
+ String[] files = dir.list(transactionFileNameFilter);
if (files.length == 0) {
return null;
}
@@ -240,11 +243,11 @@
File buddyBTreeFile = null;
File bloomFilterFile = null;
for (String fileName : files) {
- if (fileName.endsWith(BTREE_SUFFIX)) {
+ if (fileName.endsWith(BTREE_STRING)) {
bTreeFile = new File(dir.getPath() + File.separator + fileName);
- } else if (fileName.endsWith(DELETE_TREE_SUFFIX)) {
+ } else if (fileName.endsWith(BUDDY_BTREE_STRING)) {
buddyBTreeFile = new File(dir.getPath() + File.separator + fileName);
- } else if (fileName.endsWith(BLOOM_FILTER_SUFFIX)) {
+ } else if (fileName.endsWith(BLOOM_FILTER_STRING)) {
bloomFilterFile = new File(dir.getPath() + File.separator + fileName);
} else {
throw new HyracksDataException("unrecognized file found = " + fileName);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
index 792b872..259f25f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
@@ -52,6 +52,7 @@
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMBTreeUtil {
@@ -59,12 +60,13 @@
}
public static LSMBTree createLSMTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- FileReference file, IBufferCache diskBufferCache, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields, boolean durable,
- IMetadataPageManagerFactory freePageManagerFactory) throws HyracksDataException {
+ FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider,
+ ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields,
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields,
+ int[] filterFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory)
+ throws HyracksDataException {
LSMBTreeTupleWriterFactory insertTupleWriterFactory =
new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false);
LSMBTreeTupleWriterFactory deleteTupleWriterFactory =
@@ -80,14 +82,15 @@
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
ITreeIndexFrameFactory bulkLoadLeafFrameFactory = new BTreeNSMLeafFrameFactory(bulkLoadTupleWriterFactory);
- TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory,
- interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
+ TreeIndexFactory<BTree> diskBTreeFactory =
+ new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+ interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
TreeIndexFactory<BTree> bulkLoadBTreeFactory =
- new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, interiorFrameFactory,
- bulkLoadLeafFrameFactory, cmpFactories, typeTraits.length);
+ new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+ interiorFrameFactory, bulkLoadLeafFrameFactory, cmpFactories, typeTraits.length);
- BloomFilterFactory bloomFilterFactory =
- needKeyDupCheck ? new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields) : null;
+ BloomFilterFactory bloomFilterFactory = needKeyDupCheck
+ ? new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields) : null;
ComponentFilterHelper filterHelper = null;
LSMComponentFilterFrameFactory filterFrameFactory = null;
@@ -101,20 +104,20 @@
//Primary LSMBTree index has a BloomFilter.
ILSMIndexFileManager fileNameManager =
- new LSMBTreeFileManager(ioManager, file, diskBTreeFactory, needKeyDupCheck);
+ new LSMBTreeFileManager(ioManager, diskFileMapProvider, file, diskBTreeFactory, needKeyDupCheck);
return new LSMBTree(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory,
deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory,
- filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, typeTraits.length,
- cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck, btreeFields,
- filterFields, durable);
+ filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, diskFileMapProvider,
+ typeTraits.length, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck,
+ btreeFields, filterFields, durable);
}
public static ExternalBTree createExternalBTree(IIOManager ioManager, FileReference file,
- IBufferCache diskBufferCache, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
- int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
- ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
- boolean durable, IMetadataPageManagerFactory freePageManagerFactory) {
+ IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallback ioOpCallback, boolean durable, IMetadataPageManagerFactory freePageManagerFactory) {
LSMBTreeTupleWriterFactory insertTupleWriterFactory =
new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false);
LSMBTreeTupleWriterFactory deleteTupleWriterFactory =
@@ -133,31 +136,35 @@
ITreeIndexFrameFactory transactionLeafFrameFactory =
new BTreeNSMLeafFrameFactory(transactionTupleWriterFactory);
- TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory,
- interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
- TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache,
+ TreeIndexFactory<BTree> diskBTreeFactory =
+ new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+ interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
+ TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
- BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields);
+ BloomFilterFactory bloomFilterFactory =
+ new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
// This is the component factory for transactions
TreeIndexFactory<BTree> transactionBTreeFactory =
- new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, interiorFrameFactory,
- transactionLeafFrameFactory, cmpFactories, typeTraits.length);
+ new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+ interiorFrameFactory, transactionLeafFrameFactory, cmpFactories, typeTraits.length);
//TODO remove BloomFilter from external dataset's secondary LSMBTree index
- ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, file, diskBTreeFactory, true);
+ ILSMIndexFileManager fileNameManager =
+ new LSMBTreeFileManager(ioManager, diskFileMapProvider, file, diskBTreeFactory, true);
// the disk only index uses an empty ArrayList for virtual buffer caches
- return new ExternalBTree(ioManager, interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory,
- fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory,
- bloomFilterFalsePositiveRate, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback,
- transactionBTreeFactory, durable);
+ ExternalBTree lsmTree = new ExternalBTree(ioManager, interiorFrameFactory, insertLeafFrameFactory,
+ deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory,
+ bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length, cmpFactories, mergePolicy,
+ opTracker, ioScheduler, ioOpCallback, transactionBTreeFactory, durable);
+ return lsmTree;
}
public static ExternalBTreeWithBuddy createExternalBTreeWithBuddy(IIOManager ioManager, FileReference file,
- IBufferCache diskBufferCache, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
- double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields,
- boolean durable, IMetadataPageManagerFactory freePageManagerFactory) {
+ IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] cmpFactories, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ int[] buddyBTreeFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory) {
ITypeTraits[] buddyBtreeTypeTraits = new ITypeTraits[buddyBTreeFields.length];
IBinaryComparatorFactory[] buddyBtreeCmpFactories = new IBinaryComparatorFactory[buddyBTreeFields.length];
for (int i = 0; i < buddyBtreeTypeTraits.length; i++) {
@@ -177,30 +184,32 @@
ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
- TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory,
- interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
+ TreeIndexFactory<BTree> diskBTreeFactory =
+ new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+ interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
- TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache,
+ TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
int[] bloomFilterKeyFields = new int[buddyBtreeCmpFactories.length];
for (int i = 0; i < buddyBtreeCmpFactories.length; i++) {
bloomFilterKeyFields[i] = i;
}
- BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields);
+ BloomFilterFactory bloomFilterFactory =
+ new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
// buddy b-tree factory
- TreeIndexFactory<BTree> diskBuddyBTreeFactory =
- new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, buddyBtreeInteriorFrameFactory,
- buddyBtreeLeafFrameFactory, buddyBtreeCmpFactories, buddyBtreeTypeTraits.length);
+ TreeIndexFactory<BTree> diskBuddyBTreeFactory = new BTreeFactory(ioManager, diskBufferCache,
+ diskFileMapProvider, freePageManagerFactory, buddyBtreeInteriorFrameFactory, buddyBtreeLeafFrameFactory,
+ buddyBtreeCmpFactories, buddyBtreeTypeTraits.length);
- ILSMIndexFileManager fileNameManager =
- new LSMBTreeWithBuddyFileManager(ioManager, file, diskBTreeFactory, diskBuddyBTreeFactory);
+ ILSMIndexFileManager fileNameManager = new LSMBTreeWithBuddyFileManager(ioManager, diskFileMapProvider, file,
+ diskBTreeFactory, diskBuddyBTreeFactory);
// the disk only index uses an empty ArrayList for virtual buffer caches
ExternalBTreeWithBuddy lsmTree = new ExternalBTreeWithBuddy(ioManager, interiorFrameFactory,
insertLeafFrameFactory, buddyBtreeLeafFrameFactory, diskBufferCache, fileNameManager,
- bulkLoadBTreeFactory, diskBTreeFactory, diskBuddyBTreeFactory, bloomFilterFactory,
+ bulkLoadBTreeFactory, diskBTreeFactory, diskBuddyBTreeFactory, bloomFilterFactory, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, cmpFactories,
buddyBtreeCmpFactories, buddyBTreeFields, durable);
return lsmTree;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
index 37e1c49..bc1e146 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
@@ -82,9 +82,8 @@
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-util</artifactId>
- <version>${project.version}</version>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
</dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
index 18d9a67..54d64af 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
@@ -34,28 +34,28 @@
* merge).
*/
public interface ILSMIndexFileManager {
- void createDirs() throws HyracksDataException;
+ public void createDirs();
- void deleteDirs() throws HyracksDataException;
+ public void deleteDirs() throws HyracksDataException;
- LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException;
+ public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException;
- LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName)
+ public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName)
throws HyracksDataException;
- String getBaseDir();
+ public String getBaseDir();
// Deletes invalid files, and returns list of valid files from baseDir.
// The returned valid files are correctly sorted (based on the recency of data).
- List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException;
+ public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException;
- Comparator<String> getFileNameComparator();
+ public Comparator<String> getFileNameComparator();
/**
* @return delete existing transaction disk component file reference
* @throws HyracksDataException
*/
- void deleteTransactionFiles() throws HyracksDataException;
+ public void deleteTransactionFiles() throws HyracksDataException;
/**
* Rename files of a transaction removing the transaction prefix and return the component file reference in order to be committed
@@ -63,18 +63,18 @@
* @return the renamed component file references
* @throws HyracksDataException
*/
- LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException;
+ public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException;
/**
* Recover transaction files without returning them
*
* @throws HyracksDataException
*/
- void recoverTransaction() throws HyracksDataException;
+ public void recoverTransaction() throws HyracksDataException;
/**
* @return a reference to the transaction disk component file reference
* @throws IOException
*/
- LSMComponentFileReferences getNewTransactionFileReference() throws IOException;
+ public LSMComponentFileReferences getNewTransactionFileReference() throws IOException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java
index 72d5423..090ca68 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java
@@ -23,6 +23,6 @@
public class AbortRecoverLSMIndexFileManager extends AbstractLSMIndexFileManager {
public AbortRecoverLSMIndexFileManager(IIOManager ioManager, FileReference file) {
- super(ioManager, file, null);
+ super(ioManager, null, file, null);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index c5bf2ca..2be2184 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -63,6 +63,7 @@
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public abstract class AbstractLSMIndex implements ILSMIndex {
protected final ILSMHarness lsmHarness;
@@ -74,9 +75,11 @@
protected final List<ILSMMemoryComponent> memoryComponents;
protected final List<IVirtualBufferCache> virtualBufferCaches;
protected AtomicInteger currentMutableComponentId;
+
// On-disk components.
protected final IBufferCache diskBufferCache;
protected final ILSMIndexFileManager fileManager;
+ protected final IFileMapProvider diskFileMapProvider;
// components with lower indexes are newer than components with higher index
protected final List<ILSMDiskComponent> diskComponents;
protected final List<ILSMDiskComponent> inactiveDiskComponents;
@@ -92,14 +95,16 @@
protected boolean memoryComponentsAllocated = false;
public AbstractLSMIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, double bloomFilterFalsePositiveRate,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFrameFactory filterFrameFactory,
- LSMComponentFilterManager filterManager, int[] filterFields, boolean durable,
- IComponentFilterHelper filterHelper, int[] treeFields) {
+ IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider,
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
+ int[] filterFields, boolean durable, IComponentFilterHelper filterHelper,
+ int[] treeFields) {
this.ioManager = ioManager;
this.virtualBufferCaches = virtualBufferCaches;
this.diskBufferCache = diskBufferCache;
+ this.diskFileMapProvider = diskFileMapProvider;
this.fileManager = fileManager;
this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
this.ioScheduler = ioScheduler;
@@ -125,10 +130,12 @@
// The constructor used by external indexes
public AbstractLSMIndex(IIOManager ioManager, IBufferCache diskBufferCache, ILSMIndexFileManager fileManager,
- double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean durable) {
+ IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ boolean durable) {
this.ioManager = ioManager;
this.diskBufferCache = diskBufferCache;
+ this.diskFileMapProvider = diskFileMapProvider;
this.fileManager = fileManager;
this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
this.ioScheduler = ioScheduler;
@@ -155,6 +162,7 @@
if (isActive) {
throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_ACTIVE_INDEX);
}
+ fileManager.deleteDirs();
fileManager.createDirs();
diskComponents.clear();
}
@@ -230,9 +238,16 @@
throw HyracksDataException.create(ErrorCode.CANNOT_DESTROY_ACTIVE_INDEX);
}
destroyDiskComponents();
+ destroyMemoryComponents();
fileManager.deleteDirs();
}
+ protected void destroyMemoryComponents() throws HyracksDataException {
+ for (ILSMMemoryComponent c : memoryComponents) {
+ destroyMemoryComponent(c);
+ }
+ }
+
protected void destroyDiskComponents() throws HyracksDataException {
List<ILSMDiskComponent> immutableComponents = diskComponents;
for (ILSMDiskComponent c : immutableComponents) {
@@ -638,6 +653,8 @@
protected abstract void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException;
+ protected abstract void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException;
+
protected abstract void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException;
protected abstract void clearMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
index ead9db4..fca010f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
@@ -33,11 +33,11 @@
import java.util.HashSet;
import java.util.List;
+import org.apache.commons.io.FileUtils;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
-import org.apache.hyracks.api.util.IoUtil;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
@@ -45,63 +45,49 @@
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManager {
+ public static final String SPLIT_STRING = "_";
+ protected static final String BLOOM_FILTER_STRING = "f";
+ protected static final String TRANSACTION_PREFIX = ".T";
+
public enum TreeIndexState {
INVALID,
VERSION_MISMATCH,
VALID
}
- /**
- * Split different parts of the file name
- */
- public static final String DELIMITER = "_";
- /**
- * Indicates a B tree
- */
- public static final String BTREE_SUFFIX = "b";
- /**
- * Indicates an R tree
- */
- public static final String RTREE_SUFFIX = "r";
- /**
- * Indicates a bloom filter
- */
- public static final String BLOOM_FILTER_SUFFIX = "f";
- /**
- * Indicates a delete tree
- */
- public static final String DELETE_TREE_SUFFIX = "d";
- /**
- * Hides transaction components until they are either committed by removing this file or deleted along with the file
- */
- public static final String TXN_PREFIX = ".T";
-
- protected static final FilenameFilter fileNameFilter = (dir, name) -> !name.startsWith(".");
- protected static final FilenameFilter txnFileNameFilter = (dir, name) -> name.startsWith(TXN_PREFIX);
- protected static FilenameFilter bloomFilterFilter =
- (dir, name) -> !name.startsWith(".") && name.endsWith(BLOOM_FILTER_SUFFIX);
- protected static final FilenameFilter dummyFilter = (dir, name) -> true;
- protected static final Comparator<String> cmp = new FileNameComparator();
-
protected final IIOManager ioManager;
+ protected final IFileMapProvider fileMapProvider;
// baseDir should reflect dataset name and partition name and be absolute
- protected final String baseDir;
+ protected String baseDir;
protected final Format formatter = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS");
+ protected final Comparator<String> cmp = new FileNameComparator();
protected final Comparator<ComparableFileName> recencyCmp = new RecencyComparator();
protected final TreeIndexFactory<? extends ITreeIndex> treeFactory;
+
private String prevTimestamp = null;
- public AbstractLSMIndexFileManager(IIOManager ioManager, FileReference file,
+ public AbstractLSMIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
TreeIndexFactory<? extends ITreeIndex> treeFactory) {
this.ioManager = ioManager;
- this.baseDir = file.getFile().getAbsolutePath().endsWith(File.separator) ? file.getFile().getAbsolutePath()
- : file.getFile().getAbsolutePath() + File.separator;
+ this.baseDir = file.getFile().getAbsolutePath();
+ if (!baseDir.endsWith(System.getProperty("file.separator"))) {
+ baseDir += System.getProperty("file.separator");
+ }
+ this.fileMapProvider = fileMapProvider;
this.treeFactory = treeFactory;
}
+ private static FilenameFilter fileNameFilter = new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return !name.startsWith(".");
+ }
+ };
+
protected TreeIndexState isValidTreeIndex(ITreeIndex treeIndex) throws HyracksDataException {
IBufferCache bufferCache = treeIndex.getBufferCache();
treeIndex.activate();
@@ -162,7 +148,7 @@
if (!dir.canRead()) {
throw HyracksDataException.create(ErrorCode.CANNOT_READ_FILE, path);
} else if (!dir.exists()) {
- throw HyracksDataException.create(ErrorCode.FILE_DOES_NOT_EXIST, path);
+ throw HyracksDataException.create(ErrorCode.FILE_DOES_NOT_EXISTS, path);
} else if (!dir.isDirectory()) {
throw HyracksDataException.create(ErrorCode.FILE_IS_NOT_DIRECTORY, path);
}
@@ -176,31 +162,44 @@
ArrayList<ComparableFileName> tmpAllInvListsFiles = new ArrayList<>();
cleanupAndGetValidFilesInternal(filter, treeFactory, tmpAllInvListsFiles);
for (ComparableFileName cmpFileName : tmpAllInvListsFiles) {
- int index = cmpFileName.fileName.lastIndexOf(DELIMITER);
+ int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
String file = cmpFileName.fileName.substring(0, index);
if (groundTruth.contains(file)) {
validFiles.add(cmpFileName);
} else {
File invalidFile = new File(cmpFileName.fullPath);
- IoUtil.delete(invalidFile);
+ invalidFile.delete();
}
}
}
@Override
- public void createDirs() throws HyracksDataException {
+ public void createDirs() {
File f = new File(baseDir);
- if (f.exists()) {
- throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_EXISTING_INDEX);
- }
f.mkdirs();
}
@Override
public void deleteDirs() throws HyracksDataException {
- IoUtil.delete(new File(baseDir));
+ File f = new File(baseDir);
+ if (f.exists()) {
+ delete(f);
+ }
}
+ private void delete(File f) throws HyracksDataException {
+ if (!FileUtils.deleteQuietly(f)) {
+ throw HyracksDataException.create(ErrorCode.UNIDENTIFIED_IO_ERROR_DELETING_DIR, f.getPath());
+ }
+ }
+
+ protected static FilenameFilter bloomFilterFilter = new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return !name.startsWith(".") && name.endsWith(BLOOM_FILTER_STRING);
+ }
+ };
+
protected FileReference createFlushFile(String flushFileName) throws HyracksDataException {
return ioManager.resolveAbsolutePath(flushFileName);
}
@@ -213,17 +212,17 @@
public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException {
String ts = getCurrentTimestamp();
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseDir + ts + DELIMITER + ts), null, null);
+ return new LSMComponentFileReferences(createFlushFile(baseDir + ts + SPLIT_STRING + ts), null, null);
}
@Override
public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName)
throws HyracksDataException {
- String[] firstTimestampRange = firstFileName.split(DELIMITER);
- String[] lastTimestampRange = lastFileName.split(DELIMITER);
+ String[] firstTimestampRange = firstFileName.split(SPLIT_STRING);
+ String[] lastTimestampRange = lastFileName.split(SPLIT_STRING);
// Get the range of timestamps by taking the earliest and the latest timestamps
return new LSMComponentFileReferences(
- createMergeFile(baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]), null, null);
+ createMergeFile(baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1]), null, null);
}
@Override
@@ -293,7 +292,7 @@
* 2. Flushed files are sorted from newest to oldest (based on the timestamp
* string)
*/
- private static class FileNameComparator implements Comparator<String> {
+ private class FileNameComparator implements Comparator<String> {
@Override
public int compare(String a, String b) {
// Consciously ignoring locale.
@@ -308,7 +307,7 @@
@Override
public void recoverTransaction() throws HyracksDataException {
- String[] files = listDirFiles(baseDir, txnFileNameFilter);
+ String[] files = listDirFiles(baseDir, transactionFileNameFilter);
File dir = new File(baseDir);
try {
if (files.length == 0) {
@@ -335,7 +334,7 @@
this.fileRef = fileRef;
this.fullPath = fileRef.getFile().getAbsolutePath();
this.fileName = fileRef.getFile().getName();
- interval = fileName.split(DELIMITER);
+ interval = fileName.split(SPLIT_STRING);
}
@Override
@@ -362,7 +361,7 @@
// This function is used to delete transaction files for aborted transactions
@Override
public void deleteTransactionFiles() throws HyracksDataException {
- String[] files = listDirFiles(baseDir, txnFileNameFilter);
+ String[] files = listDirFiles(baseDir, transactionFileNameFilter);
if (files.length == 0) {
// Do nothing
} else if (files.length > 1) {
@@ -400,9 +399,23 @@
return null;
}
+ protected static FilenameFilter transactionFileNameFilter = new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.startsWith(".T");
+ }
+ };
+
+ protected static FilenameFilter dummyFilter = new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return true;
+ }
+ };
+
protected static FilenameFilter createTransactionFilter(String transactionFileName, final boolean inclusive) {
- final String timeStamp =
- transactionFileName.substring(transactionFileName.indexOf(TXN_PREFIX) + TXN_PREFIX.length());
+ final String timeStamp = transactionFileName
+ .substring(transactionFileName.indexOf(TRANSACTION_PREFIX) + TRANSACTION_PREFIX.length());
return new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
@@ -416,7 +429,7 @@
}
protected FilenameFilter getTransactionFileFilter(boolean inclusive) throws HyracksDataException {
- String[] files = listDirFiles(baseDir, txnFileNameFilter);
+ String[] files = listDirFiles(baseDir, transactionFileNameFilter);
if (files.length == 0) {
return dummyFilter;
} else {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
index 4eefcd8..562ed5a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
@@ -26,20 +26,21 @@
import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class BTreeFactory extends TreeIndexFactory<BTree> {
- public BTreeFactory(IIOManager ioManager, IBufferCache bufferCache, IPageManagerFactory freePageManagerFactory,
- ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
- IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
- super(ioManager, bufferCache, freePageManagerFactory, interiorFrameFactory, leafFrameFactory, cmpFactories,
- fieldCount);
+ public BTreeFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ IPageManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
+ super(ioManager, bufferCache, fileMapProvider, freePageManagerFactory, interiorFrameFactory, leafFrameFactory,
+ cmpFactories, fieldCount);
}
@Override
public BTree createIndexInstance(FileReference file) {
- return new BTree(bufferCache, freePageManagerFactory.createPageManager(bufferCache), interiorFrameFactory,
- leafFrameFactory, cmpFactories, fieldCount, file);
+ return new BTree(bufferCache, fileMapProvider, freePageManagerFactory.createPageManager(bufferCache),
+ interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, file);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java
index 7a05d6d..55fc92c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java
@@ -25,16 +25,20 @@
import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public abstract class IndexFactory<T extends IIndex> {
protected final IIOManager ioManager;
protected final IBufferCache bufferCache;
+ protected final IFileMapProvider fileMapProvider;
protected final IPageManagerFactory freePageManagerFactory;
- public IndexFactory(IIOManager ioManager, IBufferCache bufferCache, IPageManagerFactory freePageManagerFactory) {
+ public IndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ IPageManagerFactory freePageManagerFactory) {
this.ioManager = ioManager;
this.bufferCache = bufferCache;
+ this.fileMapProvider = fileMapProvider;
this.freePageManagerFactory = freePageManagerFactory;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
index 83e140c..6878fcf 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
@@ -18,8 +18,6 @@
*/
package org.apache.hyracks.storage.am.lsm.common.impls;
-import java.util.HashMap;
-import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -31,7 +29,6 @@
import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
import org.apache.hyracks.storage.common.file.IFileMapManager;
-import org.apache.hyracks.util.JSONUtil;
public class MultitenantVirtualBufferCache implements IVirtualBufferCache {
private static final Logger LOGGER = Logger.getLogger(ExternalIndexHarness.class.getName());
@@ -45,8 +42,8 @@
}
@Override
- public int createFile(FileReference fileRef) throws HyracksDataException {
- return vbc.createFile(fileRef);
+ public void createFile(FileReference fileRef) throws HyracksDataException {
+ vbc.createFile(fileRef);
}
@Override
@@ -60,8 +57,8 @@
}
@Override
- public void deleteFile(int fileId) throws HyracksDataException {
- vbc.deleteFile(fileId);
+ public void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException {
+ vbc.deleteFile(fileId, flushDirtyPages);
}
@Override
@@ -94,7 +91,6 @@
return vbc.getPageSize();
}
- @Override
public int getPageSizeWithHeader() {
return vbc.getPageSizeWithHeader();
}
@@ -137,7 +133,7 @@
@Override
public int getNumPagesOfFile(int fileId) throws HyracksDataException {
- return vbc.getNumPagesOfFile(fileId);
+ throw new UnsupportedOperationException();
}
@Override
@@ -209,28 +205,4 @@
throws HyracksDataException {
vbc.resizePage(page, multiplier, extraPageBlockHelper);
}
-
- @Override
- public String toString() {
- return JSONUtil.fromMap(toMap());
- }
-
- private Map<String, Object> toMap() {
- HashMap<String, Object> map = new HashMap<>();
- map.put("class", getClass().getSimpleName());
- map.put("vbc", vbc.toString());
- map.put("openCount", openCount);
- return map;
- }
-
- @Override
- public int openFile(FileReference fileRef) throws HyracksDataException {
- return vbc.openFile(fileRef);
- }
-
- @Override
- public void deleteFile(FileReference file) throws HyracksDataException {
- vbc.deleteFile(file);
- }
-
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java
index b6e2dd4..2a7bcca 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public abstract class TreeIndexFactory<T extends ITreeIndex> extends IndexFactory<T> {
@@ -33,10 +34,10 @@
protected final IBinaryComparatorFactory[] cmpFactories;
protected final int fieldCount;
- public TreeIndexFactory(IIOManager ioManager, IBufferCache bufferCache, IPageManagerFactory freePageManagerFactory,
- ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
- IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
- super(ioManager, bufferCache, freePageManagerFactory);
+ public TreeIndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ IPageManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
+ super(ioManager, bufferCache, fileMapProvider, freePageManagerFactory);
this.interiorFrameFactory = interiorFrameFactory;
this.leafFrameFactory = leafFrameFactory;
this.cmpFactories = cmpFactories;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
index d26677e..27d879c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
@@ -21,8 +21,6 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import java.util.logging.Level;
@@ -39,8 +37,7 @@
import org.apache.hyracks.storage.common.buffercache.VirtualPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
import org.apache.hyracks.storage.common.file.IFileMapManager;
-import org.apache.hyracks.storage.common.file.FileMapManager;
-import org.apache.hyracks.util.JSONUtil;
+import org.apache.hyracks.storage.common.file.TransientFileMapManager;
public class VirtualBufferCache implements IVirtualBufferCache {
private static final Logger LOGGER = Logger.getLogger(ExternalIndexHarness.class.getName());
@@ -62,7 +59,7 @@
public VirtualBufferCache(ICacheMemoryAllocator allocator, int pageSize, int numPages) {
this.allocator = allocator;
- this.fileMapManager = new FileMapManager();
+ this.fileMapManager = new TransientFileMapManager();
this.pageSize = pageSize;
this.numPages = 2 * (numPages / 2) + 1;
@@ -74,17 +71,12 @@
}
@Override
- public int createFile(FileReference fileRef) throws HyracksDataException {
- return fileMapManager.registerFile(fileRef);
- }
-
- @Override
- public int openFile(FileReference fileRef) throws HyracksDataException {
+ public void createFile(FileReference fileRef) throws HyracksDataException {
synchronized (fileMapManager) {
if (fileMapManager.isMapped(fileRef)) {
- return fileMapManager.lookupFileId(fileRef);
+ throw new HyracksDataException("File " + fileRef + " is already mapped");
}
- return fileMapManager.registerFile(fileRef);
+ fileMapManager.registerFile(fileRef);
}
}
@@ -97,16 +89,14 @@
}
@Override
- public void deleteFile(FileReference fileRef) throws HyracksDataException {
+ public void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException {
synchronized (fileMapManager) {
- int fileId = fileMapManager.lookupFileId(fileRef);
- deleteFile(fileId);
+ if (!fileMapManager.isMapped(fileId)) {
+ throw new HyracksDataException("File with id " + fileId + " is not mapped");
+ }
+ fileMapManager.unregisterFile(fileId);
}
- }
- @Override
- public void deleteFile(int fileId) throws HyracksDataException {
- fileMapManager.unregisterFile(fileId);
for (int i = 0; i < buckets.length; i++) {
final CacheBucket bucket = buckets[i];
bucket.bucketLock.lock();
@@ -365,7 +355,12 @@
@Override
public int getNumPagesOfFile(int fileId) throws HyracksDataException {
- return -1;
+ synchronized (fileMapManager) {
+ if (fileMapManager.isMapped(fileId)) {
+ return numPages;
+ }
+ }
+ return 0;
}
@Override
@@ -429,21 +424,7 @@
@Override
public void purgeHandle(int fileId) throws HyracksDataException {
- deleteFile(fileId);
+
}
- @Override
- public String toString() {
- return JSONUtil.fromMap(toMap());
- }
-
- private Map<String, Object> toMap() {
- HashMap<String, Object> map = new HashMap<>();
- map.put("class", getClass().getSimpleName());
- map.put("allocator", allocator.toString());
- map.put("pageSize", pageSize);
- map.put("numPages", numPages);
- map.put("open", open);
- return map;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml
index b94ebee..0f4063a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml
@@ -16,8 +16,7 @@
! specific language governing permissions and limitations
! under the License.
!-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>hyracks-storage-am-lsm-invertedindex</artifactId>
<parent>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java
index eceb7b4..7258076 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java
@@ -39,13 +39,4 @@
ITypeTraits[] getTokenTypeTraits();
IBinaryComparatorFactory[] getTokenCmpFactories();
-
- /**
- * Purge the index files out of the buffer cache.
- * Can only be called if the caller is absolutely sure the files don't contain dirty pages
- *
- * @throws HyracksDataException
- * if the index is active
- */
- void purge() throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
index 7e2ec50..45ca106 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
@@ -42,6 +42,7 @@
import org.apache.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexUtils;
import org.apache.hyracks.storage.common.IStorageManager;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMInvertedIndexLocalResource extends LsmResource {
@@ -85,23 +86,25 @@
FileReference file = ioManager.resolve(path);
List<IVirtualBufferCache> virtualBufferCaches = vbcProvider.getVirtualBufferCaches(serviceCtx, file);
IBufferCache bufferCache = storageManager.getBufferCache(serviceCtx);
+ IFileMapProvider fileMapManager = storageManager.getFileMapProvider(serviceCtx);
ILSMMergePolicy mergePolicy = mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx);
ILSMIOOperationScheduler ioScheduler = ioSchedulerProvider.getIoScheduler(serviceCtx);
if (isPartitioned) {
- return InvertedIndexUtils.createPartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, typeTraits,
- cmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache,
+ return InvertedIndexUtils.createPartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, fileMapManager,
+ typeTraits, cmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache,
file.getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy,
opTrackerProvider.getOperationTracker(serviceCtx), ioScheduler,
ioOpCallbackFactory.createIoOpCallback(), invertedIndexFields, filterTypeTraits, filterCmpFactories,
filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable,
metadataPageManagerFactory);
} else {
- return InvertedIndexUtils.createLSMInvertedIndex(ioManager, virtualBufferCaches, typeTraits, cmpFactories,
- tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache, file.getAbsolutePath(),
- bloomFilterFalsePositiveRate, mergePolicy, opTrackerProvider.getOperationTracker(serviceCtx),
- ioScheduler, ioOpCallbackFactory.createIoOpCallback(), invertedIndexFields, filterTypeTraits,
- filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps, durable, metadataPageManagerFactory);
+ return InvertedIndexUtils.createLSMInvertedIndex(ioManager, virtualBufferCaches, fileMapManager, typeTraits,
+ cmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache,
+ file.getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy,
+ opTrackerProvider.getOperationTracker(serviceCtx), ioScheduler,
+ ioOpCallbackFactory.createIoOpCallback(), invertedIndexFields, filterTypeTraits, filterCmpFactories,
+ filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable,
+ metadataPageManagerFactory);
}
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index f3f2ff0..d5edbb5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -87,6 +87,7 @@
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex {
private static final Logger LOGGER = Logger.getLogger(LSMInvertedIndex.class.getName());
@@ -109,14 +110,14 @@
OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory,
BloomFilterFactory bloomFilterFactory, IComponentFilterHelper filterHelper,
ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
- double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager, ITypeTraits[] invListTypeTraits,
- IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
- IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, int[] filterFields,
- int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable)
- throws HyracksDataException {
- super(ioManager, virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager,
+ double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider,
+ ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
+ ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
+ IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields,
+ int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps,
+ boolean durable) throws HyracksDataException {
+ super(ioManager, virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
filterManager, filterFields, durable, filterHelper, invertedIndexFields);
this.tokenizerFactory = tokenizerFactory;
@@ -133,10 +134,10 @@
for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
InMemoryInvertedIndex memInvIndex =
createInMemoryInvertedIndex(virtualBufferCache, new VirtualFreePageManager(virtualBufferCache), i);
- BTree deleteKeysBTree =
- BTreeUtils.createBTree(virtualBufferCache, new VirtualFreePageManager(virtualBufferCache),
- invListTypeTraits, invListCmpFactories, BTreeLeafFrameType.REGULAR_NSM,
- ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_del_" + i));
+ BTree deleteKeysBTree = BTreeUtils.createBTree(virtualBufferCache,
+ new VirtualFreePageManager(virtualBufferCache), virtualBufferCache.getFileMapProvider(),
+ invListTypeTraits, invListCmpFactories, BTreeLeafFrameType.REGULAR_NSM,
+ ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_del_" + i));
LSMInvertedIndexMemoryComponent mutableComponent =
new LSMInvertedIndexMemoryComponent(memInvIndex, deleteKeysBTree, virtualBufferCache,
i == 0 ? true : false, filterHelper == null ? null : filterHelper.createFilter());
@@ -176,11 +177,8 @@
protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
component.getBloomFilter().deactivate();
- component.getBloomFilter().purge();
component.getInvIndex().deactivate();
- component.getInvIndex().purge();
component.getDeletedKeysBTree().deactivate();
- component.getDeletedKeysBTree().purge();
}
@Override
@@ -191,6 +189,13 @@
component.getBloomFilter().destroy();
}
+ @Override
+ protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+ mutableComponent.getInvIndex().destroy();
+ mutableComponent.getDeletedKeysBTree().destroy();
+ }
+
/**
* The keys in the in-memory deleted-keys BTree only refer to on-disk components.
* We delete documents from the in-memory inverted index by deleting its entries directly,
@@ -736,9 +741,4 @@
mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getDeleteIndexFileReference(),
mergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir());
}
-
- @Override
- public void purge() throws HyracksDataException {
- throw new UnsupportedOperationException();
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
index 9254c45..aeb1e16 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
@@ -34,6 +34,7 @@
import org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNameMapper;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
// TODO: Refactor for better code sharing with other file managers.
public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager implements IInvertedIndexFileNameMapper {
@@ -65,32 +66,33 @@
}
};
- public LSMInvertedIndexFileManager(IIOManager ioManager, FileReference file, BTreeFactory btreeFactory) {
- super(ioManager, file, null);
+ public LSMInvertedIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
+ BTreeFactory btreeFactory) {
+ super(ioManager, fileMapProvider, file, null);
this.btreeFactory = btreeFactory;
}
@Override
public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException {
String ts = getCurrentTimestamp();
- String baseName = baseDir + ts + DELIMITER + ts;
+ String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + DICT_BTREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + DELETED_KEYS_BTREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX),
+ createFlushFile(baseName + SPLIT_STRING + DELETED_KEYS_BTREE_SUFFIX),
+ createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING));
}
@Override
public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName)
throws HyracksDataException {
- String[] firstTimestampRange = firstFileName.split(DELIMITER);
- String[] lastTimestampRange = lastFileName.split(DELIMITER);
+ String[] firstTimestampRange = firstFileName.split(SPLIT_STRING);
+ String[] lastTimestampRange = lastFileName.split(SPLIT_STRING);
- String baseName = baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1];
+ String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1];
// Get the range of timestamps by taking the earliest and the latest timestamps
- return new LSMComponentFileReferences(createMergeFile(baseName + DELIMITER + DICT_BTREE_SUFFIX),
- createMergeFile(baseName + DELIMITER + DELETED_KEYS_BTREE_SUFFIX),
- createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX),
+ createMergeFile(baseName + SPLIT_STRING + DELETED_KEYS_BTREE_SUFFIX),
+ createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING));
}
@Override
@@ -105,7 +107,7 @@
cleanupAndGetValidFilesInternal(deletedKeysBTreeFilter, btreeFactory, allDeletedKeysBTreeFiles);
HashSet<String> deletedKeysBTreeFilesSet = new HashSet<>();
for (ComparableFileName cmpFileName : allDeletedKeysBTreeFiles) {
- int index = cmpFileName.fileName.lastIndexOf(DELIMITER);
+ int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
deletedKeysBTreeFilesSet.add(cmpFileName.fileName.substring(0, index));
}
@@ -208,8 +210,8 @@
@Override
public String getInvListsFilePath(String dictBTreeFilePath) {
- int index = dictBTreeFilePath.lastIndexOf(DELIMITER);
+ int index = dictBTreeFilePath.lastIndexOf(SPLIT_STRING);
String file = dictBTreeFilePath.substring(0, index);
- return file + DELIMITER + INVLISTS_SUFFIX;
+ return file + SPLIT_STRING + INVLISTS_SUFFIX;
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
index d154356..8863427 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
@@ -41,6 +41,7 @@
import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexUtils;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class PartitionedLSMInvertedIndex extends LSMInvertedIndex {
@@ -48,17 +49,17 @@
OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory,
BloomFilterFactory bloomFilterFactory, IComponentFilterHelper filterHelper,
ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
- double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager, ITypeTraits[] invListTypeTraits,
- IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
- IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, int[] filterFields,
- int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable)
- throws HyracksDataException {
+ double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider,
+ ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
+ ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
+ IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields,
+ int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps,
+ boolean durable) throws HyracksDataException {
super(ioManager, virtualBufferCaches, diskInvIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory,
filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, fileManager,
- invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
- mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields,
+ diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories,
+ tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields,
filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
index d686634..4b673e5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
@@ -31,6 +31,7 @@
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.common.api.IPageManager;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
@@ -73,8 +74,9 @@
btreeTypeTraits[tokenTypeTraits.length + i] = invListTypeTraits[i];
btreeCmpFactories[tokenTypeTraits.length + i] = invListCmpFactories[i];
}
- this.btree = BTreeUtils.createBTree(virtualBufferCache, virtualFreePageManager, btreeTypeTraits,
- btreeCmpFactories, BTreeLeafFrameType.REGULAR_NSM, btreeFileRef);
+ this.btree = BTreeUtils.createBTree(virtualBufferCache, virtualFreePageManager,
+ ((IVirtualBufferCache) virtualBufferCache).getFileMapProvider(), btreeTypeTraits, btreeCmpFactories,
+ BTreeLeafFrameType.REGULAR_NSM, btreeFileRef);
}
@Override
@@ -217,9 +219,4 @@
public int getNumOfFilterFields() {
return 0;
}
-
- @Override
- public void purge() throws HyracksDataException {
- btree.purge();
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
index 25e4201..277eb64 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
@@ -26,7 +26,6 @@
import org.apache.hyracks.api.context.IHyracksCommonContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
@@ -61,6 +60,7 @@
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
/**
* An inverted index consists of two files: 1. a file storing (paginated)
@@ -94,6 +94,7 @@
protected BTree btree;
protected int rootPageId = 0;
protected IBufferCache bufferCache;
+ protected IFileMapProvider fileMapProvider;
protected int fileId = -1;
protected final ITypeTraits[] invListTypeTraits;
protected final IBinaryComparatorFactory[] invListCmpFactories;
@@ -108,18 +109,21 @@
protected boolean isOpen = false;
protected boolean wasOpen = false;
- public OnDiskInvertedIndex(IBufferCache bufferCache, IInvertedListBuilder invListBuilder,
- ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
- ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference btreeFile,
- FileReference invListsFile, IPageManagerFactory pageManagerFactory) throws HyracksDataException {
+ public OnDiskInvertedIndex(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ IInvertedListBuilder invListBuilder, ITypeTraits[] invListTypeTraits,
+ IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
+ IBinaryComparatorFactory[] tokenCmpFactories, FileReference btreeFile, FileReference invListsFile,
+ IPageManagerFactory pageManagerFactory) throws HyracksDataException {
this.bufferCache = bufferCache;
+ this.fileMapProvider = fileMapProvider;
this.invListBuilder = invListBuilder;
this.invListTypeTraits = invListTypeTraits;
this.invListCmpFactories = invListCmpFactories;
this.tokenTypeTraits = tokenTypeTraits;
this.tokenCmpFactories = tokenCmpFactories;
- this.btree = BTreeUtils.createBTree(bufferCache, getBTreeTypeTraits(tokenTypeTraits), tokenCmpFactories,
- BTreeLeafFrameType.REGULAR_NSM, btreeFile, pageManagerFactory.createPageManager(bufferCache));
+ this.btree = BTreeUtils.createBTree(bufferCache, fileMapProvider, getBTreeTypeTraits(tokenTypeTraits),
+ tokenCmpFactories, BTreeLeafFrameType.REGULAR_NSM, btreeFile,
+ pageManagerFactory.createPageManager(bufferCache));
this.numTokenFields = btree.getComparatorFactories().length;
this.numInvListKeys = invListCmpFactories.length;
this.invListsFile = invListsFile;
@@ -135,7 +139,25 @@
throw new HyracksDataException("Failed to create since index is already open.");
}
btree.create();
- fileId = bufferCache.createFile(invListsFile);
+ boolean fileIsMapped = false;
+ synchronized (fileMapProvider) {
+ fileIsMapped = fileMapProvider.isMapped(invListsFile);
+ if (!fileIsMapped) {
+ bufferCache.createFile(invListsFile);
+ }
+ fileId = fileMapProvider.lookupFileId(invListsFile);
+ try {
+ // Also creates the file if it doesn't exist yet.
+ bufferCache.openFile(fileId);
+ } catch (HyracksDataException e) {
+ // Revert state of buffer cache since file failed to open.
+ if (!fileIsMapped) {
+ bufferCache.deleteFile(fileId, false);
+ }
+ throw e;
+ }
+ }
+ bufferCache.closeFile(fileId);
}
@Override
@@ -144,11 +166,25 @@
throw new HyracksDataException("Failed to activate the index since it is already activated.");
}
btree.activate();
- if (fileId >= 0) {
- bufferCache.openFile(fileId);
- } else {
- fileId = bufferCache.openFile(invListsFile);
+ boolean fileIsMapped = false;
+ synchronized (fileMapProvider) {
+ fileIsMapped = fileMapProvider.isMapped(invListsFile);
+ if (!fileIsMapped) {
+ bufferCache.createFile(invListsFile);
+ }
+ fileId = fileMapProvider.lookupFileId(invListsFile);
+ try {
+ // Also creates the file if it doesn't exist yet.
+ bufferCache.openFile(fileId);
+ } catch (HyracksDataException e) {
+ // Revert state of buffer cache since file failed to open.
+ if (!fileIsMapped) {
+ bufferCache.deleteFile(fileId, false);
+ }
+ throw e;
+ }
}
+
isOpen = true;
wasOpen = true;
}
@@ -158,8 +194,10 @@
if (!isOpen && wasOpen) {
throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
}
+
btree.deactivate();
bufferCache.closeFile(fileId);
+
isOpen = false;
}
@@ -168,8 +206,15 @@
if (isOpen) {
throw new HyracksDataException("Failed to destroy since index is already open.");
}
+
btree.destroy();
- bufferCache.deleteFile(invListsFile);
+ invListsFile.delete();
+ if (fileId == -1) {
+ return;
+ }
+
+ bufferCache.deleteFile(fileId, false);
+ fileId = -1;
}
@Override
@@ -179,9 +224,27 @@
}
btree.clear();
bufferCache.closeFile(fileId);
- bufferCache.deleteFile(fileId);
- fileId = bufferCache.createFile(invListsFile);
- bufferCache.openFile(fileId);
+ bufferCache.deleteFile(fileId, false);
+ invListsFile.getFile().delete();
+
+ boolean fileIsMapped = false;
+ synchronized (fileMapProvider) {
+ fileIsMapped = fileMapProvider.isMapped(invListsFile);
+ if (!fileIsMapped) {
+ bufferCache.createFile(invListsFile);
+ }
+ fileId = fileMapProvider.lookupFileId(invListsFile);
+ try {
+ // Also creates the file if it doesn't exist yet.
+ bufferCache.openFile(fileId);
+ } catch (HyracksDataException e) {
+ // Revert state of buffer cache since file failed to open.
+ if (!fileIsMapped) {
+ bufferCache.deleteFile(fileId, false);
+ }
+ throw e;
+ }
+ }
}
@Override
@@ -634,14 +697,4 @@
public int getNumOfFilterFields() {
return 0;
}
-
- @Override
- public synchronized void purge() throws HyracksDataException {
- if (isOpen) {
- throw HyracksDataException.create(ErrorCode.CANNOT_PURGE_ACTIVE_INDEX);
- }
- btree.purge();
- bufferCache.purgeHandle(fileId);
- fileId = -1;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java
index 70fcd03..dacef3c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java
@@ -30,6 +30,7 @@
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilderFactory;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class OnDiskInvertedIndexFactory extends IndexFactory<IInvertedIndex> {
@@ -40,12 +41,12 @@
protected final IBinaryComparatorFactory[] tokenCmpFactories;
protected final IInvertedIndexFileNameMapper fileNameMapper;
- public OnDiskInvertedIndexFactory(IIOManager ioManager, IBufferCache bufferCache,
+ public OnDiskInvertedIndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider,
IInvertedListBuilderFactory invListBuilderFactory, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IInvertedIndexFileNameMapper fileNameMapper,
IPageManagerFactory pageManagerFactory) {
- super(ioManager, bufferCache, pageManagerFactory);
+ super(ioManager, bufferCache, fileMapProvider, pageManagerFactory);
this.invListBuilderFactory = invListBuilderFactory;
this.invListTypeTraits = invListTypeTraits;
this.invListCmpFactories = invListCmpFactories;
@@ -59,7 +60,8 @@
String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getAbsolutePath());
FileReference invListsFile = ioManager.resolveAbsolutePath(invListsFilePath);
IInvertedListBuilder invListBuilder = invListBuilderFactory.create();
- return new OnDiskInvertedIndex(bufferCache, invListBuilder, invListTypeTraits, invListCmpFactories,
- tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile, freePageManagerFactory);
+ return new OnDiskInvertedIndex(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits,
+ invListCmpFactories, tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile,
+ freePageManagerFactory);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
index 0c0110e..452d59b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
@@ -39,17 +39,19 @@
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class PartitionedOnDiskInvertedIndex extends OnDiskInvertedIndex implements IPartitionedInvertedIndex {
protected final int PARTITIONING_NUM_TOKENS_FIELD = 1;
- public PartitionedOnDiskInvertedIndex(IBufferCache bufferCache, IInvertedListBuilder invListBuilder,
- ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
- ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference btreeFile,
- FileReference invListsFile, IPageManagerFactory pageManagerFactory) throws HyracksDataException {
- super(bufferCache, invListBuilder, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories,
- btreeFile, invListsFile, pageManagerFactory);
+ public PartitionedOnDiskInvertedIndex(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ IInvertedListBuilder invListBuilder, ITypeTraits[] invListTypeTraits,
+ IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
+ IBinaryComparatorFactory[] tokenCmpFactories, FileReference btreeFile, FileReference invListsFile,
+ IPageManagerFactory pageManagerFactory) throws HyracksDataException {
+ super(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
+ tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory);
}
public class PartitionedOnDiskInvertedIndexAccessor extends OnDiskInvertedIndexAccessor {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java
index 5b2888a..ebc0152 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java
@@ -29,16 +29,17 @@
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilderFactory;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class PartitionedOnDiskInvertedIndexFactory extends OnDiskInvertedIndexFactory {
public PartitionedOnDiskInvertedIndexFactory(IIOManager ioManager, IBufferCache bufferCache,
- IInvertedListBuilderFactory invListBuilderFactory, ITypeTraits[] invListTypeTraits,
- IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
- IBinaryComparatorFactory[] tokenCmpFactories, IInvertedIndexFileNameMapper fileNameMapper,
- IPageManagerFactory pageManagerFactory) {
- super(ioManager, bufferCache, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
- tokenCmpFactories, fileNameMapper, pageManagerFactory);
+ IFileMapProvider fileMapProvider, IInvertedListBuilderFactory invListBuilderFactory,
+ ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
+ ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
+ IInvertedIndexFileNameMapper fileNameMapper, IPageManagerFactory pageManagerFactory) {
+ super(ioManager, bufferCache, fileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories,
+ tokenTypeTraits, tokenCmpFactories, fileNameMapper, pageManagerFactory);
}
@Override
@@ -46,7 +47,8 @@
String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getAbsolutePath());
FileReference invListsFile = ioManager.resolveAbsolutePath(invListsFilePath);
IInvertedListBuilder invListBuilder = invListBuilderFactory.create();
- return new PartitionedOnDiskInvertedIndex(bufferCache, invListBuilder, invListTypeTraits, invListCmpFactories,
- tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile, freePageManagerFactory);
+ return new PartitionedOnDiskInvertedIndex(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits,
+ invListCmpFactories, tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile,
+ freePageManagerFactory);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
index 540e100..8a36f61 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
@@ -59,6 +59,7 @@
import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.PartitionedOnDiskInvertedIndexFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class InvertedIndexUtils {
@@ -81,23 +82,25 @@
}
public static OnDiskInvertedIndex createOnDiskInvertedIndex(IIOManager ioManager, IBufferCache bufferCache,
- ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
- ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile,
+ IFileMapProvider fileMapProvider, ITypeTraits[] invListTypeTraits,
+ IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
+ IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile,
IPageManagerFactory pageManagerFactory) throws HyracksDataException {
IInvertedListBuilder builder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
FileReference btreeFile = getBTreeFile(ioManager, invListsFile);
- return new OnDiskInvertedIndex(bufferCache, builder, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
- tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory);
+ return new OnDiskInvertedIndex(bufferCache, fileMapProvider, builder, invListTypeTraits, invListCmpFactories,
+ tokenTypeTraits, tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory);
}
public static PartitionedOnDiskInvertedIndex createPartitionedOnDiskInvertedIndex(IIOManager ioManager,
- IBufferCache bufferCache, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
- ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile,
+ IBufferCache bufferCache, IFileMapProvider fileMapProvider, ITypeTraits[] invListTypeTraits,
+ IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
+ IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile,
IPageManagerFactory pageManagerFactory) throws HyracksDataException {
IInvertedListBuilder builder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
FileReference btreeFile = getBTreeFile(ioManager, invListsFile);
- return new PartitionedOnDiskInvertedIndex(bufferCache, builder, invListTypeTraits, invListCmpFactories,
- tokenTypeTraits, tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory);
+ return new PartitionedOnDiskInvertedIndex(bufferCache, fileMapProvider, builder, invListTypeTraits,
+ invListCmpFactories, tokenTypeTraits, tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory);
}
public static FileReference getBTreeFile(IIOManager ioManager, FileReference invListsFile)
@@ -105,46 +108,49 @@
return ioManager.resolveAbsolutePath(invListsFile.getFile().getPath() + "_btree");
}
- public static BTreeFactory createDeletedKeysBTreeFactory(IIOManager ioManager, ITypeTraits[] invListTypeTraits,
- IBinaryComparatorFactory[] invListCmpFactories, IBufferCache diskBufferCache,
- IPageManagerFactory freePageManagerFactory) throws HyracksDataException {
+ public static BTreeFactory createDeletedKeysBTreeFactory(IIOManager ioManager, IFileMapProvider diskFileMapProvider,
+ ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
+ IBufferCache diskBufferCache, IPageManagerFactory freePageManagerFactory) throws HyracksDataException {
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(invListTypeTraits);
ITreeIndexFrameFactory leafFrameFactory =
BTreeUtils.getLeafFrameFactory(tupleWriterFactory, BTreeLeafFrameType.REGULAR_NSM);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- return new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, interiorFrameFactory,
- leafFrameFactory, invListCmpFactories, invListCmpFactories.length);
+ BTreeFactory deletedKeysBTreeFactory =
+ new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+ interiorFrameFactory, leafFrameFactory, invListCmpFactories, invListCmpFactories.length);
+ return deletedKeysBTreeFactory;
}
public static LSMInvertedIndex createLSMInvertedIndex(IIOManager ioManager,
- List<IVirtualBufferCache> virtualBufferCaches, ITypeTraits[] invListTypeTraits,
- IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
- IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
- IBufferCache diskBufferCache, String absoluteOnDiskDir, double bloomFilterFalsePositiveRate,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps,
- int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, IMetadataPageManagerFactory pageManagerFactory)
- throws HyracksDataException {
+ List<IVirtualBufferCache> virtualBufferCaches, IFileMapProvider diskFileMapProvider,
+ ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
+ ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
+ IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String absoluteOnDiskDir,
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
+ int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable,
+ IMetadataPageManagerFactory pageManagerFactory) throws HyracksDataException {
- BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, invListTypeTraits,
- invListCmpFactories, diskBufferCache, pageManagerFactory);
+ BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider,
+ invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory);
int[] bloomFilterKeyFields = new int[invListCmpFactories.length];
for (int i = 0; i < invListCmpFactories.length; i++) {
bloomFilterKeyFields[i] = i;
}
- BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields);
+ BloomFilterFactory bloomFilterFactory =
+ new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
FileReference onDiskDirFileRef = ioManager.resolveAbsolutePath(absoluteOnDiskDir);
- LSMInvertedIndexFileManager fileManager =
- new LSMInvertedIndexFileManager(ioManager, onDiskDirFileRef, deletedKeysBTreeFactory);
+ LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider,
+ onDiskDirFileRef, deletedKeysBTreeFactory);
IInvertedListBuilderFactory invListBuilderFactory =
new FixedSizeElementInvertedListBuilderFactory(invListTypeTraits);
- OnDiskInvertedIndexFactory invIndexFactory =
- new OnDiskInvertedIndexFactory(ioManager, diskBufferCache, invListBuilderFactory, invListTypeTraits,
- invListCmpFactories, tokenTypeTraits, tokenCmpFactories, fileManager, pageManagerFactory);
+ OnDiskInvertedIndexFactory invIndexFactory = new OnDiskInvertedIndexFactory(ioManager, diskBufferCache,
+ diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
+ tokenCmpFactories, fileManager, pageManagerFactory);
ComponentFilterHelper filterHelper = null;
LSMComponentFilterFrameFactory filterFrameFactory = null;
@@ -157,40 +163,42 @@
}
return new LSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory,
bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate,
- fileManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories,
- tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields,
- filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable);
+ fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
+ tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback,
+ invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps,
+ durable);
}
public static PartitionedLSMInvertedIndex createPartitionedLSMInvertedIndex(IIOManager ioManager,
- List<IVirtualBufferCache> virtualBufferCaches, ITypeTraits[] invListTypeTraits,
- IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
- IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
- IBufferCache diskBufferCache, String absoluteOnDiskDir, double bloomFilterFalsePositiveRate,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps,
- int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, IPageManagerFactory pageManagerFactory)
- throws HyracksDataException {
+ List<IVirtualBufferCache> virtualBufferCaches, IFileMapProvider diskFileMapProvider,
+ ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories,
+ ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
+ IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String absoluteOnDiskDir,
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
+ int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable,
+ IPageManagerFactory pageManagerFactory) throws HyracksDataException {
- BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, invListTypeTraits,
- invListCmpFactories, diskBufferCache, pageManagerFactory);
+ BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider,
+ invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory);
int[] bloomFilterKeyFields = new int[invListCmpFactories.length];
for (int i = 0; i < invListCmpFactories.length; i++) {
bloomFilterKeyFields[i] = i;
}
- BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields);
+ BloomFilterFactory bloomFilterFactory =
+ new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
FileReference onDiskDirFileRef = ioManager.resolveAbsolutePath(absoluteOnDiskDir);
- LSMInvertedIndexFileManager fileManager =
- new LSMInvertedIndexFileManager(ioManager, onDiskDirFileRef, deletedKeysBTreeFactory);
+ LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider,
+ onDiskDirFileRef, deletedKeysBTreeFactory);
IInvertedListBuilderFactory invListBuilderFactory =
new FixedSizeElementInvertedListBuilderFactory(invListTypeTraits);
PartitionedOnDiskInvertedIndexFactory invIndexFactory = new PartitionedOnDiskInvertedIndexFactory(ioManager,
- diskBufferCache, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
- tokenCmpFactories, fileManager, pageManagerFactory);
+ diskBufferCache, diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories,
+ tokenTypeTraits, tokenCmpFactories, fileManager, pageManagerFactory);
ComponentFilterHelper filterHelper = null;
LSMComponentFilterFrameFactory filterFrameFactory = null;
@@ -203,8 +211,9 @@
}
return new PartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory,
bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate,
- fileManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories,
- tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields,
- filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable);
+ fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
+ tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback,
+ invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps,
+ durable);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeLocalResource.java
index 3acfe92..6139420 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeLocalResource.java
@@ -67,8 +67,9 @@
IIOManager ioManager = ncServiceCtx.getIoManager();
FileReference fileRef = ioManager.resolve(path);
return LSMRTreeUtils.createExternalRTree(ioManager, fileRef, storageManager.getBufferCache(ncServiceCtx),
- typeTraits, cmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
- bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, ncServiceCtx),
+ storageManager.getFileMapProvider(ncServiceCtx), typeTraits, cmpFactories, btreeCmpFactories,
+ valueProviderFactories, rtreePolicyType, bloomFilterFalsePositiveRate,
+ mergePolicyFactory.createMergePolicy(mergePolicyProperties, ncServiceCtx),
opTrackerProvider.getOperationTracker(ncServiceCtx), ioSchedulerProvider.getIoScheduler(ncServiceCtx),
ioOpCallbackFactory.createIoOpCallback(), linearizeCmpFactory, buddyBTreeFields, durable, isPointMBR,
metadataPageManagerFactory);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeLocalResource.java
index b272dbe..2eecb30 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeLocalResource.java
@@ -84,9 +84,9 @@
FileReference fileRef = ioManager.resolve(path);
List<IVirtualBufferCache> virtualBufferCaches = vbcProvider.getVirtualBufferCaches(ncServiceCtx, fileRef);
return LSMRTreeUtils.createLSMTree(ioManager, virtualBufferCaches, fileRef,
- storageManager.getBufferCache(ncServiceCtx), typeTraits, cmpFactories, btreeCmpFactories,
- valueProviderFactories, rtreePolicyType, bloomFilterFalsePositiveRate,
- mergePolicyFactory.createMergePolicy(mergePolicyProperties, ncServiceCtx),
+ storageManager.getBufferCache(ncServiceCtx), storageManager.getFileMapProvider(ncServiceCtx),
+ typeTraits, cmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
+ bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, ncServiceCtx),
opTrackerProvider.getOperationTracker(ncServiceCtx), ioSchedulerProvider.getIoScheduler(ncServiceCtx),
ioOpCallbackFactory.createIoOpCallback(), linearizeCmpFactory, rtreeFields, buddyBTreeFields,
filterTypeTraits, filterCmpFactories, filterFields, durable, isPointMBR, metadataPageManagerFactory);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterLocalResource.java
index 78f1fd0..429a1cd 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterLocalResource.java
@@ -79,8 +79,8 @@
FileReference file = ioManager.resolve(path);
List<IVirtualBufferCache> virtualBufferCaches = vbcProvider.getVirtualBufferCaches(serviceCtx, file);
return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(ioManager, virtualBufferCaches, file,
- storageManager.getBufferCache(serviceCtx), typeTraits, cmpFactories, btreeComparatorFactories,
- valueProviderFactories, rtreePolicyType,
+ storageManager.getBufferCache(serviceCtx), storageManager.getFileMapProvider(serviceCtx), typeTraits,
+ cmpFactories, btreeComparatorFactories, valueProviderFactories, rtreePolicyType,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
opTrackerProvider.getOperationTracker(serviceCtx), ioSchedulerProvider.getIoScheduler(serviceCtx),
ioOpCallbackFactory.createIoOpCallback(), linearizeCmpFactory, rtreeFields, filterTypeTraits,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index 4f29c89..444fd9d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -56,6 +56,7 @@
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITreeIndex {
@@ -80,24 +81,27 @@
public AbstractLSMRTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
- ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory, int fieldCount,
- IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
- ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
- double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
- IComponentFilterHelper filterHelper, ILSMComponentFilterFrameFactory filterFrameFactory,
- LSMComponentFilterManager filterManager, int[] rtreeFields, int[] filterFields, boolean durable,
- boolean isPointMBR, IBufferCache diskBufferCache) throws HyracksDataException {
- super(ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy,
- opTracker, ioScheduler, ioOpCallback, filterFrameFactory, filterManager, filterFields, durable,
- filterHelper, rtreeFields);
+ ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory,
+ IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories,
+ IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
+ int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, double bloomFilterFalsePositiveRate,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallback ioOpCallback, IComponentFilterHelper filterHelper,
+ ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
+ int[] rtreeFields, int[] filterFields, boolean durable, boolean isPointMBR, IBufferCache diskBufferCache)
+ throws HyracksDataException {
+ super(ioManager, virtualBufferCaches, diskBufferCache, fileManager, diskFileMapProvider,
+ bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
+ filterManager, filterFields, durable, filterHelper, rtreeFields);
int i = 0;
for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
- RTree memRTree = new RTree(virtualBufferCache, new VirtualFreePageManager(virtualBufferCache),
- rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories, fieldCount,
+ RTree memRTree = new RTree(virtualBufferCache, virtualBufferCache.getFileMapProvider(),
+ new VirtualFreePageManager(virtualBufferCache), rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
+ rtreeCmpFactories, fieldCount,
ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_r_" + i), isPointMBR);
- BTree memBTree = new BTree(virtualBufferCache, new VirtualFreePageManager(virtualBufferCache),
- btreeInteriorFrameFactory, btreeLeafFrameFactory, btreeCmpFactories, btreeCmpFactories.length,
+ BTree memBTree = new BTree(virtualBufferCache, virtualBufferCache.getFileMapProvider(),
+ new VirtualFreePageManager(virtualBufferCache), btreeInteriorFrameFactory, btreeLeafFrameFactory,
+ btreeCmpFactories, btreeCmpFactories.length,
ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_b_" + i));
LSMRTreeMemoryComponent mutableComponent =
new LSMRTreeMemoryComponent(memRTree, memBTree, virtualBufferCache, i == 0 ? true : false,
@@ -125,13 +129,14 @@
public AbstractLSMRTree(IIOManager ioManager, ITreeIndexFrameFactory rtreeInteriorFrameFactory,
ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory,
ITreeIndexFrameFactory btreeLeafFrameFactory, ILSMIndexFileManager fileManager,
- ILSMDiskComponentFactory componentFactory, IBinaryComparatorFactory[] rtreeCmpFactories,
- IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
- int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, double bloomFilterFalsePositiveRate,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, boolean durable, boolean isPointMBR, IBufferCache diskBufferCache) {
- super(ioManager, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
- ioScheduler, ioOpCallback, durable);
+ ILSMDiskComponentFactory componentFactory, IFileMapProvider diskFileMapProvider,
+ IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
+ ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean durable,
+ boolean isPointMBR, IBufferCache diskBufferCache) {
+ super(ioManager, diskBufferCache, fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy,
+ opTracker, ioScheduler, ioOpCallback, durable);
this.rtreeInteriorFrameFactory = rtreeInteriorFrameFactory;
this.rtreeLeafFrameFactory = rtreeLeafFrameFactory;
this.btreeInteriorFrameFactory = btreeInteriorFrameFactory;
@@ -324,6 +329,13 @@
}
@Override
+ protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+ LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+ mutableComponent.getRTree().destroy();
+ mutableComponent.getBTree().destroy();
+ }
+
+ @Override
protected LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent,
ILSMDiskComponent lastComponent) throws HyracksDataException {
RTree firstTree = ((LSMRTreeDiskComponent) firstComponent).getRTree();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
index 512e35c..987aec7 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
@@ -62,6 +62,7 @@
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
/**
* This is an lsm r-tree that does not have memory component and is modified
@@ -85,16 +86,17 @@
ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory,
ITreeIndexFrameFactory btreeLeafFrameFactory, ILSMIndexFileManager fileNameManager,
TreeIndexFactory<RTree> diskRTreeFactory, TreeIndexFactory<BTree> diskBTreeFactory,
- BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate, int fieldCount,
- IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
- ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields, boolean durable, boolean isPointMBR) {
+ BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
+ IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories,
+ IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
+ int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ int[] buddyBTreeFields, boolean durable, boolean isPointMBR) {
super(ioManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory,
btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, diskBTreeFactory, bloomFilterFactory,
- bloomFilterFalsePositiveRate, rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields,
- linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, buddyBTreeFields, durable,
- isPointMBR);
+ bloomFilterFalsePositiveRate, diskFileMapProvider, rtreeCmpFactories, btreeCmpFactories, linearizer,
+ comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, buddyBTreeFields,
+ durable, isPointMBR);
this.secondDiskComponents = new LinkedList<>();
this.fieldCount = fieldCount;
}
@@ -343,12 +345,25 @@
cb.afterFinalize(LSMOperationType.FLUSH, null);
}
- for (ILSMDiskComponent c : diskComponents) {
- deactivateDiskComponent(c);
+ for (ILSMComponent c : diskComponents) {
+ LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+ RTree rtree = component.getRTree();
+ BTree btree = component.getBTree();
+ BloomFilter bloomFilter = component.getBloomFilter();
+ rtree.deactivateCloseHandle();
+ btree.deactivateCloseHandle();
+ bloomFilter.deactivate();
}
- for (ILSMDiskComponent c : secondDiskComponents) {
+ for (ILSMComponent c : secondDiskComponents) {
+ // Only deactivate non shared components
if (!diskComponents.contains(c)) {
- deactivateDiskComponent(c);
+ LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+ RTree rtree = component.getRTree();
+ BTree btree = component.getBTree();
+ BloomFilter bloomFilter = component.getBloomFilter();
+ rtree.deactivateCloseHandle();
+ btree.deactivateCloseHandle();
+ bloomFilter.deactivate();
}
}
isActive = false;
@@ -395,13 +410,19 @@
if (isActive) {
throw new HyracksDataException("Failed to destroy the index since it is activated.");
}
- for (ILSMDiskComponent c : diskComponents) {
- destroyDiskComponent(c);
+ for (ILSMComponent c : diskComponents) {
+ LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+ component.getRTree().destroy();
+ component.getBTree().destroy();
+ component.getBloomFilter().destroy();
// Remove from second list to avoid destroying twice
secondDiskComponents.remove(c);
}
- for (ILSMDiskComponent c : secondDiskComponents) {
- destroyDiskComponent(c);
+ for (ILSMComponent c : secondDiskComponents) {
+ LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+ component.getRTree().destroy();
+ component.getBTree().destroy();
+ component.getBloomFilter().destroy();
}
diskComponents.clear();
secondDiskComponents.clear();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 6c1d7ba..56af16d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -73,6 +73,7 @@
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMRTree extends AbstractLSMRTree {
protected final int[] buddyBTreeFields;
@@ -83,18 +84,19 @@
ILSMIndexFileManager fileNameManager, TreeIndexFactory<RTree> diskRTreeFactory,
TreeIndexFactory<BTree> diskBTreeFactory, BloomFilterFactory bloomFilterFactory,
IComponentFilterHelper filterHelper, ILSMComponentFilterFrameFactory filterFrameFactory,
- LSMComponentFilterManager filterManager, double bloomFilterFalsePositiveRate, int fieldCount,
- IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
- ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, int[] rtreeFields, int[] buddyBTreeFields, int[] filterFields,
- boolean durable, boolean isPointMBR) throws HyracksDataException {
+ LSMComponentFilterManager filterManager, double bloomFilterFalsePositiveRate,
+ IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories,
+ IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
+ int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ int[] rtreeFields, int[] buddyBTreeFields, int[] filterFields, boolean durable, boolean isPointMBR)
+ throws HyracksDataException {
super(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager,
new LSMRTreeDiskComponentFactory(diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, filterHelper),
- fieldCount, rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields, linearizerArray,
- bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterHelper,
- filterFrameFactory, filterManager, rtreeFields, filterFields, durable, isPointMBR,
+ diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields,
+ linearizerArray, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback,
+ filterHelper, filterFrameFactory, filterManager, rtreeFields, filterFields, durable, isPointMBR,
diskRTreeFactory.getBufferCache());
this.buddyBTreeFields = buddyBTreeFields;
}
@@ -107,16 +109,17 @@
ITreeIndexFrameFactory btreeLeafFrameFactory, ILSMIndexFileManager fileNameManager,
TreeIndexFactory<RTree> diskRTreeFactory, TreeIndexFactory<BTree> diskBTreeFactory,
BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
- IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
- ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields, boolean durable, boolean isPointMBR) {
+ IFileMapProvider diskFileMapProvider, IBinaryComparatorFactory[] rtreeCmpFactories,
+ IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
+ int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ int[] buddyBTreeFields, boolean durable, boolean isPointMBR) {
super(ioManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory,
btreeLeafFrameFactory, fileNameManager,
new LSMRTreeDiskComponentFactory(diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, null),
- rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields, linearizerArray,
- bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, durable, isPointMBR,
- diskRTreeFactory.getBufferCache());
+ diskFileMapProvider, rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields,
+ linearizerArray, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback,
+ durable, isPointMBR, diskRTreeFactory.getBufferCache());
this.buddyBTreeFields = buddyBTreeFields;
}
@@ -134,12 +137,9 @@
RTree rtree = component.getRTree();
BTree btree = component.getBTree();
BloomFilter bloomFilter = component.getBloomFilter();
- rtree.deactivate();
- btree.deactivate();
+ rtree.deactivateCloseHandle();
+ btree.deactivateCloseHandle();
bloomFilter.deactivate();
- rtree.purge();
- btree.purge();
- bloomFilter.purge();
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
index 4d5eb5b..f9ee5c9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
@@ -37,8 +37,11 @@
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMRTreeFileManager extends AbstractLSMIndexFileManager {
+ public static final String RTREE_STRING = "r";
+ public static final String BTREE_STRING = "b";
private final TreeIndexFactory<? extends ITreeIndex> rtreeFactory;
private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
@@ -46,20 +49,20 @@
private static FilenameFilter btreeFilter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
- return !name.startsWith(".") && name.endsWith(BTREE_SUFFIX);
+ return !name.startsWith(".") && name.endsWith(BTREE_STRING);
}
};
private static FilenameFilter rtreeFilter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
- return !name.startsWith(".") && name.endsWith(RTREE_SUFFIX);
+ return !name.startsWith(".") && name.endsWith(RTREE_STRING);
}
};
- public LSMRTreeFileManager(IIOManager ioManager, FileReference file,
+ public LSMRTreeFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
TreeIndexFactory<? extends ITreeIndex> rtreeFactory, TreeIndexFactory<? extends ITreeIndex> btreeFactory) {
- super(ioManager, file, null);
+ super(ioManager, fileMapProvider, file, null);
this.rtreeFactory = rtreeFactory;
this.btreeFactory = btreeFactory;
}
@@ -67,25 +70,25 @@
@Override
public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException {
String ts = getCurrentTimestamp();
- String baseName = baseDir + ts + DELIMITER + ts;
+ String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + RTREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + BTREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + RTREE_STRING),
+ createFlushFile(baseName + SPLIT_STRING + BTREE_STRING),
+ createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING));
}
@Override
public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName)
throws HyracksDataException {
- String[] firstTimestampRange = firstFileName.split(DELIMITER);
- String[] lastTimestampRange = lastFileName.split(DELIMITER);
+ String[] firstTimestampRange = firstFileName.split(SPLIT_STRING);
+ String[] lastTimestampRange = lastFileName.split(SPLIT_STRING);
- String baseName = baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1];
+ String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1];
// Get the range of timestamps by taking the earliest and the latest
// timestamps
- return new LSMComponentFileReferences(createMergeFile(baseName + DELIMITER + RTREE_SUFFIX),
- createMergeFile(baseName + DELIMITER + BTREE_SUFFIX),
- createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + RTREE_STRING),
+ createMergeFile(baseName + SPLIT_STRING + BTREE_STRING),
+ createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING));
}
@Override
@@ -102,7 +105,7 @@
cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, btreeFilter), btreeFactory, allBTreeFiles);
HashSet<String> btreeFilesSet = new HashSet<>();
for (ComparableFileName cmpFileName : allBTreeFiles) {
- int index = cmpFileName.fileName.lastIndexOf(DELIMITER);
+ int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
}
validateFiles(btreeFilesSet, allRTreeFiles, getCompoundFilter(transactionFilter, rtreeFilter), rtreeFactory);
@@ -200,19 +203,19 @@
public LSMComponentFileReferences getNewTransactionFileReference() throws IOException {
String ts = getCurrentTimestamp();
// Create transaction lock file
- Files.createFile(Paths.get(baseDir + TXN_PREFIX + ts));
+ Files.createFile(Paths.get(baseDir + TRANSACTION_PREFIX + ts));
- String baseName = baseDir + ts + DELIMITER + ts;
- return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + RTREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + BTREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ String baseName = baseDir + ts + SPLIT_STRING + ts;
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + RTREE_STRING),
+ createFlushFile(baseName + SPLIT_STRING + BTREE_STRING),
+ createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING));
}
@Override
public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException {
FilenameFilter transactionFilter;
File dir = new File(baseDir);
- String[] files = dir.list(txnFileNameFilter);
+ String[] files = dir.list(transactionFileNameFilter);
if (files.length == 0) {
return null;
}
@@ -236,11 +239,11 @@
File bTreeFile = null;
File bloomFilterFile = null;
for (String fileName : files) {
- if (fileName.endsWith(BTREE_SUFFIX)) {
+ if (fileName.endsWith(BTREE_STRING)) {
bTreeFile = new File(dir.getPath() + File.separator + fileName);
- } else if (fileName.endsWith(RTREE_SUFFIX)) {
+ } else if (fileName.endsWith(RTREE_STRING)) {
rTreeFile = new File(dir.getPath() + File.separator + fileName);
- } else if (fileName.endsWith(BLOOM_FILTER_SUFFIX)) {
+ } else if (fileName.endsWith(BLOOM_FILTER_STRING)) {
bloomFilterFile = new File(dir.getPath() + File.separator + fileName);
} else {
throw new HyracksDataException("unrecognized file found = " + fileName);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index a5374c9..ae72884 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -69,6 +69,7 @@
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
private static final ICursorFactory cursorFactory = opCtx -> new LSMRTreeWithAntiMatterTuplesSearchCursor(opCtx);
@@ -82,18 +83,19 @@
ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
ILSMIndexFileManager fileManager, TreeIndexFactory<RTree> diskRTreeFactory,
TreeIndexFactory<RTree> bulkLoadRTreeFactory, IComponentFilterHelper filterHelper,
- ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, int fieldCount,
- IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeComparatorFactories,
- ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, int[] rtreeFields, int[] filterFields, boolean durable,
- boolean isPointMBR) throws HyracksDataException {
+ ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
+ IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories,
+ IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizer,
+ int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ int[] rtreeFields, int[] filterFields, boolean durable, boolean isPointMBR) throws HyracksDataException {
super(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
btreeInteriorFrameFactory, btreeLeafFrameFactory, fileManager,
- new LSMRTreeWithAntiMatterTuplesDiskComponentFactory(diskRTreeFactory, filterHelper), fieldCount,
- rtreeCmpFactories, btreeComparatorFactories, linearizer, comparatorFields, linearizerArray, 0,
- mergePolicy, opTracker, ioScheduler, ioOpCallback, filterHelper, filterFrameFactory, filterManager,
- rtreeFields, filterFields, durable, isPointMBR, diskRTreeFactory.getBufferCache());
+ new LSMRTreeWithAntiMatterTuplesDiskComponentFactory(diskRTreeFactory, filterHelper),
+ diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeComparatorFactories, linearizer,
+ comparatorFields, linearizerArray, 0, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterHelper,
+ filterFrameFactory, filterManager, rtreeFields, filterFields, durable, isPointMBR,
+ diskRTreeFactory.getBufferCache());
bulkLoaComponentFactory =
new LSMRTreeWithAntiMatterTuplesDiskComponentFactory(bulkLoadRTreeFactory, filterHelper);
}
@@ -106,8 +108,7 @@
@Override
protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
RTree rtree = ((LSMRTreeDiskComponent) c).getRTree();
- rtree.deactivate();
- rtree.purge();
+ rtree.deactivateCloseHandle();
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFileManager.java
index e39033f..fd1aa71 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFileManager.java
@@ -24,11 +24,13 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMRTreeWithAntiMatterTuplesFileManager extends AbstractLSMIndexFileManager {
- public LSMRTreeWithAntiMatterTuplesFileManager(IIOManager ioManager, FileReference file,
+ public LSMRTreeWithAntiMatterTuplesFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider,
+ FileReference file,
TreeIndexFactory<? extends ITreeIndex> rtreeFactory) {
- super(ioManager, file, rtreeFactory);
+ super(ioManager, fileMapProvider, file, rtreeFactory);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
index 55e70b2..8901c86 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
@@ -27,24 +27,25 @@
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.am.rtree.impls.RTree;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class RTreeFactory extends TreeIndexFactory<RTree> {
private final boolean isPointMBR;
- public RTreeFactory(IIOManager ioManager, IBufferCache bufferCache,
+ public RTreeFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider,
IMetadataPageManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory,
ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount,
boolean isPointMBR) {
- super(ioManager, bufferCache, freePageManagerFactory, interiorFrameFactory, leafFrameFactory, cmpFactories,
- fieldCount);
+ super(ioManager, bufferCache, fileMapProvider, freePageManagerFactory, interiorFrameFactory, leafFrameFactory,
+ cmpFactories, fieldCount);
this.isPointMBR = isPointMBR;
}
@Override
public RTree createIndexInstance(FileReference file) {
- return new RTree(bufferCache, freePageManagerFactory.createPageManager(bufferCache), interiorFrameFactory,
- leafFrameFactory, cmpFactories, fieldCount, file, isPointMBR);
+ return new RTree(bufferCache, fileMapProvider, freePageManagerFactory.createPageManager(bufferCache),
+ interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, file, isPointMBR);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
index 48cd589..e37fa4a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
@@ -68,14 +68,15 @@
import org.apache.hyracks.storage.am.rtree.linearize.ZCurveDoubleComparatorFactory;
import org.apache.hyracks.storage.am.rtree.linearize.ZCurveIntComparatorFactory;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMRTreeUtils {
public static LSMRTree createLSMTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- FileReference file, IBufferCache diskBufferCache, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
- IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
- double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider,
+ ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
+ IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
+ RTreePolicyType rtreePolicyType, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, int[] buddyBTreeFields,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
boolean durable, boolean isPointMBR, IMetadataPageManagerFactory freePageManagerFactory)
@@ -103,10 +104,12 @@
valueProviderFactories, rtreePolicyType, isPointMBR);
ITreeIndexFrameFactory btreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(btreeTupleWriterFactory);
ITreeIndexFrameFactory btreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(btreeTupleWriterFactory);
- TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, freePageManagerFactory,
- rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories, typeTraits.length, isPointMBR);
- TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory,
- btreeInteriorFrameFactory, btreeLeafFrameFactory, btreeCmpFactories, btreeTypeTraits.length);
+ TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
+ freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories,
+ typeTraits.length, isPointMBR);
+ TreeIndexFactory<BTree> diskBTreeFactory =
+ new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+ btreeInteriorFrameFactory, btreeLeafFrameFactory, btreeCmpFactories, btreeTypeTraits.length);
int[] comparatorFields = { 0 };
IBinaryComparatorFactory[] linearizerArray = { linearizeCmpFactory };
@@ -115,7 +118,8 @@
for (int i = 0; i < btreeCmpFactories.length; i++) {
bloomFilterKeyFields[i] = i;
}
- BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields);
+ BloomFilterFactory bloomFilterFactory =
+ new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
ComponentFilterHelper filterHelper = null;
LSMComponentFilterFrameFactory filterFrameFactory = null;
@@ -127,19 +131,19 @@
filterManager = new LSMComponentFilterManager(filterFrameFactory);
}
ILSMIndexFileManager fileNameManager =
- new LSMRTreeFileManager(ioManager, file, diskRTreeFactory, diskBTreeFactory);
+ new LSMRTreeFileManager(ioManager, diskFileMapProvider, file, diskRTreeFactory, diskBTreeFactory);
return new LSMRTree(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, diskBTreeFactory,
bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate,
- typeTraits.length, rtreeCmpFactories, btreeCmpFactories, linearizeCmpFactory, comparatorFields,
- linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, rtreeFields, buddyBTreeFields,
- filterFields, durable, isPointMBR);
+ diskFileMapProvider, typeTraits.length, rtreeCmpFactories, btreeCmpFactories, linearizeCmpFactory,
+ comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, rtreeFields,
+ buddyBTreeFields, filterFields, durable, isPointMBR);
}
public static LSMRTreeWithAntiMatterTuples createLSMTreeWithAntiMatterTuples(IIOManager ioManager,
List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
- ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
- IBinaryComparatorFactory[] btreeComparatorFactories,
+ IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeComparatorFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, ILinearizeComparatorFactory linearizerCmpFactory, int[] rtreeFields,
@@ -180,12 +184,13 @@
ITreeIndexFrameFactory copyTupleLeafFrameFactory = new RTreeNSMLeafFrameFactory(
rtreeLeafFrameCopyTupleWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
- TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, freePageManagerFactory,
- rtreeInteriorFrameFactory, copyTupleLeafFrameFactory, rtreeCmpFactories, typeTraits.length, isPointMBR);
+ TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
+ freePageManagerFactory, rtreeInteriorFrameFactory, copyTupleLeafFrameFactory, rtreeCmpFactories,
+ typeTraits.length, isPointMBR);
- TreeIndexFactory<RTree> bulkLoadRTreeFactory =
- new RTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, rtreeInteriorFrameFactory,
- rtreeLeafFrameBulkLoadFactory, rtreeCmpFactories, typeTraits.length, isPointMBR);
+ TreeIndexFactory<RTree> bulkLoadRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
+ freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameBulkLoadFactory, rtreeCmpFactories,
+ typeTraits.length, isPointMBR);
// The first field is for the sorted curve (e.g. Hilbert curve), and the
// second field is for the primary key.
@@ -214,20 +219,21 @@
filterManager = new LSMComponentFilterManager(filterFrameFactory);
}
ILSMIndexFileManager fileNameManager =
- new LSMRTreeWithAntiMatterTuplesFileManager(ioManager, file, diskRTreeFactory);
+ new LSMRTreeWithAntiMatterTuplesFileManager(ioManager, diskFileMapProvider, file, diskRTreeFactory);
return new LSMRTreeWithAntiMatterTuples(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory,
rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager,
diskRTreeFactory, bulkLoadRTreeFactory, filterHelper, filterFrameFactory, filterManager,
- typeTraits.length, rtreeCmpFactories, btreeComparatorFactories, linearizerCmpFactory, comparatorFields,
- linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, rtreeFields, filterFields, durable,
- isPointMBR);
+ diskFileMapProvider, typeTraits.length, rtreeCmpFactories, btreeComparatorFactories,
+ linearizerCmpFactory, comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler,
+ ioOpCallback, rtreeFields, filterFields, durable, isPointMBR);
}
public static ExternalRTree createExternalRTree(IIOManager ioManager, FileReference file,
- IBufferCache diskBufferCache, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
- IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
- ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
+ IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
ILinearizeComparatorFactory linearizeCmpFactory, int[] buddyBTreeFields, boolean durable,
boolean isPointMBR, IMetadataPageManagerFactory freePageManagerFactory) throws HyracksDataException {
@@ -254,10 +260,12 @@
valueProviderFactories, rtreePolicyType, isPointMBR);
ITreeIndexFrameFactory btreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(btreeTupleWriterFactory);
ITreeIndexFrameFactory btreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(btreeTupleWriterFactory);
- TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, freePageManagerFactory,
- rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories, typeTraits.length, isPointMBR);
- TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory,
- btreeInteriorFrameFactory, btreeLeafFrameFactory, btreeCmpFactories, btreeTypeTraits.length);
+ TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
+ freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories,
+ typeTraits.length, isPointMBR);
+ TreeIndexFactory<BTree> diskBTreeFactory =
+ new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+ btreeInteriorFrameFactory, btreeLeafFrameFactory, btreeCmpFactories, btreeTypeTraits.length);
int[] comparatorFields = { 0 };
IBinaryComparatorFactory[] linearizerArray = { linearizeCmpFactory };
@@ -265,15 +273,17 @@
for (int i = 0; i < btreeCmpFactories.length; i++) {
bloomFilterKeyFields[i] = i;
}
- BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields);
+ BloomFilterFactory bloomFilterFactory =
+ new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
ILSMIndexFileManager fileNameManager =
- new LSMRTreeFileManager(ioManager, file, diskRTreeFactory, diskBTreeFactory);
- return new ExternalRTree(ioManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory,
- btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, diskBTreeFactory, bloomFilterFactory,
- bloomFilterFalsePositiveRate, typeTraits.length, rtreeCmpFactories, btreeCmpFactories,
- linearizeCmpFactory, comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler,
- ioOpCallback, buddyBTreeFields, durable, isPointMBR);
+ new LSMRTreeFileManager(ioManager, diskFileMapProvider, file, diskRTreeFactory, diskBTreeFactory);
+ ExternalRTree lsmTree = new ExternalRTree(ioManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
+ btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, diskBTreeFactory,
+ bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length,
+ rtreeCmpFactories, btreeCmpFactories, linearizeCmpFactory, comparatorFields, linearizerArray,
+ mergePolicy, opTracker, ioScheduler, ioOpCallback, buddyBTreeFields, durable, isPointMBR);
+ return lsmTree;
}
public static ILinearizeComparatorFactory proposeBestLinearizer(ITypeTraits[] typeTraits, int numKeyFields)
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeResource.java
index df4fbf2..5dc0a4e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeResource.java
@@ -59,8 +59,9 @@
public IIndex createInstance(INCServiceContext ctx) throws HyracksDataException {
IIOManager ioManager = ctx.getIoManager();
FileReference resourceRef = ioManager.resolve(path);
- return RTreeUtils.createRTree(storageManager.getBufferCache(ctx), typeTraits, valueProviderFactories,
- comparatorFactories, rtreePolicyType, resourceRef, false, pageManagerFactory);
+ return RTreeUtils.createRTree(storageManager.getBufferCache(ctx), storageManager.getFileMapProvider(ctx),
+ typeTraits, valueProviderFactories, comparatorFactories, rtreePolicyType, resourceRef, false,
+ pageManagerFactory);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
index 1750067..1160da7 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
@@ -60,6 +60,7 @@
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class RTree extends AbstractTreeIndex {
@@ -69,10 +70,11 @@
private final int maxTupleSize;
private final boolean isPointMBR; // used for reducing storage space to store point objects.
- public RTree(IBufferCache bufferCache, IPageManager freePageManager, ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount,
- FileReference file, boolean isPointMBR) {
- super(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, file);
+ public RTree(IBufferCache bufferCache, IFileMapProvider fileMapProvider, IPageManager freePageManager,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
+ IBinaryComparatorFactory[] cmpFactories, int fieldCount, FileReference file, boolean isPointMBR) {
+ super(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
+ fieldCount, file);
globalNsn = new AtomicLong();
ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
ITreeIndexFrame interiorFrame = interiorFrameFactory.createFrame();
@@ -99,7 +101,7 @@
public void printTree(int pageId, ICachedPage parent, boolean unpin, IRTreeLeafFrame leafFrame,
IRTreeInteriorFrame interiorFrame, byte treeHeight, ISerializerDeserializer[] keySerdes,
StringBuilder strBuilder, MultiComparator cmp) throws Exception {
- ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), pageId), false);
+ ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
node.acquireReadLatch();
try {
if (parent != null && unpin == true) {
@@ -169,7 +171,7 @@
while (true) {
if (!writeLatched) {
- node = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), pageId), false);
+ node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
ctx.getInteriorFrame().setPage(node);
isLeaf = ctx.getInteriorFrame().isLeaf();
if (isLeaf) {
@@ -227,7 +229,7 @@
readLatched = false;
bufferCache.unpin(node);
- node = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), pageId), false);
+ node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
node.acquireWriteLatch();
writeLatched = true;
ctx.getInteriorFrame().setPage(node);
@@ -343,8 +345,7 @@
case INSUFFICIENT_SPACE: {
int rightPageId = freePageManager.takePage(ctx.getMetaFrame());
- ICachedPage rightNode =
- bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), rightPageId), true);
+ ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true);
rightNode.acquireWriteLatch();
try {
@@ -389,7 +390,7 @@
if (pageId == rootPage) {
int newLeftId = freePageManager.takePage(ctx.getMetaFrame());
ICachedPage newLeftNode =
- bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), newLeftId), true);
+ bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, newLeftId), true);
newLeftNode.acquireWriteLatch();
succeeded = false;
try {
@@ -447,7 +448,7 @@
boolean succeeded = false;
boolean writeLatched = false;
int parentId = ctx.getPathList().getLastPageId();
- ICachedPage parentNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), parentId), false);
+ ICachedPage parentNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, parentId), false);
parentNode.acquireWriteLatch();
writeLatched = true;
ctx.getInteriorFrame().setPage(parentNode);
@@ -473,7 +474,7 @@
}
parentId = rightPage;
- parentNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), parentId), false);
+ parentNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, parentId), false);
parentNode.acquireWriteLatch();
writeLatched = true;
ctx.getInteriorFrame().setPage(parentNode);
@@ -527,7 +528,7 @@
pageId = ctx.getTraverseList().getFirstPageId();
parentIndex = ctx.getTraverseList().getFirstPageIndex();
- node = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), pageId), false);
+ node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
node.acquireReadLatch();
readLatched = true;
ctx.getInteriorFrame().setPage(node);
@@ -610,7 +611,7 @@
int pageId = ctx.getPathList().getLastPageId();
long parentLsn = ctx.getPathList().getLastPageLsn();
ctx.getPathList().moveLast();
- node = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), pageId), false);
+ node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
node.acquireReadLatch();
readLatched = true;
ctx.getInteriorFrame().setPage(node);
@@ -643,7 +644,7 @@
readLatched = false;
bufferCache.unpin(node);
- node = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), pageId), false);
+ node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
node.acquireWriteLatch();
writeLatched = true;
ctx.getLeafFrame().setPage(node);
@@ -713,7 +714,7 @@
ctx.setCursor(cursor);
cursor.setBufferCache(bufferCache);
- cursor.setFileId(getFileId());
+ cursor.setFileId(fileId);
ctx.getCursorInitialState().setRootPage(rootPage);
ctx.getCursor().open(ctx.getCursorInitialState(), searchPred);
}
@@ -732,11 +733,11 @@
int currentPageId = bulkloadLeafStart;
int maxPageId = freePageManager.getMaxPageId(ctx.getMetaFrame());
- ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), currentPageId), false);
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false);
page.acquireReadLatch();
try {
cursor.setBufferCache(bufferCache);
- cursor.setFileId(getFileId());
+ cursor.setFileId(fileId);
cursor.setCurrentPageId(currentPageId);
cursor.setMaxPageId(maxPageId);
ctx.getCursorInitialState().setOriginialKeyComparator(ctx.getCmp());
@@ -937,8 +938,8 @@
}
pagesToWrite.clear();
- leafFrontier.page = bufferCache
- .confiscatePage(BufferedFileHandle.getDiskPageId(getFileId(), leafFrontier.pageId));
+ leafFrontier.page =
+ bufferCache.confiscatePage(BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId));
leafFrame.setPage(leafFrontier.page);
leafFrame.initBuffer((byte) 0);
@@ -991,7 +992,7 @@
int finalPageId = freePageManager.takePage(metaFrame);
n.pageId = finalPageId;
- bufferCache.setPageDiskId(n.page, BufferedFileHandle.getDiskPageId(getFileId(), finalPageId));
+ bufferCache.setPageDiskId(n.page, BufferedFileHandle.getDiskPageId(fileId, finalPageId));
//else we are looking at a leaf
}
//set next guide MBR
@@ -1054,7 +1055,7 @@
} else {
prevNodeFrontierPages.set(level, finalPageId);
}
- bufferCache.setPageDiskId(frontier.page, BufferedFileHandle.getDiskPageId(getFileId(), finalPageId));
+ bufferCache.setPageDiskId(frontier.page, BufferedFileHandle.getDiskPageId(fileId, finalPageId));
pagesToWrite.add(frontier.page);
lowerFrame = prevInteriorFrame;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/util/RTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/util/RTreeUtils.java
index 650b6a8..a8e668b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/util/RTreeUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/util/RTreeUtils.java
@@ -36,20 +36,23 @@
import org.apache.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriterFactory;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class RTreeUtils {
- public static RTree createRTree(IBufferCache bufferCache, ITypeTraits[] typeTraits,
- IPrimitiveValueProviderFactory[] valueProviderFactories, IBinaryComparatorFactory[] cmpFactories,
- RTreePolicyType rtreePolicyType, FileReference file, boolean isPointMBR,
- IPageManagerFactory pageManagerFactory) {
+ public static RTree createRTree(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ ITypeTraits[] typeTraits, IPrimitiveValueProviderFactory[] valueProviderFactories,
+ IBinaryComparatorFactory[] cmpFactories, RTreePolicyType rtreePolicyType, FileReference file,
+ boolean isPointMBR, IPageManagerFactory pageManagerFactory) {
RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(typeTraits);
ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(tupleWriterFactory,
valueProviderFactories, rtreePolicyType, isPointMBR);
- ITreeIndexFrameFactory leafFrameFactory =
- new RTreeNSMLeafFrameFactory(tupleWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
- return new RTree(bufferCache, pageManagerFactory.createPageManager(bufferCache), interiorFrameFactory,
- leafFrameFactory, cmpFactories, typeTraits.length, file, isPointMBR);
+ ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(tupleWriterFactory,
+ valueProviderFactories, rtreePolicyType, isPointMBR);
+ RTree rtree = new RTree(bufferCache, fileMapProvider, pageManagerFactory.createPageManager(bufferCache),
+ interiorFrameFactory, leafFrameFactory,
+ cmpFactories, typeTraits.length, file, isPointMBR);
+ return rtree;
}
// Creates a new MultiComparator by constructing new IBinaryComparators.
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IStorageManager.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IStorageManager.java
index dfc2f41..68f4cbb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IStorageManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IStorageManager.java
@@ -22,6 +22,7 @@
import org.apache.hyracks.api.application.INCServiceContext;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.storage.common.file.IResourceIdFactory;
/**
@@ -38,6 +39,13 @@
/**
* @param ctx
* the nc service context
+ * @return the file map provider {@link org.apache.hyracks.storage.common.file.IFileMapProvider}
+ */
+ IFileMapProvider getFileMapProvider(INCServiceContext ctx);
+
+ /**
+ * @param ctx
+ * the nc service context
* @return the local resource repository {@link org.apache.hyracks.storage.am.common.build.ILocalResourceRepository}
*/
ILocalResourceRepository getLocalResourceRepository(INCServiceContext ctx);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
index d4e6c77..8391e5c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
@@ -24,10 +24,12 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Executor;
@@ -45,7 +47,6 @@
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.lifecycle.ILifeCycleComponent;
import org.apache.hyracks.api.replication.IIOReplicationManager;
-import org.apache.hyracks.api.util.IoUtil;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
import org.apache.hyracks.storage.common.file.IFileMapManager;
@@ -107,7 +108,7 @@
closed = false;
fifoWriter = new AsyncFIFOPageQueueManager(this);
- if (DEBUG) {
+ if ( DEBUG ) {
confiscatedPages = new ArrayList<>();
confiscatedPagesOwner = new HashMap<>();
confiscateLock = new ReentrantLock();
@@ -217,7 +218,7 @@
cPage.valid = true;
}
pageReplacementStrategy.notifyCachePageAccess(cPage);
- if (DEBUG) {
+ if (DEBUG){
pinnedPageOwner.put(cPage, Thread.currentThread().getStackTrace());
}
return cPage;
@@ -334,9 +335,9 @@
}
int victimHash = hash(victim.dpid);
if (victimHash == hash) {
- /*
- * Case 2a.
- */
+ /*
+ * Case 2a.
+ */
bucket.bucketLock.lock();
try {
if (!victim.pinCount.compareAndSet(0, 1)) {
@@ -371,9 +372,9 @@
}
return victim;
} else {
- /*
- * Case 2b.
- */
+ /*
+ * Case 2b.
+ */
CacheBucket victimBucket = pageMap[victimHash];
if (victimHash < hash) {
victimBucket.bucketLock.lock();
@@ -474,7 +475,7 @@
}
}
buffer.append("Number of cached pages: ").append(nCachedPages).append('\n');
- if (DEBUG) {
+ if (DEBUG){
confiscateLock.lock();
try {
buffer.append("Number of confiscated pages: ").append(confiscatedPages.size()).append('\n');
@@ -486,28 +487,29 @@
}
@Override
- public boolean isClean() {
+ public boolean isClean(){
List<Long> reachableDpids = new LinkedList<>();
synchronized (cachedPages) {
for (ICachedPageInternal internalPage : cachedPages) {
- CachedPage c = (CachedPage) internalPage;
- if (c.confiscated() || c.latch.getReadLockCount() != 0 || c.latch.getWriteHoldCount() != 0) {
+ CachedPage c = (CachedPage) internalPage;
+ if (c.confiscated() ||
+ c.latch.getReadLockCount() != 0 || c.latch.getWriteHoldCount() != 0) {
return false;
}
- if (c.valid) {
+ if (c.valid){
reachableDpids.add(c.dpid);
}
}
}
- for (Long l : reachableDpids) {
- if (!canFindValidCachedPage(l)) {
+ for(Long l: reachableDpids){
+ if (!canFindValidCachedPage(l)){
return false;
}
}
return true;
}
- private boolean canFindValidCachedPage(long dpid) {
+ private boolean canFindValidCachedPage(long dpid){
int hash = hash(dpid);
CachedPage cPage = null;
CacheBucket bucket = pageMap[hash];
@@ -570,16 +572,14 @@
}
BufferedFileHandle getFileInfo(int fileId) throws HyracksDataException {
- BufferedFileHandle fInfo;
synchronized (fileInfoMap) {
- fInfo = fileInfoMap.get(fileId);
+ BufferedFileHandle fInfo = fileInfoMap.get(fileId);
+ if (fInfo == null) {
+ throw new HyracksDataException("No such file mapped");
+ }
+ return fInfo;
}
- if (fInfo == null) {
- throw new HyracksDataException("No such file mapped");
- }
- return fInfo;
}
-
private BufferCacheHeaderHelper checkoutHeaderHelper() {
BufferCacheHeaderHelper helper = headerPageCache.poll();
if (helper == null) {
@@ -609,8 +609,8 @@
getOffsetForPage(BufferedFileHandle.getPageId(cPage.dpid)),
header.prepareWrite(cPage, buf));
- if (bytesWritten != (contiguousLargePages ? pageSize * (totalPages - 1) : 0)
- + getPageSizeWithHeader()) {
+ if (bytesWritten !=
+ (contiguousLargePages ? pageSize * (totalPages - 1) : 0) + getPageSizeWithHeader()) {
throw new HyracksDataException("Failed to write completely: " + bytesWritten);
}
} finally {
@@ -636,6 +636,7 @@
}
}
+
private int hash(long dpid) {
int hashValue = (int) dpid ^ (Integer.reverse((int) (dpid >>> 32)) >>> 1);
return hashValue % pageMap.length;
@@ -789,39 +790,13 @@
}
@Override
- public int createFile(FileReference fileRef) throws HyracksDataException {
+ public void createFile(FileReference fileRef) throws HyracksDataException {
if (LOGGER.isLoggable(fileOpsLevel)) {
LOGGER.log(fileOpsLevel, "Creating file: " + fileRef + " in cache: " + this);
}
- IoUtil.create(fileRef);
- try {
- return fileMapManager.registerFile(fileRef);
- } catch (Exception e) {
- // If file registration failed for any reason, we need to undo the file creation
- try {
- IoUtil.delete(fileRef);
- } catch (Exception deleteException) {
- e.addSuppressed(deleteException);
- }
- throw HyracksDataException.create(e);
- }
- }
-
- @Override
- public int openFile(FileReference fileRef) throws HyracksDataException {
- if (LOGGER.isLoggable(fileOpsLevel)) {
- LOGGER.log(fileOpsLevel, "Opening file: " + fileRef + " in cache: " + this);
- }
- int fileId;
synchronized (fileInfoMap) {
- if (fileMapManager.isMapped(fileRef)) {
- fileId = fileMapManager.lookupFileId(fileRef);
- } else {
- fileId = fileMapManager.registerFile(fileRef);
- }
- openFile(fileId);
+ fileMapManager.registerFile(fileRef);
}
- return fileId;
}
@Override
@@ -833,9 +808,10 @@
BufferedFileHandle fInfo;
fInfo = fileInfoMap.get(fileId);
if (fInfo == null) {
+
+ // map is full, make room by cleaning up unreferenced files
boolean unreferencedFileFound = true;
while (fileInfoMap.size() >= maxOpenFiles && unreferencedFileFound) {
- // map is full, make room by cleaning up unreferenced files
unreferencedFileFound = false;
for (Map.Entry<Integer, BufferedFileHandle> entry : fileInfoMap.entrySet()) {
if (entry.getValue().getReferenceCount() <= 0) {
@@ -853,10 +829,12 @@
}
}
}
+
if (fileInfoMap.size() >= maxOpenFiles) {
throw new HyracksDataException("Could not open fileId " + fileId + ". Max number of files "
+ maxOpenFiles + " already opened and referenced.");
}
+
// create, open, and map new file reference
FileReference fileRef = fileMapManager.lookupFileName(fileId);
IFileHandle fh = ioManager.open(fileRef, IIOManager.FileReadWriteMode.READ_WRITE,
@@ -961,49 +939,30 @@
}
@Override
- public void deleteFile(FileReference fileRef) throws HyracksDataException {
- synchronized (fileInfoMap) {
- if (fileMapManager.isMapped(fileRef)) {
- int fileId = fileMapManager.lookupFileId(fileRef);
- deleteFile(fileId);
- return;
- } else {
- IoUtil.delete(fileRef);
- }
- }
- }
-
- @Override
- public void deleteFile(int fileId) throws HyracksDataException {
+ public synchronized void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException {
if (LOGGER.isLoggable(fileOpsLevel)) {
LOGGER.log(fileOpsLevel, "Deleting file: " + fileId + " in cache: " + this);
}
synchronized (fileInfoMap) {
- sweepAndFlush(fileId, false);
+ sweepAndFlush(fileId, flushDirtyPages);
BufferedFileHandle fInfo = null;
try {
fInfo = fileInfoMap.get(fileId);
if (fInfo != null && fInfo.getReferenceCount() > 0) {
throw new HyracksDataException("Deleting open file");
}
- } catch (Exception e) {
- throw HyracksDataException.create(e);
} finally {
- FileReference fileRef = fileMapManager.unregisterFile(fileId);
- try {
- if (fInfo != null) {
- // Mark the fInfo as deleted,
- // such that when its pages are reclaimed in openFile(),
- // the pages are not flushed to disk but only invalidated.
- synchronized (fInfo) {
- if (!fInfo.fileHasBeenDeleted()) {
- ioManager.close(fInfo.getFileHandle());
- fInfo.markAsDeleted();
- }
+ fileMapManager.unregisterFile(fileId);
+ if (fInfo != null) {
+ // Mark the fInfo as deleted,
+ // such that when its pages are reclaimed in openFile(),
+ // the pages are not flushed to disk but only invalidated.
+ synchronized (fInfo) {
+ if (!fInfo.fileHasBeenDeleted()) {
+ ioManager.close(fInfo.getFileHandle());
+ fInfo.markAsDeleted();
}
}
- } finally {
- IoUtil.delete(fileRef);
}
}
}
@@ -1052,7 +1011,7 @@
@Override
public boolean removePage(ICachedPageInternal victimPage) {
- CachedPage victim = (CachedPage) victimPage;
+ CachedPage victim = (CachedPage)victimPage;
// Case 1 from findPage()
if (victim.dpid < 0) { // new page
if (!victim.pinCount.compareAndSet(0, 1)) {
@@ -1120,7 +1079,7 @@
ICachedPageInternal old = cachedPages.set(victim.cpid, null);
if (DEBUG) {
assert old == victim;
- } ;
+ };
}
return true;
}
@@ -1158,11 +1117,12 @@
public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId)
throws HyracksDataException {
ICachedPage cachedPage = confiscatePage(dpid, multiplier);
- ((ICachedPageInternal) cachedPage).setExtraBlockPageId(extraBlockPageId);
+ ((ICachedPageInternal)cachedPage).setExtraBlockPageId(extraBlockPageId);
return cachedPage;
}
- private ICachedPage confiscatePage(long dpid, int multiplier) throws HyracksDataException {
+ private ICachedPage confiscatePage(long dpid, int multiplier)
+ throws HyracksDataException {
return getPageLoop(dpid, multiplier, true);
}
@@ -1254,7 +1214,8 @@
return null;
}
- private ICachedPage getPageLoop(long dpid, int multiplier, boolean confiscate) throws HyracksDataException {
+ private ICachedPage getPageLoop(long dpid, int multiplier, boolean confiscate)
+ throws HyracksDataException {
final long startingPinCount = DEBUG ? masterPinCount.get() : -1;
int cycleCount = 0;
try {
@@ -1298,16 +1259,16 @@
if (cycleCount > MAX_PIN_ATTEMPT_CYCLES) {
cycleCount = 0; // suppress warning below
throw new HyracksDataException("Unable to find free page in buffer cache after "
- + MAX_PIN_ATTEMPT_CYCLES + " cycles (buffer cache undersized?)"
- + (DEBUG ? " ; " + (masterPinCount.get() - startingPinCount)
- + " successful pins since start of cycle" : ""));
+ + MAX_PIN_ATTEMPT_CYCLES + " cycles (buffer cache undersized?)" + (DEBUG ? " ; "
+ + (masterPinCount.get() - startingPinCount) + " successful pins since start of cycle"
+ : ""));
}
}
} finally {
if (cycleCount > PIN_ATTEMPT_CYCLES_WARNING_THRESHOLD && LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.warning("Took " + cycleCount + " cycles to find free page in buffer cache. (buffer cache "
- + "undersized?)" + (DEBUG ? " ; " + (masterPinCount.get() - startingPinCount)
- + " successful pins since start of cycle" : ""));
+ LOGGER.warning("Took " + cycleCount + " cycles to find free page in buffer cache. (buffer cache " +
+ "undersized?)" + (DEBUG ? " ; " + (masterPinCount.get() - startingPinCount) +
+ " successful pins since start of cycle" : ""));
}
}
}
@@ -1321,7 +1282,7 @@
public void returnPage(ICachedPage page, boolean reinsert) {
CachedPage cPage = (CachedPage) page;
CacheBucket bucket;
- if (!page.confiscated()) {
+ if (!page.confiscated()){
return;
}
if (reinsert) {
@@ -1337,7 +1298,7 @@
cPage.next = bucket.cachedPage;
bucket.cachedPage = cPage;
cPage.pinCount.decrementAndGet();
- if (DEBUG) {
+ if (DEBUG){
assert cPage.pinCount.get() == 0;
assert cPage.latch.getReadLockCount() == 0;
assert cPage.latch.getWriteHoldCount() == 0;
@@ -1353,7 +1314,7 @@
} else {
cPage.invalidate();
cPage.pinCount.decrementAndGet();
- if (DEBUG) {
+ if (DEBUG){
assert cPage.pinCount.get() == 0;
assert cPage.latch.getReadLockCount() == 0;
assert cPage.latch.getWriteHoldCount() == 0;
@@ -1402,23 +1363,23 @@
* _ONLY_ call this if you absolutely, positively know this file has no dirty pages in the cache!
* Bypasses the normal lifecycle of a file handle and evicts all references to it immediately.
*/
- public void purgeHandle(int fileId) throws HyracksDataException {
- synchronized (fileInfoMap) {
- BufferedFileHandle fh = fileInfoMap.get(fileId);
- if (fh != null) {
- ioManager.close(fh.getFileHandle());
- fileInfoMap.remove(fileId);
- fileMapManager.unregisterFile(fileId);
- }
+ public void purgeHandle(int fileId) throws HyracksDataException{
+ synchronized(fileInfoMap){
+ BufferedFileHandle fh = fileInfoMap.get(fileId);
+ if (fh != null){
+ ioManager.close(fh.getFileHandle());
+ fileInfoMap.remove(fileId);
+ fileMapManager.unregisterFile(fileId);
+ }
}
}
static class BufferCacheHeaderHelper {
private static final int FRAME_MULTIPLIER_OFF = 0;
- private static final int EXTRA_BLOCK_PAGE_ID_OFF = FRAME_MULTIPLIER_OFF + 4; // 4
+ private static final int EXTRA_BLOCK_PAGE_ID_OFF = FRAME_MULTIPLIER_OFF + 4; // 4
private final ByteBuffer buf;
- private final ByteBuffer[] array;
+ private final ByteBuffer [] array;
private BufferCacheHeaderHelper(int pageSize) {
buf = ByteBuffer.allocate(RESERVED_HEADER_BYTES + pageSize);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
index 8f7a965..4461069 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
@@ -52,10 +52,9 @@
}
@Override
- public int createFile(FileReference fileRef) throws HyracksDataException {
- int fileId = bufferCache.createFile(fileRef);
+ public void createFile(FileReference fileRef) throws HyracksDataException {
+ bufferCache.createFile(fileRef);
createFileCount.addAndGet(1);
- return fileId;
}
@Override
@@ -71,8 +70,8 @@
}
@Override
- public void deleteFile(int fileId) throws HyracksDataException {
- bufferCache.deleteFile(fileId);
+ public void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException {
+ bufferCache.deleteFile(fileId, flushDirtyPages);
deleteFileCount.addAndGet(1);
}
@@ -250,16 +249,4 @@
throws HyracksDataException {
bufferCache.resizePage(page, multiplier, extraPageBlockHelper);
}
-
- @Override
- public int openFile(FileReference fileRef) throws HyracksDataException {
- openFileCount.incrementAndGet();
- return bufferCache.openFile(fileRef);
- }
-
- @Override
- public void deleteFile(FileReference file) throws HyracksDataException {
- deleteFileCount.incrementAndGet();
- bufferCache.deleteFile(file);
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
index 789f7b7..a3fc0e2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
@@ -28,65 +28,13 @@
int INVALID_PAGEID = -1;
int RESERVED_HEADER_BYTES = 8;
- /**
- * Create file on disk
- *
- * @param fileRef
- * the file to create
- * @return the file id
- * @throws HyracksDataException
- * if the file already exists or attempt to create the file failed
- */
- int createFile(FileReference fileRef) throws HyracksDataException;
+ void createFile(FileReference fileRef) throws HyracksDataException;
- /**
- * Open the file and register it (if not registered) with the file map manager
- *
- * @param fileRef
- * the file to open
- * @return the file id
- * @throws HyracksDataException
- * if the file doesn't exist or buffer cache failed to open the file
- */
- int openFile(FileReference fileRef) throws HyracksDataException;
-
- /**
- * Open the mapped file with the passed file id
- *
- * @param fileId
- * the file id
- * @throws HyracksDataException
- * if the file doesn't exist or buffer cache fails to open the file
- */
void openFile(int fileId) throws HyracksDataException;
- /**
- * close the file
- *
- * @param fileId
- * the file id
- * @throws HyracksDataException
- * if file doesn't exist or is not open
- */
void closeFile(int fileId) throws HyracksDataException;
- /**
- * delete the file from memory and disk
- *
- * @param fileId
- * the file id
- * @throws HyracksDataException
- * if the file doesn't exist or if a failure to delete takes place
- */
- void deleteFile(int fileId) throws HyracksDataException;
-
- /**
- * Delete from memory if registered and from disk
- *
- * @param file
- * @throws HyracksDataException
- */
- void deleteFile(FileReference file) throws HyracksDataException;
+ void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException;
ICachedPage tryPin(long dpid) throws HyracksDataException;
@@ -126,9 +74,9 @@
void setPageDiskId(ICachedPage page, long dpid);
- boolean isReplicationEnabled();
+ public boolean isReplicationEnabled();
- IIOReplicationManager getIOReplicationManager();
+ public IIOReplicationManager getIOReplicationManager();
void purgeHandle(int fileId) throws HyracksDataException;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/IFileMapManager.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/IFileMapManager.java
index 9a633b3..0d81776 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/IFileMapManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/IFileMapManager.java
@@ -34,9 +34,8 @@
* - file reference to register
* @throws HyracksDataException
* - if a mapping for the file already exists.
- * @return the file id
*/
- int registerFile(FileReference fileRef) throws HyracksDataException;
+ public void registerFile(FileReference fileRef) throws HyracksDataException;
/**
* Unregister a file mapping
@@ -45,8 +44,7 @@
* - The file id whose mapping is to be unregistered.
* @throws HyracksDataException
* - If the fileid is not mapped currently in this manager.
- * @return the file reference
*/
- FileReference unregisterFile(int fileId) throws HyracksDataException;
+ public void unregisterFile(int fileId) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/FileMapManager.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/TransientFileMapManager.java
similarity index 64%
rename from hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/FileMapManager.java
rename to hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/TransientFileMapManager.java
index 209974e..66d54b1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/FileMapManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/TransientFileMapManager.java
@@ -18,25 +18,24 @@
*/
package org.apache.hyracks.storage.common.file;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
+import java.util.HashMap;
+import java.util.Map;
-import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
-public class FileMapManager implements IFileMapManager {
+public class TransientFileMapManager implements IFileMapManager {
private static final long serialVersionUID = 1L;
- private ConcurrentMap<Integer, FileReference> id2nameMap = new ConcurrentHashMap<>();
- private ConcurrentMap<FileReference, Integer> name2IdMap = new ConcurrentHashMap<>();
+ private Map<Integer, FileReference> id2nameMap = new HashMap<Integer, FileReference>();
+ private Map<FileReference, Integer> name2IdMap = new HashMap<FileReference, Integer>();
private int idCounter = 0;
@Override
public FileReference lookupFileName(int fileId) throws HyracksDataException {
FileReference fRef = id2nameMap.get(fileId);
if (fRef == null) {
- throw HyracksDataException.create(ErrorCode.NO_MAPPING_FOR_FILE_ID, fileId);
+ throw new HyracksDataException("No mapping found for id: " + fileId);
}
return fRef;
}
@@ -45,7 +44,7 @@
public int lookupFileId(FileReference fileRef) throws HyracksDataException {
Integer fileId = name2IdMap.get(fileRef);
if (fileId == null) {
- throw HyracksDataException.create(ErrorCode.NO_MAPPING_FOR_FILENAME, fileRef);
+ throw new HyracksDataException("No mapping found for name: " + fileRef);
}
return fileId;
}
@@ -61,24 +60,16 @@
}
@Override
- public synchronized FileReference unregisterFile(int fileId) throws HyracksDataException {
+ public void unregisterFile(int fileId) throws HyracksDataException {
FileReference fileRef = id2nameMap.remove(fileId);
- if (fileRef == null) {
- throw HyracksDataException.create(ErrorCode.NO_MAPPING_FOR_FILE_ID, fileId);
- }
name2IdMap.remove(fileRef);
- return fileRef;
}
@Override
- public synchronized int registerFile(FileReference fileRef) throws HyracksDataException {
- if (isMapped(fileRef)) {
- throw HyracksDataException.create(ErrorCode.FILE_ALREADY_MAPPED, fileRef);
- }
- int fileId = idCounter++;
+ public void registerFile(FileReference fileRef) throws HyracksDataException {
+ Integer fileId = idCounter++;
id2nameMap.put(fileId, fileRef);
name2IdMap.put(fileRef, fileId);
- return fileId;
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/AbstractIndexLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/AbstractIndexLifecycleTest.java
index aac4df5..8211685 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/AbstractIndexLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/AbstractIndexLifecycleTest.java
@@ -18,13 +18,14 @@
*/
package org.apache.hyracks.storage.am.common;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.common.IIndex;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.common.IIndex;
+
public abstract class AbstractIndexLifecycleTest {
protected IIndex index;
@@ -47,16 +48,10 @@
@Test
public void validSequenceTest() throws Exception {
- // Double create is invalid
+ // Double create is valid
index.create();
Assert.assertTrue(persistentStateExists());
- boolean exceptionCaught = false;
- try {
- index.create();
- } catch (Exception e) {
- exceptionCaught = true;
- }
- Assert.assertTrue(exceptionCaught);
+ index.create();
Assert.assertTrue(persistentStateExists());
// Double open is valid
@@ -74,6 +69,8 @@
// Insert more stuff
performInsertions();
+
+ // Double close is valid
index.deactivate();
// Check that the inserted stuff is still there
@@ -81,16 +78,10 @@
checkInsertions();
index.deactivate();
- // Double destroy is invalid
+ // Double destroy is valid
index.destroy();
Assert.assertFalse(persistentStateExists());
- exceptionCaught = false;
- try {
- index.destroy();
- } catch (Exception e) {
- exceptionCaught = true;
- }
- Assert.assertTrue(exceptionCaught);
+ index.destroy();
Assert.assertFalse(persistentStateExists());
}
@@ -98,34 +89,20 @@
public void invalidSequenceTest1() throws Exception {
index.create();
index.activate();
- try {
- index.create();
- } finally {
- index.deactivate();
- index.destroy();
- }
+ index.create();
}
@Test(expected = HyracksDataException.class)
public void invalidSequenceTest2() throws Exception {
index.create();
index.activate();
- try {
- index.destroy();
- } finally {
- index.deactivate();
- index.destroy();
- }
+ index.destroy();
}
@Test(expected = HyracksDataException.class)
public void invalidSequenceTest3() throws Exception {
index.create();
- try {
- index.clear();
- } finally {
- index.destroy();
- }
+ index.clear();
}
@Test(expected = HyracksDataException.class)
@@ -137,12 +114,7 @@
public void invalidSequenceTest5() throws Exception {
index.create();
index.activate();
- try {
- index.activate();
- } finally {
- index.deactivate();
- index.destroy();
- }
+ index.activate();
}
@Test(expected = HyracksDataException.class)
@@ -150,10 +122,6 @@
index.create();
index.activate();
index.deactivate();
- try {
- index.deactivate();
- } finally {
- index.destroy();
- }
+ index.deactivate();
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManager.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManager.java
index 350de74..3caeedd 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManager.java
@@ -24,6 +24,7 @@
import org.apache.hyracks.storage.common.IResourceLifecycleManager;
import org.apache.hyracks.storage.common.IStorageManager;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.storage.common.file.ResourceIdFactory;
public class TestStorageManager implements IStorageManager {
@@ -35,6 +36,11 @@
}
@Override
+ public IFileMapProvider getFileMapProvider(INCServiceContext ctx) {
+ return TestStorageManagerComponentHolder.getFileMapProvider();
+ }
+
+ @Override
public ILocalResourceRepository getLocalResourceRepository(INCServiceContext ctx) {
return TestStorageManagerComponentHolder.getLocalResourceRepository();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java
index 3060b25..d9c4b85 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java
@@ -43,12 +43,12 @@
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
import org.apache.hyracks.storage.common.buffercache.IPageReplacementStrategy;
-import org.apache.hyracks.storage.common.file.FileMapManager;
import org.apache.hyracks.storage.common.file.IFileMapManager;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.storage.common.file.ILocalResourceRepositoryFactory;
import org.apache.hyracks.storage.common.file.ResourceIdFactory;
import org.apache.hyracks.storage.common.file.ResourceIdFactoryProvider;
+import org.apache.hyracks.storage.common.file.TransientFileMapManager;
import org.apache.hyracks.storage.common.file.TransientLocalResourceRepositoryFactory;
public class TestStorageManagerComponentHolder {
@@ -98,9 +98,9 @@
return bufferCache;
}
- private synchronized static IFileMapProvider getFileMapProvider() {
+ public synchronized static IFileMapProvider getFileMapProvider() {
if (fileMapProvider == null) {
- fileMapProvider = new FileMapManager();
+ fileMapProvider = new TransientFileMapManager();
}
return fileMapProvider;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/BloomFilterTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/BloomFilterTest.java
index 35779ac..26d6d85 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/BloomFilterTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/BloomFilterTest.java
@@ -62,12 +62,13 @@
int numElements = 100;
int[] keyFields = { 0 };
- BloomFilter bf = new BloomFilter(bufferCache, harness.getFileReference(), keyFields);
+ BloomFilter bf = new BloomFilter(bufferCache, harness.getFileMapProvider(), harness.getFileReference(),
+ keyFields);
double acceptanleFalsePositiveRate = 0.1;
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
- BloomFilterSpecification bloomFilterSpec =
- BloomCalculations.computeBloomSpec(maxBucketsPerElement, acceptanleFalsePositiveRate);
+ BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
+ acceptanleFalsePositiveRate);
bf.create();
bf.activate();
@@ -120,12 +121,13 @@
int numElements = 10000;
int[] keyFields = { 2, 4, 1 };
- BloomFilter bf = new BloomFilter(bufferCache, harness.getFileReference(), keyFields);
+ BloomFilter bf = new BloomFilter(bufferCache, harness.getFileMapProvider(), harness.getFileReference(),
+ keyFields);
double acceptanleFalsePositiveRate = 0.1;
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
- BloomFilterSpecification bloomFilterSpec =
- BloomCalculations.computeBloomSpec(maxBucketsPerElement, acceptanleFalsePositiveRate);
+ BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
+ acceptanleFalsePositiveRate);
bf.create();
bf.activate();
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/util/BloomFilterTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/util/BloomFilterTestHarness.java
index 6ccb2be..04519e5 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/util/BloomFilterTestHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/util/BloomFilterTestHarness.java
@@ -29,6 +29,7 @@
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.apache.hyracks.test.support.TestUtils;
@@ -44,6 +45,7 @@
protected IIOManager ioManager;
protected IHyracksTaskContext ctx;
protected IBufferCache bufferCache;
+ protected IFileMapProvider fileMapProvider;
protected FileReference file;
protected final Random rnd = new Random();
@@ -68,6 +70,7 @@
TestStorageManagerComponentHolder.init(pageSize, numPages, maxOpenFiles);
ioManager = ctx.getIoManager();
bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
+ fileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider();
file = ioManager.getFileReference(0, simpleDateFormat.format(new Date()));
rnd.setSeed(RANDOM_SEED);
}
@@ -85,6 +88,10 @@
return bufferCache;
}
+ public IFileMapProvider getFileMapProvider() {
+ return fileMapProvider;
+ }
+
public Random getRandom() {
return rnd;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeBulkLoadTest.java
index 394f989..76a9078 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeBulkLoadTest.java
@@ -50,9 +50,10 @@
@SuppressWarnings("rawtypes")
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
- BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return BTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes, numKeys,
- leafType, harness.getPageManagerFactory().createPageManager(harness.getBufferCache()));
+ BTreeLeafFrameType leafType, boolean filtered) throws Exception {
+ return BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, numKeys, leafType, harness.getPageManagerFactory()
+ .createPageManager(harness.getBufferCache()));
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeDeleteTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeDeleteTest.java
index edf02fc..905fd22 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeDeleteTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeDeleteTest.java
@@ -50,9 +50,10 @@
@SuppressWarnings("rawtypes")
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
- BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return BTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes, numKeys,
- leafType, harness.getPageManagerFactory().createPageManager(harness.getBufferCache()));
+ BTreeLeafFrameType leafType, boolean filtered) throws Exception {
+ return BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, numKeys, leafType, harness.getPageManagerFactory()
+ .createPageManager(harness.getBufferCache()));
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeExamplesTest.java
index 18580b0..6c6161f 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeExamplesTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeExamplesTest.java
@@ -52,7 +52,7 @@
ITreeIndexMetadataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
LinkedMetaDataPageManager freePageManager =
new LinkedMetaDataPageManager(harness.getBufferCache(), metaFrameFactory);
- return BTreeUtils.createBTree(harness.getBufferCache(), typeTraits, cmpFactories,
+ return BTreeUtils.createBTree(harness.getBufferCache(), harness.getFileMapProvider(), typeTraits, cmpFactories,
BTreeLeafFrameType.REGULAR_NSM, harness.getFileReference(), freePageManager);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeInsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeInsertTest.java
index 66c80fa..14810a7 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeInsertTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeInsertTest.java
@@ -33,10 +33,10 @@
* Tests the BTree insert operation with strings and integer fields using
* various numbers of key and payload fields. Each tests first fills a BTree with
* randomly generated tuples. We compare the following operations against expected results:
- * 1) Point searches for all tuples
- * 2) Ordered scan
- * 3) Disk-order scan
- * 4) Range search (and prefix search for composite keys)
+ * 1) Point searches for all tuples
+ * 2) Ordered scan
+ * 3) Disk-order scan
+ * 4) Range search (and prefix search for composite keys)
*/
public class BTreeInsertTest extends OrderedIndexInsertTest {
@@ -59,9 +59,10 @@
@SuppressWarnings("rawtypes")
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
- BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return BTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes, numKeys,
- leafType, harness.getPageManagerFactory().createPageManager(harness.getBufferCache()));
+ BTreeLeafFrameType leafType, boolean filtered) throws Exception {
+ return BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, numKeys, leafType, harness.getPageManagerFactory()
+ .createPageManager(harness.getBufferCache()));
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeLifecycleTest.java
index 7caa40c..a096d59 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeLifecycleTest.java
@@ -36,8 +36,7 @@
private final TreeIndexTestUtils titu = new OrderedIndexTestUtils();
@SuppressWarnings("rawtypes")
- private final ISerializerDeserializer[] fieldSerdes =
- new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE };
+ private final ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE };
private ITreeIndexFrame frame = null;
@@ -47,14 +46,20 @@
@Override
public void setup() throws Exception {
harness.setUp();
- testCtx = BTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes,
- fieldSerdes.length, BTreeLeafFrameType.REGULAR_NSM,
- harness.getPageManagerFactory().createPageManager(harness.getBufferCache()));
+ testCtx = BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, fieldSerdes.length, BTreeLeafFrameType.REGULAR_NSM, harness
+ .getPageManagerFactory().createPageManager(harness.getBufferCache()));
index = testCtx.getIndex();
}
@Override
public void tearDown() throws HyracksDataException {
+ try {
+ testCtx.getIndex().deactivate();
+ } catch (Exception e) {
+ } finally {
+ testCtx.getIndex().destroy();
+ }
harness.tearDown();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeModificationOperationCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeModificationOperationCallbackTest.java
index 304bf14..defb0bf 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeModificationOperationCallbackTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeModificationOperationCallbackTest.java
@@ -37,9 +37,9 @@
protected void createIndexInstance() throws Exception {
IMetadataPageManagerFactory freePageManagerFactory = new LinkedMetadataPageManagerFactory();
IMetadataPageManager freePageManager = freePageManagerFactory.createPageManager(harness.getBufferCache());
- index = BTreeUtils.createBTree(harness.getBufferCache(), SerdeUtils.serdesToTypeTraits(keySerdes),
- SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), BTreeLeafFrameType.REGULAR_NSM,
- harness.getFileReference(), freePageManager);
+ index = BTreeUtils.createBTree(harness.getBufferCache(), harness.getFileMapProvider(), SerdeUtils
+ .serdesToTypeTraits(keySerdes), SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length),
+ BTreeLeafFrameType.REGULAR_NSM, harness.getFileReference(), freePageManager);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java
index cc4f444..1b6f42b 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java
@@ -98,16 +98,16 @@
IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
- fieldCount, harness.getFileReference());
+ BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory,
+ leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());
btree.create();
btree.activate();
ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
- ITreeIndexAccessor indexAccessor =
- btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
+ ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE,
+ TestOperationCallback.INSTANCE);
// generate keys
int numKeys = 50;
@@ -174,16 +174,16 @@
IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
- fieldCount, harness.getFileReference());
+ BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory,
+ leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());
btree.create();
btree.activate();
ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
- ITreeIndexAccessor indexAccessor =
- btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
+ ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE,
+ TestOperationCallback.INSTANCE);
// generate keys
int numKeys = 50;
@@ -247,16 +247,16 @@
IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
- fieldCount, harness.getFileReference());
+ BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory,
+ leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());
btree.create();
btree.activate();
ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
- ITreeIndexAccessor indexAccessor =
- btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
+ ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE,
+ TestOperationCallback.INSTANCE);
// generate keys
int numKeys = 50;
@@ -309,8 +309,8 @@
searchCmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
MultiComparator searchCmp = new MultiComparator(searchCmps);
- RangePredicate rangePred =
- new RangePredicate(lowKey, highKey, lowKeyInclusive, highKeyInclusive, searchCmp, searchCmp);
+ RangePredicate rangePred = new RangePredicate(lowKey, highKey, lowKeyInclusive, highKeyInclusive, searchCmp,
+ searchCmp);
return rangePred;
}
@@ -356,8 +356,8 @@
ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor(leafFrame, false);
RangePredicate rangePred = createRangePredicate(lowKey, highKey, lowKeyInclusive, highKeyInclusive);
- ITreeIndexAccessor indexAccessor =
- btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
+ ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE,
+ TestOperationCallback.INSTANCE);
indexAccessor.search(rangeCursor, rangePred);
try {
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchOperationCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchOperationCallbackTest.java
index 003dd39..c9248a6 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchOperationCallbackTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchOperationCallbackTest.java
@@ -36,9 +36,10 @@
@Override
protected void createIndexInstance() throws Exception {
ITreeIndexMetadataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- LinkedMetaDataPageManager freePageManager =
- new LinkedMetaDataPageManager(harness.getBufferCache(), metaFrameFactory);
- index = BTreeUtils.createBTree(harness.getBufferCache(), SerdeUtils.serdesToTypeTraits(keySerdes),
+ LinkedMetaDataPageManager freePageManager = new LinkedMetaDataPageManager(harness.getBufferCache(),
+ metaFrameFactory);
+ index = BTreeUtils.createBTree(harness.getBufferCache(), harness.getFileMapProvider(),
+ SerdeUtils.serdesToTypeTraits(keySerdes),
SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), BTreeLeafFrameType.REGULAR_NSM,
harness.getFileReference(), freePageManager);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSortedInsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSortedInsertTest.java
index 08f3cf6..164f699 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSortedInsertTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSortedInsertTest.java
@@ -59,9 +59,10 @@
@SuppressWarnings("rawtypes")
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
- BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return BTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes, numKeys,
- leafType, harness.getPageManagerFactory().createPageManager(harness.getBufferCache()));
+ BTreeLeafFrameType leafType, boolean filtered) throws Exception {
+ return BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, numKeys, leafType, harness.getPageManagerFactory()
+ .createPageManager(harness.getBufferCache()));
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java
index 7281da6..ee7e968 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java
@@ -58,6 +58,7 @@
import org.apache.hyracks.storage.am.common.util.TreeIndexStats;
import org.apache.hyracks.storage.am.common.util.TreeIndexStatsGatherer;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.apache.hyracks.test.support.TestUtils;
import org.junit.Test;
@@ -76,6 +77,7 @@
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
IBufferCache bufferCache = harness.getBufferCache();
+ IFileMapProvider fmp = harness.getFileMapProvider();
// declare fields
int fieldCount = 2;
@@ -99,7 +101,7 @@
IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
+ BTree btree = new BTree(bufferCache, fmp, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
fieldCount, harness.getFileReference());
btree.create();
btree.activate();
@@ -161,15 +163,16 @@
}
}
- TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, freePageManager,
- harness.getFileReference(), btree.getRootPageId());
+ int fileId = fmp.lookupFileId(harness.getFileReference());
+ TreeIndexStatsGatherer statsGatherer =
+ new TreeIndexStatsGatherer(bufferCache, freePageManager, fileId, btree.getRootPageId());
TreeIndexStats stats = statsGatherer.gatherStats(leafFrame, interiorFrame, metaFrame);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("\n" + stats.toString());
}
TreeIndexBufferCacheWarmup bufferCacheWarmup =
- new TreeIndexBufferCacheWarmup(bufferCache, freePageManager, harness.getFileReference());
+ new TreeIndexBufferCacheWarmup(bufferCache, freePageManager, fileId);
bufferCacheWarmup.warmup(leafFrame, metaFrame, new int[] { 1, 2 }, new int[] { 2, 5 });
btree.deactivate();
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateSearchTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
index 78c92eb..78023a2 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
@@ -82,8 +82,8 @@
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
- fieldCount, harness.getFileReference());
+ BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory,
+ leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());
btree.create();
btree.activate();
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateTest.java
index 3374fec..a8567e4 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateTest.java
@@ -50,9 +50,10 @@
@SuppressWarnings("rawtypes")
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
- BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return BTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes, numKeys,
- leafType, harness.getPageManagerFactory().createPageManager(harness.getBufferCache()));
+ BTreeLeafFrameType leafType, boolean filtered) throws Exception {
+ return BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, numKeys, leafType, harness.getPageManagerFactory()
+ .createPageManager(harness.getBufferCache()));
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpsertTest.java
index bc6a94f..2d10a25 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpsertTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpsertTest.java
@@ -33,10 +33,10 @@
* Tests the BTree insert operation with strings and integer fields using
* various numbers of key and payload fields. Each tests first fills a BTree with
* randomly generated tuples. We compare the following operations against expected results:
- * 1) Point searches for all tuples
- * 2) Ordered scan
- * 3) Disk-order scan
- * 4) Range search (and prefix search for composite keys)
+ * 1) Point searches for all tuples
+ * 2) Ordered scan
+ * 3) Disk-order scan
+ * 4) Range search (and prefix search for composite keys)
*/
public class BTreeUpsertTest extends OrderedIndexUpsertTest {
@@ -59,9 +59,10 @@
@SuppressWarnings("rawtypes")
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
- BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return BTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes, numKeys,
- leafType, harness.getPageManagerFactory().createPageManager(harness.getBufferCache()));
+ BTreeLeafFrameType leafType, boolean filtered) throws Exception {
+ return BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, numKeys, leafType, harness.getPageManagerFactory()
+ .createPageManager(harness.getBufferCache()));
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
index 33177b0..2f1b89f 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
@@ -49,6 +49,7 @@
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.junit.Assert;
import org.junit.Test;
@@ -127,8 +128,10 @@
rnd.setSeed(50);
IBufferCache bufferCache = harness.getBufferCache();
+ IFileMapProvider fileMapProvider = harness.getFileMapProvider();
bufferCache.createFile(harness.getFileReference());
- int btreeFileId = bufferCache.openFile(harness.getFileReference());
+ int btreeFileId = fileMapProvider.lookupFileId(harness.getFileReference());
+ bufferCache.openFile(btreeFileId);
IHyracksTaskContext ctx = harness.getHyracksTaskContext();
ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(btreeFileId, 0), true);
try {
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageFileAccessTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageFileAccessTest.java
index 6dcb3a4..d62d0ad 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageFileAccessTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageFileAccessTest.java
@@ -24,13 +24,14 @@
import java.util.Random;
import java.util.logging.Level;
+import org.junit.Test;
+
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.btree.util.AbstractBTreeTest;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
import org.apache.hyracks.storage.common.sync.LatchType;
-import org.junit.Test;
public class StorageFileAccessTest extends AbstractBTreeTest {
public class PinnedLatchedPage {
@@ -65,7 +66,7 @@
private int loopCount = 0;
private boolean fileIsOpen = false;
private Random rnd = new Random(50);
- private List<PinnedLatchedPage> pinnedPages = new LinkedList<>();
+ private List<PinnedLatchedPage> pinnedPages = new LinkedList<PinnedLatchedPage>();
public FileAccessWorker(int workerId, IBufferCache bufferCache, FileAccessType fta, int fileId, int maxPages,
int maxPinnedPages, int maxLoopCount, int closeFileChance, long thinkTime) {
@@ -258,7 +259,8 @@
public void oneThreadOneFileTest() throws Exception {
IBufferCache bufferCache = harness.getBufferCache();
bufferCache.createFile(harness.getFileReference());
- int btreeFileId = bufferCache.openFile(harness.getFileReference());
+ int btreeFileId = harness.getFileMapProvider().lookupFileId(harness.getFileReference());
+ bufferCache.openFile(btreeFileId);
Thread worker = new Thread(new FileAccessWorker(0, harness.getBufferCache(), FileAccessType.FTA_UNLATCHED,
btreeFileId, 10, 10, 100, 10, 0));
worker.start();
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java
index fc58452..fdc5e95 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java
@@ -52,7 +52,7 @@
@Override
protected ITreeIndex createIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
int[] bloomFilterKeyFields) throws HyracksDataException {
- return BTreeUtils.createBTree(harness.getBufferCache(), typeTraits, cmpFactories,
+ return BTreeUtils.createBTree(harness.getBufferCache(), harness.getFileMapProvider(), typeTraits, cmpFactories,
BTreeLeafFrameType.REGULAR_NSM, harness.getFileReference(),
harness.getPageManagerFactory().createPageManager(harness.getBufferCache()));
}
@@ -84,8 +84,9 @@
ProbabilityHelper.getUniformProbDist(insertDeleteUpdateUpsertOps.length)));
// All operations mixed.
- TestOperation[] allOps = new TestOperation[] { TestOperation.INSERT, TestOperation.DELETE, TestOperation.UPDATE,
- TestOperation.UPSERT, TestOperation.POINT_SEARCH, TestOperation.SCAN, TestOperation.DISKORDER_SCAN };
+ TestOperation[] allOps = new TestOperation[] { TestOperation.INSERT, TestOperation.DELETE,
+ TestOperation.UPDATE, TestOperation.UPSERT, TestOperation.POINT_SEARCH, TestOperation.SCAN,
+ TestOperation.DISKORDER_SCAN };
workloadConfs.add(new TestWorkloadConf(allOps, ProbabilityHelper.getUniformProbDist(allOps.length)));
return workloadConfs;
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/util/BTreeTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/util/BTreeTestContext.java
index c6c43c8..2abd547 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/util/BTreeTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/util/BTreeTestContext.java
@@ -31,6 +31,7 @@
import org.apache.hyracks.storage.am.common.api.IPageManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
@SuppressWarnings("rawtypes")
public class BTreeTestContext extends OrderedIndexTestContext {
@@ -51,12 +52,14 @@
return btree.getComparatorFactories();
}
- public static BTreeTestContext create(IBufferCache bufferCache, FileReference file,
- ISerializerDeserializer[] fieldSerdes, int numKeyFields, BTreeLeafFrameType leafType,
- IPageManager pageManager) throws Exception {
+ public static BTreeTestContext create(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ FileReference file, ISerializerDeserializer[] fieldSerdes, int numKeyFields, BTreeLeafFrameType leafType,
+ IPageManager pageManager)
+ throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
- BTree btree = BTreeUtils.createBTree(bufferCache, typeTraits, cmpFactories, leafType, file, pageManager);
+ BTree btree = BTreeUtils.createBTree(bufferCache, fileMapProvider, typeTraits, cmpFactories, leafType, file,
+ pageManager);
BTreeTestContext testCtx = new BTreeTestContext(fieldSerdes, btree);
return testCtx;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/util/BTreeTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/util/BTreeTestHarness.java
index 230e247..f40d536 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/util/BTreeTestHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/util/BTreeTestHarness.java
@@ -32,6 +32,7 @@
import org.apache.hyracks.storage.am.common.freepage.LinkedMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.apache.hyracks.test.support.TestUtils;
@@ -48,6 +49,7 @@
protected IHyracksTaskContext ctx;
protected IBufferCache bufferCache;
+ protected IFileMapProvider fileMapProvider;
protected FileReference file;
protected IMetadataPageManagerFactory pageManagerFactory;
@@ -74,6 +76,7 @@
ctx = TestUtils.create(getHyracksFrameSize());
TestStorageManagerComponentHolder.init(pageSize, numPages, maxOpenFiles);
bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
+ fileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider();
file = ctx.getIoManager().getFileReference(0, simpleDateFormat.format(new Date()));
pageManagerFactory = new LinkedMetadataPageManagerFactory();
rnd.setSeed(RANDOM_SEED);
@@ -92,6 +95,10 @@
return bufferCache;
}
+ public IFileMapProvider getFileMapProvider() {
+ return fileMapProvider;
+ }
+
public FileReference getFileReference() {
return file;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
index 52279b8..f96dd90 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
@@ -52,12 +52,12 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
- BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
+ BTreeLeafFrameType leafType, boolean filtered) throws Exception {
+ return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
+ harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory(),
- false);
+ harness.getIOScheduler(), harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory(), false);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
index 2be956d..0d2159b 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
@@ -52,12 +52,12 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
- BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
+ BTreeLeafFrameType leafType, boolean filtered) throws Exception {
+ return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
+ harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory(),
- false);
+ harness.getIOScheduler(), harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory(), false);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
index 4b82349..f246047 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
@@ -49,8 +49,8 @@
int[] bloomFilterKeyFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
int[] btreeFields, int[] filterFields) throws HyracksDataException {
return LSMBTreeUtil.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), typeTraits, cmpFactories,
- bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits,
+ cmpFactories, bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), true,
filterTypeTraits, filterCmpFactories, btreeFields, filterFields, true,
harness.getMetadataPageManagerFactory());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java
index 282623c..c8aaaab3 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java
@@ -18,8 +18,6 @@
*/
package org.apache.hyracks.storage.am.lsm.btree;
-import java.io.File;
-
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
@@ -36,6 +34,8 @@
import org.junit.Before;
import org.junit.Test;
+import java.io.File;
+
public class LSMBTreeFileManagerTest {
private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
@@ -53,11 +53,12 @@
@Test
public void deleteOrphanedFilesTest() throws Exception {
ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE };
- LSMBTreeTestContext ctx = LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, 1,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory(),
- false);
+ LSMBTreeTestContext ctx =
+ LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
+ fieldSerdes, 1, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness
+ .getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
+ harness.getMetadataPageManagerFactory(), false);
ctx.getIndex().create();
ctx.getIndex().activate();
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFilterMergeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFilterMergeTest.java
index 29259c0..96f29e4 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFilterMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFilterMergeTest.java
@@ -51,12 +51,12 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
- BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
+ BTreeLeafFrameType leafType, boolean filtered) throws Exception {
+ return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
+ harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory(),
- filtered);
+ harness.getIOScheduler(), harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory(), filtered);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
index 28eedd0..fe1082f 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
@@ -52,12 +52,12 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
- BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory(),
- false);
+ BTreeLeafFrameType leafType, boolean filtered) throws Exception {
+ return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ numKeys, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness
+ .getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), harness
+ .getMetadataPageManagerFactory(), false);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
index 98c5fb0..adeb67d 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
@@ -32,8 +32,7 @@
public class LSMBTreeLifecycleTest extends AbstractIndexLifecycleTest {
@SuppressWarnings("rawtypes")
- private final ISerializerDeserializer[] fieldSerdes =
- new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE };
+ private final ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE };
private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
private final TreeIndexTestUtils titu = new OrderedIndexTestUtils();
@@ -57,16 +56,22 @@
@Override
public void setup() throws Exception {
harness.setUp();
- testCtx = LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, fieldSerdes.length,
+ testCtx = LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
+ harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, fieldSerdes.length,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory(),
- false);
+ harness.getIOScheduler(), harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory(), false);
index = testCtx.getIndex();
}
@Override
public void tearDown() throws Exception {
+ try {
+ index.deactivate();
+ } catch (Exception e) {
+ } finally {
+ index.destroy();
+ }
harness.tearDown();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
index f24db13..76ff003 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
@@ -51,12 +51,12 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
- BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
+ BTreeLeafFrameType leafType, boolean filtered) throws Exception {
+ return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
+ harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory(),
- filtered);
+ harness.getIOScheduler(), harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory(), filtered);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
index 3f6e74a..27b012e 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
@@ -47,7 +47,8 @@
@Override
protected void createIndexInstance() throws Exception {
index = LSMBTreeUtil.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), SerdeUtils.serdesToTypeTraits(keySerdes),
+ harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
+ SerdeUtils.serdesToTypeTraits(keySerdes),
SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), bloomFilterKeyFields,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
NoOpOperationTrackerFactory.INSTANCE.getOperationTracker(null), harness.getIOScheduler(),
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
index 0ce9051..ea16368 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
@@ -53,12 +53,12 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
- BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
+ BTreeLeafFrameType leafType, boolean filtered) throws Exception {
+ return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
+ harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory(),
- false);
+ harness.getIOScheduler(), harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory(), false);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeScanDiskComponentsTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeScanDiskComponentsTest.java
index cb20b81..3ba53dd 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeScanDiskComponentsTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeScanDiskComponentsTest.java
@@ -19,7 +19,7 @@
package org.apache.hyracks.storage.am.lsm.btree;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
import java.io.ByteArrayInputStream;
import java.io.DataInput;
@@ -78,10 +78,10 @@
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType, boolean filtered) throws Exception {
return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory(),
- false);
+ harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ numKeys, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
+ harness.getMetadataPageManagerFactory(), false);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
index f841d99..0b260b4 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
@@ -52,13 +52,15 @@
@Override
protected void createIndexInstance() throws Exception {
- index = LSMBTreeUtil.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), SerdeUtils.serdesToTypeTraits(keySerdes),
+ index = LSMBTreeUtil.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
+ harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
+ SerdeUtils.serdesToTypeTraits(keySerdes),
SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), bloomFilterKeyFields,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
NoOpOperationTrackerFactory.INSTANCE.getOperationTracker(null), harness.getIOScheduler(),
- harness.getIOOperationCallback(), true, null, null, null, null, true,
- harness.getMetadataPageManagerFactory());
+ harness.getIOOperationCallback(), true, null, null, null, null, true, harness
+ .getMetadataPageManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
index 7a4af94..fdd1ec3 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
@@ -52,12 +52,12 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
- BTreeLeafFrameType leafType, boolean filtered) throws Exception {
- return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
- harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler(), harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory(),
- false);
+ BTreeLeafFrameType leafType, boolean filtered) throws Exception {
+ return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ numKeys, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness
+ .getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), harness
+ .getMetadataPageManagerFactory(), false);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
index 531a4fd..9abc321 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
@@ -53,8 +53,8 @@
protected ITreeIndex createIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
int[] bloomFilterKeyFields) throws HyracksDataException {
return LSMBTreeUtil.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), typeTraits, cmpFactories,
- bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits,
+ cmpFactories, bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), true, null,
null, null, null, true, harness.getMetadataPageManagerFactory());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/BTreeRunner.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/BTreeRunner.java
index df6558c..0a36548 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/BTreeRunner.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/BTreeRunner.java
@@ -28,6 +28,7 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrameFactory;
import org.apache.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import org.apache.hyracks.storage.am.common.freepage.LinkedMetaDataPageManager;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.apache.hyracks.test.support.TestUtils;
@@ -46,9 +47,10 @@
IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
TestStorageManagerComponentHolder.init(pageSize, numPages, MAX_OPEN_FILES);
bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider();
ITreeIndexMetadataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
LinkedMetaDataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
- btree = BTreeUtils.createBTree(bufferCache, typeTraits, cmpFactories, BTreeLeafFrameType.REGULAR_NSM, file,
+ btree = BTreeUtils.createBTree(bufferCache, fmp, typeTraits, cmpFactories, BTreeLeafFrameType.REGULAR_NSM, file,
freePageManager);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java
index d0b798a..9ae5aae 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java
@@ -41,6 +41,7 @@
import org.apache.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.TransientFileMapManager;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
public class InMemoryBTreeRunner extends Thread implements IExperimentRunner {
@@ -72,8 +73,8 @@
ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
IPageManager freePageManager = new VirtualFreePageManager(bufferCache);
- btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
- typeTraits.length, file);
+ btree = new BTree(bufferCache, new TransientFileMapManager(), freePageManager, interiorFrameFactory,
+ leafFrameFactory, cmpFactories, typeTraits.length, file);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
index 0c31795..15f2821 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
@@ -46,6 +46,7 @@
import org.apache.hyracks.storage.common.IIndexAccessor;
import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.apache.hyracks.test.support.TestUtils;
@@ -94,6 +95,7 @@
ioDeviceId = 0;
file = ioManager.resolveAbsolutePath(onDiskDir);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider();
List<IVirtualBufferCache> virtualBufferCaches = new ArrayList<>();
for (int i = 0; i < 2; i++) {
@@ -105,7 +107,7 @@
this.ioScheduler = AsynchronousScheduler.INSTANCE;
AsynchronousScheduler.INSTANCE.init(threadFactory);
- lsmtree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, bufferCache, typeTraits,
+ lsmtree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, bufferCache, fmp, typeTraits,
cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, new NoMergePolicy(),
new ThreadCountingTracker(), ioScheduler, NoOpIOOperationCallbackFactory.INSTANCE.createIoOpCallback(),
true, null, null, null, null, true, TestStorageManagerComponentHolder.getMetadataPageManagerFactory());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
index 8667d85..d8d6e5c 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
@@ -41,12 +41,12 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
@SuppressWarnings("rawtypes")
public final class LSMBTreeTestContext extends OrderedIndexTestContext {
- public LSMBTreeTestContext(ISerializerDeserializer[] fieldSerdes, ITreeIndex treeIndex, boolean filtered)
- throws HyracksDataException {
+ public LSMBTreeTestContext(ISerializerDeserializer[] fieldSerdes, ITreeIndex treeIndex, boolean filtered) throws HyracksDataException {
super(fieldSerdes, treeIndex, filtered);
}
@@ -71,10 +71,12 @@
}
public static LSMBTreeTestContext create(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- FileReference file, IBufferCache diskBufferCache, ISerializerDeserializer[] fieldSerdes, int numKeyFields,
- double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
- IMetadataPageManagerFactory metadataPageManagerFactory, boolean filtered) throws Exception {
+ FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider,
+ ISerializerDeserializer[] fieldSerdes,
+ int numKeyFields, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ IMetadataPageManagerFactory metadataPageManagerFactory, boolean filtered)
+ throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
int[] bloomFilterKeyFields = new int[numKeyFields];
@@ -82,23 +84,25 @@
bloomFilterKeyFields[i] = i;
}
LSMBTree lsmTree;
- if (filtered) {
+ if(filtered) {
ITypeTraits[] filterTypeTraits = new ITypeTraits[1];
filterTypeTraits[0] = typeTraits[0];
int[] btreefields = new int[typeTraits.length];
for (int i = 0; i < btreefields.length; i++) {
btreefields[i] = i;
}
- int[] filterfields = { btreefields.length };
- IBinaryComparatorFactory[] filterCmp = { cmpFactories[0] };
- lsmTree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache, typeTraits,
- cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
- ioScheduler, ioOpCallback, true, filterTypeTraits, filterCmp, btreefields, filterfields, true,
+ int[] filterfields = {btreefields.length};
+ IBinaryComparatorFactory[] filterCmp = {cmpFactories[0]};
+ lsmTree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache,
+ diskFileMapProvider, typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate,
+ mergePolicy, opTracker, ioScheduler, ioOpCallback, true, filterTypeTraits, filterCmp, btreefields, filterfields, true,
metadataPageManagerFactory);
- } else {
- lsmTree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache, typeTraits,
- cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
- ioScheduler, ioOpCallback, true, null, null, null, null, true, metadataPageManagerFactory);
+ }
+ else{
+ lsmTree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache,
+ diskFileMapProvider, typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate,
+ mergePolicy, opTracker, ioScheduler, ioOpCallback, true, null,null,null,null, true,
+ metadataPageManagerFactory);
}
LSMBTreeTestContext testCtx = new LSMBTreeTestContext(fieldSerdes, lsmTree, filtered);
return testCtx;
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
index 95a979f..2b5b4fb 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
@@ -49,6 +49,7 @@
import org.apache.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.apache.hyracks.test.support.TestUtils;
@@ -72,6 +73,7 @@
protected IOManager ioManager;
protected int ioDeviceId;
protected IBufferCache diskBufferCache;
+ protected IFileMapProvider diskFileMapProvider;
protected List<IVirtualBufferCache> virtualBufferCaches;
protected IHyracksTaskContext ctx;
protected ILSMIOOperationScheduler ioScheduler;
@@ -111,6 +113,7 @@
TestStorageManagerComponentHolder.init(diskPageSize, diskNumPages, diskMaxOpenFiles);
file = ioManager.resolveAbsolutePath(onDiskDir);
diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
+ diskFileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider();
virtualBufferCaches = new ArrayList<>();
for (int i = 0; i < numMutableComponents; i++) {
IVirtualBufferCache virtualBufferCache =
@@ -176,6 +179,10 @@
return diskBufferCache;
}
+ public IFileMapProvider getDiskFileMapProvider() {
+ return diskFileMapProvider;
+ }
+
public List<IVirtualBufferCache> getVirtualBufferCaches() {
return virtualBufferCaches;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
index 1c75387..9d8f208 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
@@ -29,11 +29,12 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class TestLsmIndexFileManager extends AbstractLSMIndexFileManager {
- public TestLsmIndexFileManager(IIOManager ioManager, FileReference file) {
- super(ioManager, file, null);
+ public TestLsmIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file) {
+ super(ioManager, fileMapProvider, file, null);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMIndexFileManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMIndexFileManagerTest.java
index 56618e7..67a6f69 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMIndexFileManagerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMIndexFileManagerTest.java
@@ -36,12 +36,12 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IODeviceHandle;
-import org.apache.hyracks.api.util.IoUtil;
import org.apache.hyracks.control.nc.io.DefaultDeviceResolver;
import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.component.TestLsmIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.junit.After;
import org.junit.Before;
@@ -55,6 +55,7 @@
protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
protected final static String sep = System.getProperty("file.separator");
protected IOManager ioManager;
+ protected IFileMapProvider fileMapProvider;
protected String baseDir;
protected FileReference file;
@@ -62,6 +63,7 @@
public void setUp() throws HyracksDataException {
TestStorageManagerComponentHolder.init(DEFAULT_PAGE_SIZE, DEFAULT_NUM_PAGES, DEFAULT_MAX_OPEN_FILES);
ioManager = TestStorageManagerComponentHolder.getIOManager();
+ fileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider();
baseDir = ioManager.getIODevices().get(DEFAULT_IO_DEVICE_ID).getMount() + sep + "lsm_tree"
+ simpleDateFormat.format(new Date()) + sep;
File f = new File(baseDir);
@@ -76,7 +78,7 @@
}
public void sortOrderTest(boolean testFlushFileName) throws InterruptedException, HyracksDataException {
- ILSMIndexFileManager fileManager = new TestLsmIndexFileManager(ioManager, file);
+ ILSMIndexFileManager fileManager = new TestLsmIndexFileManager(ioManager, fileMapProvider, file);
LinkedList<String> fileNames = new LinkedList<>();
int numFileNames = 100;
@@ -119,11 +121,9 @@
String dirPath = ioManager.getIODevices().get(DEFAULT_IO_DEVICE_ID).getMount() + sep + "lsm_tree"
+ simpleDateFormat.format(new Date()) + sep;
File f = new File(dirPath);
- if (f.exists()) {
- IoUtil.delete(f);
- }
+ f.mkdirs();
FileReference file = ioManager.resolveAbsolutePath(f.getAbsolutePath());
- ILSMIndexFileManager fileManager = new TestLsmIndexFileManager(ioManager, file);
+ ILSMIndexFileManager fileManager = new TestLsmIndexFileManager(ioManager, fileMapProvider, file);
fileManager.createDirs();
List<FileReference> flushFiles = new ArrayList<>();
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualBufferCacheTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualBufferCacheTest.java
index 5ff5a11..59afa19 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualBufferCacheTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualBufferCacheTest.java
@@ -121,7 +121,7 @@
private void deleteFiles() throws Exception {
for (int i = 0; i < NUM_FILES; i++) {
- vbc.deleteFile(fileStates[i].fileId);
+ vbc.deleteFile(fileStates[i].fileId, false);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
index 51fbb58..daf400f 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/LSMInvertedIndexTestHarness.java
@@ -48,6 +48,7 @@
import org.apache.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.apache.hyracks.test.support.TestUtils;
@@ -67,6 +68,7 @@
protected IOManager ioManager;
protected int ioDeviceId;
protected IBufferCache diskBufferCache;
+ protected IFileMapProvider diskFileMapProvider;
protected List<IVirtualBufferCache> virtualBufferCaches;
protected IHyracksTaskContext ctx;
protected ILSMIOOperationScheduler ioScheduler;
@@ -107,6 +109,7 @@
ctx = TestUtils.create(getHyracksFrameSize());
TestStorageManagerComponentHolder.init(diskPageSize, diskNumPages, diskMaxOpenFiles);
diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
+ diskFileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider();
virtualBufferCaches = new ArrayList<>();
for (int i = 0; i < numMutableComponents; i++) {
IVirtualBufferCache virtualBufferCache = new MultitenantVirtualBufferCache(
@@ -181,6 +184,10 @@
return diskBufferCache;
}
+ public IFileMapProvider getDiskFileMapProvider() {
+ return diskFileMapProvider;
+ }
+
public List<IVirtualBufferCache> getVirtualBufferCaches() {
return virtualBufferCaches;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexLifecycleTest.java
index 0db06e5..b05b838 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexLifecycleTest.java
@@ -53,22 +53,29 @@
public void setup() throws Exception {
harness.setUp();
ITypeTraits[] tokenTypeTraits = new ITypeTraits[] { UTF8StringPointable.TYPE_TRAITS };
- IBinaryComparatorFactory[] tokenCmpFactories =
- new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) };
+ IBinaryComparatorFactory[] tokenCmpFactories = new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) };
ITypeTraits[] invListTypeTraits = new ITypeTraits[] { IntegerPointable.TYPE_TRAITS };
- IBinaryComparatorFactory[] invListCmpFactories =
- new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) };
+ IBinaryComparatorFactory[] invListCmpFactories = new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
+ .of(IntegerPointable.FACTORY) };
IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
- FileReference btreeFile = harness.getIOManager()
- .resolveAbsolutePath(harness.getInvListsFileRef().getFile().getAbsolutePath() + "_btree");
- index = new OnDiskInvertedIndex(harness.getDiskBufferCache(), invListBuilder, invListTypeTraits,
- invListCmpFactories, tokenTypeTraits, tokenCmpFactories, harness.getInvListsFileRef(), btreeFile,
- harness.getMetadataPageManagerFactory());
+ FileReference btreeFile = harness.getIOManager().resolveAbsolutePath(harness.getInvListsFileRef().getFile()
+ .getAbsolutePath()
+ + "_btree");
+ index = new OnDiskInvertedIndex(harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), invListBuilder,
+ invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories,
+ harness.getInvListsFileRef(), btreeFile, harness.getMetadataPageManagerFactory());
}
@Override
public void tearDown() throws Exception {
+ try {
+ index.deactivate();
+ } catch (Exception e) {
+ } finally {
+ index.destroy();
+ }
harness.tearDown();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
index d868118..dcc9064 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
@@ -143,29 +143,20 @@
}
case ONDISK: {
invIndex = InvertedIndexUtils.createOnDiskInvertedIndex(ioManager, harness.getDiskBufferCache(),
- invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories,
- harness.getInvListsFileRef(), harness.getMetadataPageManagerFactory());
+ harness.getDiskFileMapProvider(), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
+ tokenCmpFactories, harness.getInvListsFileRef(), harness.getMetadataPageManagerFactory());
break;
}
case PARTITIONED_ONDISK: {
invIndex = InvertedIndexUtils.createPartitionedOnDiskInvertedIndex(ioManager,
- harness.getDiskBufferCache(), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
- tokenCmpFactories, harness.getInvListsFileRef(), harness.getMetadataPageManagerFactory());
+ harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), invListTypeTraits,
+ invListCmpFactories, tokenTypeTraits, tokenCmpFactories, harness.getInvListsFileRef(),
+ harness.getMetadataPageManagerFactory());
break;
}
case LSM: {
invIndex = InvertedIndexUtils.createLSMInvertedIndex(ioManager, harness.getVirtualBufferCaches(),
- invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
- harness.getDiskBufferCache(), harness.getOnDiskDir(), harness.getBoomFilterFalsePositiveRate(),
- harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
- harness.getIOOperationCallback(), invertedIndexFields, filterTypeTraits, filterCmpFactories,
- filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, true,
- harness.getMetadataPageManagerFactory());
- break;
- }
- case PARTITIONED_LSM: {
- invIndex = InvertedIndexUtils.createPartitionedLSMInvertedIndex(ioManager,
- harness.getVirtualBufferCaches(), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
+ harness.getDiskFileMapProvider(), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, tokenizerFactory, harness.getDiskBufferCache(), harness.getOnDiskDir(),
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
@@ -174,6 +165,17 @@
harness.getMetadataPageManagerFactory());
break;
}
+ case PARTITIONED_LSM: {
+ invIndex = InvertedIndexUtils.createPartitionedLSMInvertedIndex(ioManager,
+ harness.getVirtualBufferCaches(), harness.getDiskFileMapProvider(), invListTypeTraits,
+ invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
+ harness.getDiskBufferCache(), harness.getOnDiskDir(), harness.getBoomFilterFalsePositiveRate(),
+ harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
+ harness.getIOOperationCallback(), invertedIndexFields, filterTypeTraits, filterCmpFactories,
+ filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, true,
+ harness.getMetadataPageManagerFactory());
+ break;
+ }
default: {
throw HyracksDataException.create(ErrorCode.UNKNOWN_INVERTED_INDEX_TYPE, invIndexType);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
index 8b8da4a..85070a8 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
@@ -56,9 +56,10 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
+ harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
+ numKeys, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
harness.getMetadataPageManagerFactory());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
index 1cb0155..5437713 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
@@ -56,11 +56,12 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
- harness.getMetadataPageManagerFactory());
+ return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
+ harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
+ numKeys, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), harness
+ .getMetadataPageManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
index 90e805d..00cb223 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
@@ -44,10 +44,10 @@
RTreePolicyType rtreePolicyType, int[] rtreeFields, int[] btreeFields, ITypeTraits[] filterTypeTraits,
IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) throws HyracksDataException {
return LSMRTreeUtils.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), typeTraits, rtreeCmpFactories,
- btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(),
- harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
- harness.getIOOperationCallback(),
+ harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits,
+ rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
+ harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
+ harness.getIOScheduler(), harness.getIOOperationCallback(),
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), rtreeFields, btreeFields,
filterTypeTraits, filterCmpFactories, filterFields, true, false,
harness.getMetadataPageManagerFactory());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
index fa8de56..7b112c6 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
@@ -56,11 +56,12 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
- harness.getMetadataPageManagerFactory());
+ return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
+ harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
+ numKeys, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), harness
+ .getMetadataPageManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java
index 6a7106b..d49e709 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeLifecycleTest.java
@@ -39,8 +39,8 @@
private final ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
- private final IPrimitiveValueProviderFactory[] valueProviderFactories =
- RTreeUtils.createPrimitiveValueProviderFactories(4, IntegerPointable.FACTORY);
+ private final IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils
+ .createPrimitiveValueProviderFactories(4, IntegerPointable.FACTORY);
private final int numKeys = 4;
private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
@@ -66,16 +66,23 @@
@Override
public void setup() throws Exception {
harness.setUp();
- testCtx = LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- RTreePolicyType.RTREE, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
- harness.getMetadataPageManagerFactory());
+ testCtx = LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(),
+ harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
+ numKeys, RTreePolicyType.RTREE, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), harness
+ .getMetadataPageManagerFactory());
index = testCtx.getIndex();
}
@Override
public void tearDown() throws Exception {
+ try {
+ index.deactivate();
+ } catch (Exception e) {
+ } finally {
+ index.destroy();
+ }
harness.tearDown();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
index 04ec368..feed391 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
@@ -55,11 +55,11 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
- harness.getMetadataPageManagerFactory());
+ return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), harness
+ .getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ valueProviderFactories, numKeys, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness
+ .getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(), harness
+ .getIOOperationCallback(), harness.getMetadataPageManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
index 42b4f68..91fc323 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
@@ -57,9 +57,10 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- rtreePolicyType, harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
- harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory());
+ harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
+ fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType, harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
+ harness.getMetadataPageManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
index d643b1c..6ab4967 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
@@ -57,9 +57,10 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- rtreePolicyType, harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
- harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory());
+ harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
+ fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType, harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
+ harness.getMetadataPageManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
index ea90a43..835ae64 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
@@ -44,8 +44,8 @@
RTreePolicyType rtreePolicyType, int[] rtreeFields, int[] btreeFields, ITypeTraits[] filterTypeTraits,
IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) throws HyracksDataException {
return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), typeTraits, rtreeCmpFactories,
- btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getMergePolicy(),
+ harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits,
+ rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), rtreeFields,
filterTypeTraits, filterCmpFactories, filterFields, true, false,
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
index abc9271..c881b39 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
@@ -57,9 +57,10 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- rtreePolicyType, harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
- harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory());
+ harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
+ fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType, harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), harness
+ .getMetadataPageManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java
index cd8a465..acd40dd 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesLifecycleTest.java
@@ -39,8 +39,8 @@
private final ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
- private final IPrimitiveValueProviderFactory[] valueProviderFactories =
- RTreeUtils.createPrimitiveValueProviderFactories(4, IntegerPointable.FACTORY);
+ private final IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils
+ .createPrimitiveValueProviderFactories(4, IntegerPointable.FACTORY);
private final int numKeys = 4;
private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
@@ -66,16 +66,22 @@
@Override
public void setup() throws Exception {
harness.setUp();
- testCtx = LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getIOManager(),
- harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes,
- valueProviderFactories, numKeys, RTreePolicyType.RTREE, harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
- harness.getMetadataPageManagerFactory());
+ testCtx = LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getIOManager(), harness
+ .getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(), harness
+ .getDiskFileMapProvider(), fieldSerdes, valueProviderFactories, numKeys, RTreePolicyType.RTREE,
+ harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(), harness
+ .getIOOperationCallback(), harness.getMetadataPageManagerFactory());
index = testCtx.getIndex();
}
@Override
public void tearDown() throws Exception {
+ try {
+ index.deactivate();
+ } catch (Exception e) {
+ } finally {
+ index.destroy();
+ }
harness.tearDown();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
index cdd60f9..59c64db 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
@@ -56,9 +56,10 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
- rtreePolicyType, harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
- harness.getIOOperationCallback(), harness.getMetadataPageManagerFactory());
+ harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(),
+ fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType, harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
+ harness.getMetadataPageManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
index 91b38cb..e319d1d 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
@@ -61,10 +61,10 @@
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, int[] btreeFields) throws HyracksDataException {
return LSMRTreeUtils.createLSMTree(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), typeTraits, rtreeCmpFactories,
- btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(),
- harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
- harness.getIOOperationCallback(),
+ harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits,
+ rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
+ harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
+ harness.getIOScheduler(), harness.getIOOperationCallback(),
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), null, btreeFields, null,
null, null, true, false, harness.getMetadataPageManagerFactory());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
index 954538b..987e015 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
@@ -62,8 +62,8 @@
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, int[] btreeFields) throws HyracksDataException {
return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getIOManager(), harness.getVirtualBufferCaches(),
- harness.getFileReference(), harness.getDiskBufferCache(), typeTraits, rtreeCmpFactories,
- btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getMergePolicy(),
+ harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits,
+ rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), null, null, null, null, true,
false, harness.getMetadataPageManagerFactory());
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
index 753fc93..d55daef 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
@@ -43,6 +43,7 @@
import org.apache.hyracks.storage.am.rtree.RTreeCheckTuple;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
@SuppressWarnings("rawtypes")
public final class LSMRTreeTestContext extends AbstractRTreeTestContext {
@@ -74,14 +75,15 @@
}
public static LSMRTreeTestContext create(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- FileReference file, IBufferCache diskBufferCache, ISerializerDeserializer[] fieldSerdes,
+ FileReference file,
+ IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeyFields, RTreePolicyType rtreePolicyType,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
IMetadataPageManagerFactory metadataPageManagerFactory) throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
- IBinaryComparatorFactory[] rtreeCmpFactories =
- SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
+ IBinaryComparatorFactory[] rtreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes,
+ numKeyFields);
int numBtreeFields = fieldSerdes.length - numKeyFields;
ISerializerDeserializer[] btreeFieldSerdes = new ISerializerDeserializer[numBtreeFields];
int[] btreeFields = new int[numBtreeFields];
@@ -89,10 +91,11 @@
btreeFields[i] = numKeyFields + i;
btreeFieldSerdes[i] = fieldSerdes[numKeyFields + i];
}
- IBinaryComparatorFactory[] btreeCmpFactories =
- SerdeUtils.serdesToComparatorFactories(btreeFieldSerdes, numBtreeFields);
+ IBinaryComparatorFactory[] btreeCmpFactories = SerdeUtils.serdesToComparatorFactories(btreeFieldSerdes,
+ numBtreeFields);
LSMRTree lsmTree = LSMRTreeUtils.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache,
- typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
+ diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories,
+ rtreePolicyType,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback,
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), null, btreeFields, null,
null, null, true, false, metadataPageManagerFactory);
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
index 4af7c6e..1c459a1 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
@@ -48,6 +48,7 @@
import org.apache.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.apache.hyracks.test.support.TestUtils;
@@ -68,6 +69,7 @@
protected IOManager ioManager;
protected int ioDeviceId;
protected IBufferCache diskBufferCache;
+ protected IFileMapProvider diskFileMapProvider;
protected List<IVirtualBufferCache> virtualBufferCaches;
protected IHyracksTaskContext ctx;
protected ILSMIOOperationScheduler ioScheduler;
@@ -107,6 +109,7 @@
ctx = TestUtils.create(getHyracksFrameSize());
TestStorageManagerComponentHolder.init(diskPageSize, diskNumPages, diskMaxOpenFiles);
diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
+ diskFileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider();
virtualBufferCaches = new ArrayList<>();
for (int i = 0; i < numMutableComponents; i++) {
IVirtualBufferCache virtualBufferCache =
@@ -171,6 +174,10 @@
return diskBufferCache;
}
+ public IFileMapProvider getDiskFileMapProvider() {
+ return diskFileMapProvider;
+ }
+
public List<IVirtualBufferCache> getVirtualBufferCaches() {
return virtualBufferCaches;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
index 975a3d3..0653ddf 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
@@ -43,6 +43,7 @@
import org.apache.hyracks.storage.am.rtree.RTreeCheckTuple;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
@SuppressWarnings("rawtypes")
public final class LSMRTreeWithAntiMatterTuplesTestContext extends AbstractRTreeTestContext {
@@ -74,23 +75,26 @@
}
public static LSMRTreeWithAntiMatterTuplesTestContext create(IIOManager ioManager,
- List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
+ List<IVirtualBufferCache> virtualBufferCaches,
+ FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider,
ISerializerDeserializer[] fieldSerdes, IPrimitiveValueProviderFactory[] valueProviderFactories,
int numKeyFields, RTreePolicyType rtreePolicyType, ILSMMergePolicy mergePolicy,
ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
- IMetadataPageManagerFactory metadataPageManagerFactory) throws Exception {
+ IMetadataPageManagerFactory metadataPageManagerFactory)
+ throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
- IBinaryComparatorFactory[] rtreeCmpFactories =
- SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
- IBinaryComparatorFactory[] btreeCmpFactories =
- SerdeUtils.serdesToComparatorFactories(fieldSerdes, fieldSerdes.length);
+ IBinaryComparatorFactory[] rtreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes,
+ numKeyFields);
+ IBinaryComparatorFactory[] btreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes,
+ fieldSerdes.length);
LSMRTreeWithAntiMatterTuples lsmTree = LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(ioManager,
- virtualBufferCaches, file, diskBufferCache, typeTraits, rtreeCmpFactories, btreeCmpFactories,
+ virtualBufferCaches,
+ file, diskBufferCache, diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories,
valueProviderFactories, rtreePolicyType, mergePolicy, opTracker, ioScheduler, ioOpCallback,
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), null, null, null, null, true,
false, metadataPageManagerFactory);
- LSMRTreeWithAntiMatterTuplesTestContext testCtx =
- new LSMRTreeWithAntiMatterTuplesTestContext(fieldSerdes, lsmTree);
+ LSMRTreeWithAntiMatterTuplesTestContext testCtx = new LSMRTreeWithAntiMatterTuplesTestContext(fieldSerdes,
+ lsmTree);
return testCtx;
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeBulkLoadTest.java
index 6f934cd..4cc6c7d 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeBulkLoadTest.java
@@ -54,8 +54,9 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return RTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes,
- valueProviderFactories, numKeys, rtreePolicyType, harness.getMetadataManagerFactory());
+ return RTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType,
+ harness.getMetadataManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeDeleteTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeDeleteTest.java
index ce1ba96..803a130 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeDeleteTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeDeleteTest.java
@@ -54,8 +54,9 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return RTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes,
- valueProviderFactories, numKeys, rtreePolicyType, harness.getMetadataManagerFactory());
+ return RTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType,
+ harness.getMetadataManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeExamplesTest.java
index 999fd65..ecc6fe4 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeExamplesTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeExamplesTest.java
@@ -53,8 +53,9 @@
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, int[] rtreeFields, int[] btreeFields, ITypeTraits[] filterTypeTraits,
IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) throws HyracksDataException {
- return RTreeUtils.createRTree(harness.getBufferCache(), typeTraits, valueProviderFactories, rtreeCmpFactories,
- rtreePolicyType, harness.getFileReference(), false, harness.getMetadataManagerFactory());
+ return RTreeUtils.createRTree(harness.getBufferCache(), harness.getFileMapProvider(), typeTraits,
+ valueProviderFactories, rtreeCmpFactories, rtreePolicyType, harness.getFileReference(), false,
+ harness.getMetadataManagerFactory());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeInsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeInsertTest.java
index 93be987..57a2ac3 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeInsertTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeInsertTest.java
@@ -54,8 +54,9 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return RTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes,
- valueProviderFactories, numKeys, rtreePolicyType, harness.getMetadataManagerFactory());
+ return RTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType,
+ harness.getMetadataManagerFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeLifecycleTest.java
index 9745c08..b7eb961 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeLifecycleTest.java
@@ -42,8 +42,8 @@
private final ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
- private final IPrimitiveValueProviderFactory[] valueProviderFactories =
- RTreeUtils.createPrimitiveValueProviderFactories(4, IntegerPointable.FACTORY);
+ private final IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils
+ .createPrimitiveValueProviderFactories(4, IntegerPointable.FACTORY);
private final int numKeys = 4;
@SuppressWarnings("rawtypes")
@@ -53,13 +53,20 @@
@Override
public void setup() throws Exception {
harness.setUp();
- testCtx = RTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes,
- valueProviderFactories, numKeys, RTreePolicyType.RTREE, harness.getMetadataManagerFactory());
+ testCtx = RTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, valueProviderFactories, numKeys, RTreePolicyType.RTREE,
+ harness.getMetadataManagerFactory());
index = testCtx.getIndex();
}
@Override
public void tearDown() throws HyracksDataException {
+ try {
+ testCtx.getIndex().deactivate();
+ } catch (Exception e) {
+ } finally {
+ testCtx.getIndex().destroy();
+ }
harness.tearDown();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeSearchCursorTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
index 31d54ae..bdb6267 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
@@ -121,8 +121,8 @@
IRTreeLeafFrame leafFrame = (IRTreeLeafFrame) leafFrameFactory.createFrame();
IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
- RTree rtree = new RTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
- fieldCount, harness.getFileReference(), false);
+ RTree rtree = new RTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory,
+ leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference(), false);
rtree.create();
rtree.activate();
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java
index faa63c5..0a27be3 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java
@@ -60,8 +60,9 @@
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, int[] btreeFields) throws HyracksDataException {
- return RTreeUtils.createRTree(harness.getBufferCache(), typeTraits, valueProviderFactories, rtreeCmpFactories,
- rtreePolicyType, harness.getFileReference(), false, harness.getMetadataManagerFactory());
+ return RTreeUtils.createRTree(harness.getBufferCache(), harness.getFileMapProvider(), typeTraits,
+ valueProviderFactories, rtreeCmpFactories, rtreePolicyType, harness.getFileReference(), false,
+ harness.getMetadataManagerFactory());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/utils/RTreeTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/utils/RTreeTestContext.java
index bf70dc1..a738e2f 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/utils/RTreeTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/utils/RTreeTestContext.java
@@ -33,6 +33,7 @@
import org.apache.hyracks.storage.am.rtree.impls.RTree;
import org.apache.hyracks.storage.am.rtree.util.RTreeUtils;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
@SuppressWarnings("rawtypes")
public class RTreeTestContext extends AbstractRTreeTestContext {
@@ -53,14 +54,15 @@
return rtree.getComparatorFactories();
}
- public static RTreeTestContext create(IBufferCache bufferCache, FileReference file,
- ISerializerDeserializer[] fieldSerdes, IPrimitiveValueProviderFactory[] valueProviderFactories,
- int numKeyFields, RTreePolicyType rtreePolicyType, IPageManagerFactory pageManagerFactory)
+ public static RTreeTestContext create(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ FileReference file, ISerializerDeserializer[] fieldSerdes,
+ IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeyFields, RTreePolicyType rtreePolicyType,
+ IPageManagerFactory pageManagerFactory)
throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
- RTree rtree = RTreeUtils.createRTree(bufferCache, typeTraits, valueProviderFactories, cmpFactories,
- rtreePolicyType, file, false, pageManagerFactory);
+ RTree rtree = RTreeUtils.createRTree(bufferCache, fileMapProvider, typeTraits, valueProviderFactories,
+ cmpFactories, rtreePolicyType, file, false, pageManagerFactory);
RTreeTestContext testCtx = new RTreeTestContext(fieldSerdes, rtree);
return testCtx;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/utils/RTreeTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/utils/RTreeTestHarness.java
index 5ad777d..c9975cd 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/utils/RTreeTestHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/utils/RTreeTestHarness.java
@@ -32,6 +32,7 @@
import org.apache.hyracks.storage.am.common.freepage.LinkedMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.apache.hyracks.test.support.TestUtils;
@@ -46,6 +47,7 @@
protected IHyracksTaskContext ctx;
protected IBufferCache bufferCache;
+ protected IFileMapProvider fileMapProvider;
protected int treeFileId;
protected final Random rnd = new Random();
@@ -76,6 +78,7 @@
file = ioManager.resolve(fileName);
ctx = TestUtils.create(getHyracksFrameSize());
bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
+ fileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider();
rnd.setSeed(RANDOM_SEED);
}
@@ -93,6 +96,10 @@
return bufferCache;
}
+ public IFileMapProvider getFileMapProvider() {
+ return fileMapProvider;
+ }
+
public String getFileName() {
return fileName;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java
index 3a857cc..cccbc5d 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java
@@ -34,10 +34,10 @@
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.apache.hyracks.test.support.TestUtils;
import org.junit.After;
-import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -73,13 +73,7 @@
@Test
public void testFlushBehaviorOnFileEviction() throws IOException {
flushBehaviorTest(true);
- boolean exceptionCaught = false;
- try {
- flushBehaviorTest(false);
- } catch (Exception e) {
- exceptionCaught = true;
- }
- Assert.assertTrue(exceptionCaught);
+ flushBehaviorTest(false);
}
private void flushBehaviorTest(boolean deleteFile) throws IOException {
@@ -87,10 +81,12 @@
IBufferCache bufferCache =
TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider();
IOManager ioManager = TestStorageManagerComponentHolder.getIOManager();
FileReference firstFileRef = ioManager.resolve(fileName);
- int firstFileId = bufferCache.createFile(firstFileRef);
+ bufferCache.createFile(firstFileRef);
+ int firstFileId = fmp.lookupFileId(firstFileRef);
bufferCache.openFile(firstFileId);
// Fill the first page with known data and make it dirty by write
@@ -108,12 +104,13 @@
}
bufferCache.closeFile(firstFileId);
if (deleteFile) {
- bufferCache.deleteFile(firstFileId);
+ bufferCache.deleteFile(firstFileId, false);
}
// Create a file with the same name.
FileReference secondFileRef = ioManager.resolve(fileName);
- int secondFileId = bufferCache.createFile(secondFileRef);
+ bufferCache.createFile(secondFileRef);
+ int secondFileId = fmp.lookupFileId(secondFileRef);
// This open will replace the firstFileRef's slot in the BufferCache,
// causing it's pages to be cleaned up. We want to make sure that those
@@ -150,7 +147,7 @@
ioManager.close(testFileHandle);
bufferCache.closeFile(secondFileId);
if (deleteFile) {
- bufferCache.deleteFile(secondFileId);
+ bufferCache.deleteFile(secondFileId, false);
}
bufferCache.close();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheTest.java
index e688ee5..4a08881 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheTest.java
@@ -35,6 +35,7 @@
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.apache.hyracks.test.support.TestUtils;
import org.junit.AfterClass;
@@ -64,11 +65,12 @@
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
IBufferCache bufferCache =
TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
-
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider();
IIOManager ioManager = TestStorageManagerComponentHolder.getIOManager();
String fileName = getFileName();
FileReference file = ioManager.resolve(fileName);
- int fileId = bufferCache.createFile(file);
+ bufferCache.createFile(file);
+ int fileId = fmp.lookupFileId(file);
int num = 10;
int testPageId = 0;
@@ -146,6 +148,7 @@
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
IBufferCache bufferCache =
TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider();
IIOManager ioManager = TestStorageManagerComponentHolder.getIOManager();
List<Integer> fileIds = new ArrayList<>();
@@ -154,7 +157,8 @@
String fileName = getFileName();
FileReference file = ioManager.resolve(fileName);
- int fileId = bufferCache.createFile(file);
+ bufferCache.createFile(file);
+ int fileId = fmp.lookupFileId(file);
bufferCache.openFile(fileId);
fileIds.add(fileId);
}
@@ -165,7 +169,8 @@
try {
String fileName = getFileName();
FileReference file = ioManager.resolve(fileName);
- int fileId = bufferCache.createFile(file);
+ bufferCache.createFile(file);
+ int fileId = fmp.lookupFileId(file);
bufferCache.openFile(fileId);
} catch (HyracksDataException e) {
exceptionThrown = true;
@@ -182,7 +187,8 @@
try {
String fileName = getFileName();
FileReference file = ioManager.resolve(fileName);
- int fileId = bufferCache.createFile(file);
+ bufferCache.createFile(file);
+ int fileId = fmp.lookupFileId(file);
bufferCache.openFile(fileId);
fileIds.add(fileId);
@@ -203,6 +209,7 @@
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
IBufferCache bufferCache =
TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider();
IIOManager ioManager = TestStorageManagerComponentHolder.getIOManager();
List<Integer> fileIds = new ArrayList<>();
@@ -214,7 +221,8 @@
for (int i = 0; i < MAX_OPEN_FILES; i++) {
String fileName = getFileName();
FileReference file = ioManager.resolve(fileName);
- int fileId = bufferCache.createFile(file);
+ bufferCache.createFile(file);
+ int fileId = fmp.lookupFileId(file);
bufferCache.openFile(fileId);
fileIds.add(fileId);
@@ -241,7 +249,8 @@
try {
String fileName = getFileName();
FileReference file = ioManager.resolve(fileName);
- int fileId = bufferCache.createFile(file);
+ bufferCache.createFile(file);
+ int fileId = fmp.lookupFileId(file);
bufferCache.openFile(fileId);
} catch (HyracksDataException e) {
exceptionThrown = true;
@@ -262,7 +271,8 @@
for (int i = 0; i < filesToClose; i++) {
String fileName = getFileName();
FileReference file = ioManager.resolve(fileName);
- int fileId = bufferCache.createFile(file);
+ bufferCache.createFile(file);
+ int fileId = fmp.lookupFileId(file);
bufferCache.openFile(fileId);
fileIds.add(fileId);
}
@@ -271,7 +281,8 @@
try {
String fileName = getFileName();
FileReference file = ioManager.resolve(fileName);
- int fileId = bufferCache.createFile(file);
+ bufferCache.createFile(file);
+ int fileId = fmp.lookupFileId(file);
bufferCache.openFile(fileId);
} catch (HyracksDataException e) {
exceptionThrown = true;
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java
index 0cb87dd..7075417 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java
@@ -20,8 +20,6 @@
import java.io.IOException;
import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
import java.util.logging.Logger;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -176,45 +174,4 @@
return null;
}
}
-
- /**
- * Write map as a json string. if an object is a string and starts with a { or [
- * then it assumes that it is a json object or a json array and so it doesn't surround
- * it with "
- *
- * @param map
- * a map representing the json object
- * @return
- * a String representation of the json object
- */
- public static String fromMap(Map<String, Object> map) {
- StringBuilder aString = new StringBuilder();
- aString.append("{ ");
- boolean first = true;
- for (Entry<String, Object> entry : map.entrySet()) {
- if (!first) {
- aString.append(", ");
- }
- aString.append("\"");
- aString.append(entry.getKey());
- aString.append("\"");
- aString.append(" : ");
- Object value = entry.getValue();
- if (value instanceof String) {
- String strValue = (String) value;
- if (strValue.startsWith("{") || strValue.startsWith("[")) {
- aString.append(value);
- } else {
- aString.append("\"");
- aString.append(value);
- aString.append("\"");
- }
- } else {
- aString.append(value);
- }
- first = false;
- }
- aString.append(" }");
- return aString.toString();
- }
}