refactored index interfaces to allow proper lifecycle management
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_lsm_tree@1628 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java
index 85cfe85..35cf8ed 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java
@@ -138,10 +138,10 @@
invListsTypeTraits[0] = IntegerPointable.TYPE_TRAITS;
invListsComparatorFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- createPrimaryIndex();
+ createPrimaryIndex();
loadPrimaryIndex();
printPrimaryIndex();
- createInvertedIndex();
+ createInvertedIndex();
loadInvertedIndex();
}
@@ -158,10 +158,9 @@
public void createInvertedIndex() throws Exception {
JobSpecification spec = new JobSpecification();
InvertedIndexCreateOperatorDescriptor invIndexCreateOp = new InvertedIndexCreateOperatorDescriptor(spec,
- storageManager, btreeFileSplitProvider, invListsFileSplitProvider,
- indexRegistryProvider, tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits,
- invListsComparatorFactories, tokenizerFactory, btreeDataflowHelperFactory,
- NoOpOperationCallbackProvider.INSTANCE);
+ storageManager, btreeFileSplitProvider, invListsFileSplitProvider, indexRegistryProvider,
+ tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits, invListsComparatorFactories,
+ tokenizerFactory, btreeDataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, invIndexCreateOp, NC1_ID);
spec.addRoot(invIndexCreateOp);
runTest(spec);
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
index 41a63a6..cbd584e 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
@@ -32,7 +32,8 @@
@Override
public ITreeIndex createIndexInstance() throws HyracksDataException {
try {
- return BTreeUtils.createBTree(opDesc.getStorageManager().getBufferCache(ctx), treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc
+ return BTreeUtils.createBTree(opDesc.getStorageManager().getBufferCache(ctx), opDesc.getStorageManager()
+ .getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc
.getTreeIndexComparatorFactories(), BTreeLeafFrameType.REGULAR_NSM);
} catch (BTreeException e) {
throw new HyracksDataException(e);
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
index 6fe7720..6960077 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
@@ -52,20 +52,23 @@
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class BTree extends AbstractTreeIndex {
public static final float DEFAULT_FILL_FACTOR = 0.7f;
private final static long RESTART_OP = Long.MIN_VALUE;
+
private final static int MAX_RESTARTS = 10;
private final ReadWriteLock treeLatch;
- public BTree(IBufferCache bufferCache, IFreePageManager freePageManager,
+ public BTree(IBufferCache bufferCache, IFileMapProvider fileMapProvider, IFreePageManager freePageManager,
ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
- super(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount);
+ super(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
+ fieldCount);
this.treeLatch = new ReentrantReadWriteLock(true);
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java
index ff02ce4..3fa1309 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java
@@ -19,20 +19,25 @@
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class BTreeUtils {
- public static BTree createBTree(IBufferCache bufferCache, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, BTreeLeafFrameType leafType) throws BTreeException {
+ public static BTree createBTree(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, BTreeLeafFrameType leafType)
+ throws BTreeException {
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, typeTraits.length);
+ BTree btree = new BTree(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory,
+ cmpFactories, typeTraits.length);
return btree;
}
-
+
// Creates a new MultiComparator by constructing new IBinaryComparators.
- public static MultiComparator getSearchMultiComparator(IBinaryComparatorFactory[] cmpFactories, ITupleReference searchKey) {
+ public static MultiComparator getSearchMultiComparator(IBinaryComparatorFactory[] cmpFactories,
+ ITupleReference searchKey) {
if (searchKey == null || cmpFactories.length == searchKey.getFieldCount()) {
return MultiComparator.create(cmpFactories);
}
@@ -42,11 +47,12 @@
}
return new MultiComparator(newCmps);
}
-
- public static ITreeIndexFrameFactory getLeafFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory, BTreeLeafFrameType leafType) throws BTreeException {
- switch(leafType) {
+
+ public static ITreeIndexFrameFactory getLeafFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory,
+ BTreeLeafFrameType leafType) throws BTreeException {
+ switch (leafType) {
case REGULAR_NSM: {
- return new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ return new BTreeNSMLeafFrameFactory(tupleWriterFactory);
}
case FIELD_PREFIX_COMPRESSED_NSM: {
return new BTreeFieldPrefixNSMLeafFrameFactory(tupleWriterFactory);
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndex.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndex.java
index ac55236..5afd0fd 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndex.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndex.java
@@ -16,6 +16,7 @@
package edu.uci.ics.hyracks.storage.am.common.dataflow;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
@@ -34,20 +35,20 @@
* Initializes the persistent state of an index, e.g., the root page, and
* metadata pages.
*
- * @param indexFileId
+ * @param fileReference
* The file id to use for this index.
* @throws HyracksDataException
* If the BufferCache throws while un/pinning or un/latching.
*/
- public void create(int indexFileId) throws HyracksDataException;
+ public void create(FileReference file) throws HyracksDataException;
/**
* Opens the index backed by the given file id.
*
- * @param indexFileId
+ * @param fileReference
* The file id backing this index.
*/
- public void open(int indexFileId) throws HyracksDataException;
+ public void open(FileReference file) throws HyracksDataException;
/**
* Closes the index.
@@ -61,15 +62,19 @@
* on the same IIndex
*
* @returns IIndexAccessor An accessor for this tree.
- * @param modificationCallback TODO
- * @param searchCallback TODO
+ * @param modificationCallback
+ * TODO
+ * @param searchCallback
+ * TODO
*/
public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback);
/**
- * @param fillFactor TODO
- * @throws IndexException TODO
+ * @param fillFactor
+ * TODO
+ * @throws IndexException
+ * TODO
*/
public IIndexBulkLoader createBulkLoader(float fillFactor) throws IndexException;
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
index 3f2df81..95b14d5 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
@@ -27,8 +27,6 @@
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
import edu.uci.ics.hyracks.storage.common.file.IIndexArtifactMap;
public abstract class IndexDataflowHelper {
@@ -48,30 +46,9 @@
}
public void init(boolean forceCreate) throws HyracksDataException {
- IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
- IFileMapProvider fileMapProvider = opDesc.getStorageManager().getFileMapProvider(ctx);
IndexRegistry<IIndex> indexRegistry = opDesc.getIndexRegistryProvider().getRegistry(ctx);
- FileReference fileRef = getFilereference();
IIndexArtifactMap indexArtifactMap = opDesc.getStorageManager().getIndexArtifactMap(ctx);
- int fileId = -1;
- boolean fileIsMapped = false;
- synchronized (fileMapProvider) {
- fileIsMapped = fileMapProvider.isMapped(fileRef);
- if (!fileIsMapped) {
- bufferCache.createFile(fileRef);
- }
- fileId = fileMapProvider.lookupFileId(fileRef);
- try {
- // Also creates the file if it doesn't exist yet.
- bufferCache.openFile(fileId);
- } catch (HyracksDataException e) {
- // Revert state of buffer cache since file failed to open.
- if (!fileIsMapped) {
- bufferCache.deleteFile(fileId, false);
- }
- throw e;
- }
- }
+ FileReference fileRef = getFilereference();
// check whether the requested index instance already exists by
// retrieving indexRegistry with resourceId
@@ -94,8 +71,6 @@
// get the corresponding resourceId with the fullDir
long resourceId = indexArtifactMap.get(fullDir);
- // Only set indexFileId member after openFile() succeeds.
- indexFileId = fileId;
// Create new index instance and register it.
synchronized (indexRegistry) {
// Check if the index has already been registered.
@@ -106,7 +81,7 @@
register = true;
}
if (forceCreate) {
- index.create(indexFileId);
+ index.create(fileRef);
// Create new resourceId
try {
resourceId = indexArtifactMap.create(baseDir, ioDeviceHandles);
@@ -114,12 +89,12 @@
throw new HyracksDataException(e);
}
}
- index.open(indexFileId);
+ index.open(fileRef);
if (register) {
indexRegistry.register(resourceId, index);
}
}
-
+
//set operationCallback object
modificationOperationCallback = opDesc.getOpCallbackProvider().getModificationOperationCallback(resourceId);
searchOperationCallback = opDesc.getOpCallbackProvider().getSearchOperationCallback(resourceId);
@@ -133,11 +108,7 @@
}
public void deinit() throws HyracksDataException {
- if (indexFileId != -1) {
- IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
- bufferCache.closeFile(indexFileId);
- indexFileId = -1;
- }
+ index.close();
}
public IIndex getIndex() {
@@ -159,13 +130,13 @@
public IOperationCallbackProvider getOpCallbackProvider() {
return opDesc.getOpCallbackProvider();
}
-
+
public IModificationOperationCallback getModificationOperationCallback() {
return modificationOperationCallback;
}
-
+
public ISearchOperationCallback getSearchOperationCallback() {
return searchOperationCallback;
}
-
+
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorNodePushable.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorNodePushable.java
index 62bfe12..e30ce85 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorNodePushable.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorNodePushable.java
@@ -22,8 +22,8 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
public class TreeIndexBulkLoadOperatorNodePushable extends AbstractUnaryInputSinkOperatorNodePushable {
private float fillFactor;
@@ -54,10 +54,10 @@
try {
treeIndexHelper.init(false);
treeIndex = (ITreeIndex) treeIndexHelper.getIndex();
- treeIndex.open(treeIndexHelper.getIndexFileId());
bulkLoader = treeIndex.createBulkLoader(fillFactor);
} catch (Exception e) {
// cleanup in case of failure
+ System.out.println("help");
treeIndexHelper.deinit();
throw new HyracksDataException(e);
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
index 50486f2..df82359 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
@@ -29,6 +29,7 @@
import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexStats;
import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexStatsGatherer;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class TreeIndexStatsOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable {
private final TreeIndexDataflowHelper treeIndexHelper;
@@ -58,8 +59,11 @@
treeIndexHelper.init(false);
ITreeIndex treeIndex = (ITreeIndex) treeIndexHelper.getIndex();
IBufferCache bufferCache = treeIndexHelper.getOperatorDescriptor().getStorageManager().getBufferCache(ctx);
- statsGatherer = new TreeIndexStatsGatherer(bufferCache, treeIndex.getFreePageManager(),
- treeIndexHelper.getIndexFileId(), treeIndex.getRootPageId());
+ IFileMapProvider fileMapProvider = treeIndexHelper.getOperatorDescriptor().getStorageManager()
+ .getFileMapProvider(ctx);
+ int indexFileId = fileMapProvider.lookupFileId(treeIndexHelper.getFilereference());
+ statsGatherer = new TreeIndexStatsGatherer(bufferCache, treeIndex.getFreePageManager(), indexFileId,
+ treeIndex.getRootPageId());
TreeIndexStats stats = statsGatherer.gatherStats(treeIndex.getLeafFrameFactory().createFrame(), treeIndex
.getInteriorFrameFactory().createFrame(), treeIndex.getFreePageManager().getMetaDataFrameFactory()
.createFrame());
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java
index 157b563..1c373d5 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java
@@ -16,19 +16,21 @@
package edu.uci.ics.hyracks.storage.am.common.freepage;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManagerFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-public class LinkedListFreePageManagerFactory {
+public class LinkedListFreePageManagerFactory implements IFreePageManagerFactory {
- private final ITreeIndexMetaDataFrameFactory metaDataFrameFactory;
- private final IBufferCache bufferCache;
-
- public LinkedListFreePageManagerFactory(IBufferCache bufferCache, ITreeIndexMetaDataFrameFactory metaDataFrameFactory) {
- this.metaDataFrameFactory = metaDataFrameFactory;
- this.bufferCache = bufferCache;
- }
-
+ private final ITreeIndexMetaDataFrameFactory metaDataFrameFactory;
+ private final IBufferCache bufferCache;
+
+ public LinkedListFreePageManagerFactory(IBufferCache bufferCache,
+ ITreeIndexMetaDataFrameFactory metaDataFrameFactory) {
+ this.metaDataFrameFactory = metaDataFrameFactory;
+ this.bufferCache = bufferCache;
+ }
+
public IFreePageManager createFreePageManager() {
return new LinkedListFreePageManager(bufferCache, 0, metaDataFrameFactory);
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/AbstractTreeIndex.java
index bb98306..271764b 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/AbstractTreeIndex.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/AbstractTreeIndex.java
@@ -1,11 +1,25 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.common.impls;
import java.util.ArrayList;
-import java.util.logging.Level;
-import java.util.logging.Logger;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
@@ -21,24 +35,35 @@
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public abstract class AbstractTreeIndex implements ITreeIndex {
protected final static int rootPage = 1;
protected final IBufferCache bufferCache;
+
+ protected final IFileMapProvider fileMapProvider;
+
protected final IFreePageManager freePageManager;
+
protected final ITreeIndexFrameFactory interiorFrameFactory;
+
protected final ITreeIndexFrameFactory leafFrameFactory;
+
protected final IBinaryComparatorFactory[] cmpFactories;
+
protected final int fieldCount;
+ protected FileReference file;
+
protected int fileId;
- public AbstractTreeIndex(IBufferCache bufferCache, IFreePageManager freePageManager,
- ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
- IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
+ public AbstractTreeIndex(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ IFreePageManager freePageManager, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
this.bufferCache = bufferCache;
+ this.fileMapProvider = fileMapProvider;
this.freePageManager = freePageManager;
this.interiorFrameFactory = interiorFrameFactory;
this.leafFrameFactory = leafFrameFactory;
@@ -46,6 +71,61 @@
this.fieldCount = fieldCount;
}
+ private void initFile(FileReference file, boolean create) throws HyracksDataException {
+ this.file = file;
+ fileId = -1;
+ boolean fileIsMapped = false;
+ synchronized (fileMapProvider) {
+ fileIsMapped = fileMapProvider.isMapped(file);
+ if (!fileIsMapped) {
+ bufferCache.createFile(file);
+ }
+ fileId = fileMapProvider.lookupFileId(file);
+ try {
+ // Also creates the file if it doesn't exist yet.
+ bufferCache.openFile(fileId);
+ } catch (HyracksDataException e) {
+ // Revert state of buffer cache since file failed to open.
+ if (!fileIsMapped) {
+ bufferCache.deleteFile(fileId, false);
+ }
+ throw e;
+ }
+ }
+ freePageManager.open(fileId);
+
+ if (create) {
+ ITreeIndexFrame frame = leafFrameFactory.createFrame();
+ ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
+
+ freePageManager.init(metaFrame, rootPage);
+
+ ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true);
+ rootNode.acquireWriteLatch();
+ try {
+ frame.setPage(rootNode);
+ frame.initBuffer((byte) 0);
+ } finally {
+ rootNode.releaseWriteLatch();
+ bufferCache.unpin(rootNode);
+ }
+ }
+ }
+
+ public synchronized void create(FileReference file) throws HyracksDataException {
+ initFile(file, true);
+ }
+
+ public void open(FileReference file) throws HyracksDataException {
+ initFile(file, false);
+ }
+
+ public void close() throws HyracksDataException {
+ bufferCache.closeFile(fileId);
+ freePageManager.close();
+ fileId = -1;
+ }
+
public boolean isEmptyTree(ITreeIndexFrame frame) throws HyracksDataException {
ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
rootNode.acquireReadLatch();
@@ -62,41 +142,6 @@
}
}
- public synchronized void create(int fileId) throws HyracksDataException {
- ITreeIndexFrame frame = leafFrameFactory.createFrame();
- ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
- freePageManager.open(fileId);
- freePageManager.init(metaFrame, rootPage);
-
- ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true);
- rootNode.acquireWriteLatch();
- try {
- frame.setPage(rootNode);
- frame.initBuffer((byte) 0);
- } finally {
- rootNode.releaseWriteLatch();
- bufferCache.unpin(rootNode);
- }
- }
-
- public void open(int fileId) {
- this.fileId = fileId;
- freePageManager.open(fileId);
- }
-
- public void close() {
- fileId = -1;
- freePageManager.close();
- }
-
- public int getFileId() {
- return fileId;
- }
-
- public IBufferCache getBufferCache() {
- return bufferCache;
- }
-
public byte getTreeHeight(ITreeIndexFrame frame) throws HyracksDataException {
ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
rootNode.acquireReadLatch();
@@ -109,6 +154,18 @@
}
}
+ public int getFileId() {
+ return fileId;
+ }
+
+ public FileReference getFileReference() {
+ return file;
+ }
+
+ public IBufferCache getBufferCache() {
+ return bufferCache;
+ }
+
public ITreeIndexFrameFactory getInteriorFrameFactory() {
return interiorFrameFactory;
}
@@ -135,27 +192,24 @@
public abstract IIndexBulkLoader createBulkLoader(float fillFactor) throws TreeIndexException;
- public TreeIndexInsertBulkLoader createInsertBulkLoader() throws TreeIndexException {
- final Logger LOGGER = Logger.getLogger(TreeIndexInsertBulkLoader.class.getName());
-
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Using insert bulkload. This might negatively impact your performance.");
- }
-
- return new TreeIndexInsertBulkLoader();
- }
-
public abstract class AbstractTreeIndexBulkLoader implements IIndexBulkLoader {
protected final MultiComparator cmp;
+
protected final int slotSize;
+
protected final int leafMaxBytes;
+
protected final int interiorMaxBytes;
- // we maintain a frontier of nodes for each level
+
protected final ArrayList<NodeFrontier> nodeFrontiers = new ArrayList<NodeFrontier>();
+
protected final ITreeIndexMetaDataFrame metaFrame;
+
protected final ITreeIndexTupleWriter tupleWriter;
- protected ITreeIndexFrame leafFrame, interiorFrame;
+ protected ITreeIndexFrame leafFrame;
+
+ protected ITreeIndexFrame interiorFrame;
public AbstractTreeIndexBulkLoader(float fillFactor) throws TreeIndexException, HyracksDataException {
leafFrame = leafFrameFactory.createFrame();
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexDataflowHelper.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexDataflowHelper.java
index 7f5f913..f04c02c 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexDataflowHelper.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexDataflowHelper.java
@@ -52,6 +52,7 @@
invIndexOpDesc.getInvListsTypeTraits());
return new InvertedIndex(opDesc.getStorageManager().getBufferCache(ctx), btree,
invIndexOpDesc.getInvListsTypeTraits(), invIndexOpDesc.getInvListsComparatorFactories(),
- invListBuilder, invIndexOpDesc.getTokenizerFactory().createTokenizer());
+ invListBuilder, invIndexOpDesc.getTokenizerFactory().createTokenizer(), opDesc.getStorageManager()
+ .getFileMapProvider(ctx));
}
}
\ No newline at end of file
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
index 91cac3f..ebd09b7 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
@@ -21,6 +21,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
@@ -50,11 +51,11 @@
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
/**
* An inverted index consists of two files: 1. a file storing (paginated)
* inverted lists 2. a BTree-file mapping from tokens to inverted lists.
- *
* Implemented features: bulk loading and searching (based on T-Occurrence) Not
* implemented features: updates (insert/update/delete) Limitations: a query
* cannot exceed the size of a Hyracks frame.
@@ -65,7 +66,9 @@
private BTree btree;
private int rootPageId = 0;
private IBufferCache bufferCache;
+ private IFileMapProvider fileMapProvider;
private int fileId;
+ private FileReference file;
private final ITypeTraits[] invListTypeTraits;
private final IBinaryComparatorFactory[] invListCmpFactories;
private final IInvertedListBuilder invListBuilder;
@@ -75,8 +78,9 @@
public InvertedIndex(IBufferCache bufferCache, BTree btree, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, IInvertedListBuilder invListBuilder,
- IBinaryTokenizer tokenizer) {
+ IBinaryTokenizer tokenizer, IFileMapProvider fileMapProvider) {
this.bufferCache = bufferCache;
+ this.fileMapProvider = fileMapProvider;
this.btree = btree;
this.invListTypeTraits = invListTypeTraits;
this.invListCmpFactories = invListCmpFactories;
@@ -86,13 +90,38 @@
this.numInvListKeys = invListCmpFactories.length;
}
- @Override
- public void open(int fileId) {
- this.fileId = fileId;
+ private void initFile(FileReference file, boolean create) throws HyracksDataException {
+ this.file = file;
+ fileId = -1;
+ boolean fileIsMapped = false;
+ synchronized (fileMapProvider) {
+ fileIsMapped = fileMapProvider.isMapped(file);
+ if (!fileIsMapped) {
+ bufferCache.createFile(file);
+ }
+ fileId = fileMapProvider.lookupFileId(file);
+ try {
+ // Also creates the file if it doesn't exist yet.
+ bufferCache.openFile(fileId);
+ } catch (HyracksDataException e) {
+ // Revert state of buffer cache since file failed to open.
+ if (!fileIsMapped) {
+ bufferCache.deleteFile(fileId, false);
+ }
+ throw e;
+ }
+ }
+
}
@Override
- public void create(int indexFileId) throws HyracksDataException {
+ public void create(FileReference file) throws HyracksDataException {
+ initFile(file, true);
+ }
+
+ @Override
+ public void open(FileReference file) throws HyracksDataException {
+ initFile(file, false);
}
@Override
@@ -187,7 +216,6 @@
* The next invListCmp.getKeyFieldCount() fields in tuple are keys of the
* inverted list (e.g., primary key).
* Key fields of inverted list are fixed size.
- *
*/
@Override
public void add(ITupleReference tuple) throws HyracksDataException {
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
index 907a2b5..0582dbd 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
@@ -32,6 +32,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
+import edu.uci.ics.hyracks.storage.common.smi.TransientFileMapManager;
public class LSMBTreeDataflowHelper extends TreeIndexDataflowHelper {
private static int DEFAULT_MEM_PAGE_SIZE = 32768;
@@ -68,16 +69,16 @@
public ITreeIndex createIndexInstance() throws HyracksDataException {
ITreeIndexMetaDataFrameFactory metaDataFrameFactory = new LIFOMetaDataFrameFactory();
InMemoryBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), memPageSize,
- memNumPages);
+ memNumPages, new TransientFileMapManager());
IFileSplitProvider fileSplitProvider = opDesc.getFileSplitProvider();
FileReference file = fileSplitProvider.getFileSplits()[partition].getLocalFile();
if (file.getFile().exists() && !file.getFile().isDirectory()) {
file.delete();
}
InMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(memNumPages, metaDataFrameFactory);
- return LSMBTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ctx.getIOManager(), file.getFile()
- .getPath(), opDesc.getStorageManager().getBufferCache(ctx), opDesc.getStorageManager()
- .getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc
- .getTreeIndexComparatorFactories(), flushController, mergePolicy, opTracker, ioScheduler);
+ return LSMBTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ctx.getIOManager(), file, opDesc
+ .getStorageManager().getBufferCache(ctx), opDesc.getStorageManager().getFileMapProvider(ctx),
+ treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc.getTreeIndexComparatorFactories(), flushController,
+ mergePolicy, opTracker, ioScheduler);
}
}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 1e5e0cd..2565d3c 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -54,6 +54,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMHarness;
@@ -69,6 +70,7 @@
// In-memory components.
private final BTree memBTree;
+ private final FileReference memBtreeFile = new FileReference(new File("memBtree"));
private final InMemoryFreePageManager memFreePageManager;
// On-disk components.
@@ -98,8 +100,8 @@
BTreeFactory diskBTreeFactory, BTreeFactory bulkLoadBTreeFactory, IFileMapProvider diskFileMapProvider,
int fieldCount, IBinaryComparatorFactory[] cmpFactories, ILSMFlushController flushController,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOScheduler ioScheduler) {
- memBTree = new BTree(memBufferCache, memFreePageManager, interiorFrameFactory, insertLeafFrameFactory,
- cmpFactories, fieldCount);
+ memBTree = new BTree(memBufferCache, ((InMemoryBufferCache) memBufferCache).getFileMapProvider(),
+ memFreePageManager, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, fieldCount);
this.memFreePageManager = memFreePageManager;
this.insertLeafFrameFactory = insertLeafFrameFactory;
this.deleteLeafFrameFactory = deleteLeafFrameFactory;
@@ -115,8 +117,8 @@
}
@Override
- public void create(int indexFileId) throws HyracksDataException {
- memBTree.create(indexFileId);
+ public void create(FileReference fileReference) throws HyracksDataException {
+ memBTree.create(memBtreeFile);
fileManager.createDirs();
}
@@ -124,18 +126,19 @@
* Opens LSMBTree, cleaning up invalid files from base dir, and registering
* all valid files as on-disk BTrees.
*
- * @param indexFileId
+ * @param fileReference
* Dummy file id used for in-memory BTree.
* @throws HyracksDataException
*/
@Override
- public void open(int indexFileId) throws HyracksDataException {
+ public void open(FileReference fileReference) throws HyracksDataException {
synchronized (this) {
if (isOpen) {
return;
}
- memBTree.open(indexFileId);
+ memBTree.open(memBtreeFile);
BTree dummyBTree = diskBTreeFactory.createIndexInstance();
+ dummyBTree.create(new FileReference(new File("dummy")));
List<Object> validFileNames = fileManager.cleanupAndGetValidFiles(dummyBTree, componentFinalizer);
for (Object o : validFileNames) {
String fileName = (String) o;
@@ -155,8 +158,6 @@
}
for (Object o : diskBTrees) {
BTree btree = (BTree) o;
- diskBufferCache.closeFile(btree.getFileId());
- diskBufferCache.deleteFile(btree.getFileId(), false);
btree.close();
}
diskBTrees.clear();
@@ -211,7 +212,7 @@
@Override
public void resetInMemoryComponent() throws HyracksDataException {
memFreePageManager.reset();
- memBTree.create(memBTree.getFileId());
+ memBTree.create(memBTree.getFileReference());
}
private BTree createBulkLoadTarget() throws HyracksDataException {
@@ -239,18 +240,13 @@
private BTree createDiskBTree(BTreeFactory factory, FileReference fileRef, boolean createBTree)
throws HyracksDataException {
- // File will be deleted during cleanup of merge().
- diskBufferCache.createFile(fileRef);
- int diskBTreeFileId = diskFileMapProvider.lookupFileId(fileRef);
- // File will be closed during cleanup of merge().
- diskBufferCache.openFile(diskBTreeFileId);
// Create new BTree instance.
BTree diskBTree = factory.createIndexInstance();
if (createBTree) {
- diskBTree.create(diskBTreeFileId);
+ diskBTree.create(fileRef);
}
// BTree will be closed during cleanup of merge().
- diskBTree.open(diskBTreeFileId);
+ diskBTree.open(fileRef);
return diskBTree;
}
@@ -332,8 +328,6 @@
for (Object o : mergedComponents) {
BTree oldBTree = (BTree) o;
FileReference fileRef = diskFileMapProvider.lookupFileName(oldBTree.getFileId());
- diskBufferCache.closeFile(oldBTree.getFileId());
- diskBufferCache.deleteFile(oldBTree.getFileId(), false);
oldBTree.close();
fileRef.getFile().delete();
}
diff --git a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
index cd3f159..fa47a53 100644
--- a/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
+++ b/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
@@ -17,9 +17,11 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManagerFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
@@ -41,7 +43,7 @@
public class LSMBTreeUtils {
public static LSMBTree createLSMTree(InMemoryBufferCache memBufferCache,
- InMemoryFreePageManager memFreePageManager, IIOManager ioManager, String onDiskDir,
+ InMemoryFreePageManager memFreePageManager, IIOManager ioManager, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] cmpFactories, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
ILSMOperationTracker opTracker, ILSMIOScheduler ioScheduler) {
@@ -56,17 +58,17 @@
ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
+ IFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
metaFrameFactory);
- BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, cmpFactories,
- typeTraits.length, interiorFrameFactory, copyTupleLeafFrameFactory);
- BTreeFactory bulkLoadBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, cmpFactories,
- typeTraits.length, interiorFrameFactory, insertLeafFrameFactory);
- ILSMFileManager fileNameManager = new LSMTreeFileManager(ioManager, diskFileMapProvider, onDiskDir);
+ BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+ interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
+ BTreeFactory bulkLoadBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
+ ILSMFileManager fileNameManager = new LSMTreeFileManager(ioManager, diskFileMapProvider, file);
LSMBTree lsmTree = new LSMBTree(memBufferCache, memFreePageManager, interiorFrameFactory,
insertLeafFrameFactory, deleteLeafFrameFactory, fileNameManager, diskBTreeFactory,
- bulkLoadBTreeFactory, diskFileMapProvider, typeTraits.length, cmpFactories, flushController, mergePolicy,
- opTracker, ioScheduler);
+ bulkLoadBTreeFactory, diskFileMapProvider, typeTraits.length, cmpFactories, flushController,
+ mergePolicy, opTracker, ioScheduler);
return lsmTree;
}
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryBufferCache.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryBufferCache.java
index f554778..4432a16 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryBufferCache.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryBufferCache.java
@@ -28,15 +28,20 @@
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPageInternal;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapManager;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class InMemoryBufferCache implements IBufferCacheInternal {
protected final ICacheMemoryAllocator allocator;
+ protected final IFileMapManager fileMapManager;
protected final int pageSize;
protected final CachedPage[] pages;
protected final List<CachedPage> overflowPages = new ArrayList<CachedPage>();
- public InMemoryBufferCache(ICacheMemoryAllocator allocator, int pageSize, int numPages) {
+ public InMemoryBufferCache(ICacheMemoryAllocator allocator, int pageSize, int numPages,
+ IFileMapManager fileMapManager) {
this.allocator = allocator;
+ this.fileMapManager = fileMapManager;
this.pageSize = pageSize;
ByteBuffer[] buffers = allocator.allocate(pageSize, numPages);
pages = new CachedPage[buffers.length];
@@ -93,7 +98,9 @@
@Override
public void createFile(FileReference fileRef) throws HyracksDataException {
- // Do nothing.
+ synchronized (fileMapManager) {
+ fileMapManager.registerFile(fileRef);
+ }
}
@Override
@@ -108,7 +115,9 @@
@Override
public void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException {
- // Do nothing.
+ synchronized (fileMapManager) {
+ fileMapManager.unregisterFile(fileId);
+ }
}
@Override
@@ -182,4 +191,8 @@
@Override
public void flushDirtyPage(ICachedPage page) throws HyracksDataException {
}
+
+ public IFileMapProvider getFileMapProvider() {
+ return fileMapManager;
+ }
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BTreeFactory.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
index 8f502fa..bf642c2 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
@@ -17,22 +17,24 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManagerFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class BTreeFactory extends TreeFactory<BTree> {
- public BTreeFactory(IBufferCache bufferCache, LinkedListFreePageManagerFactory freePageManagerFactory,
- IBinaryComparatorFactory[] cmpFactories, int fieldCount, ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory) {
- super(bufferCache, freePageManagerFactory, cmpFactories, fieldCount, interiorFrameFactory, leafFrameFactory);
+ public BTreeFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ IFreePageManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
+ super(bufferCache, fileMapProvider, freePageManagerFactory, interiorFrameFactory, leafFrameFactory,
+ cmpFactories, fieldCount);
}
@Override
public BTree createIndexInstance() {
- return new BTree(bufferCache, freePageManagerFactory.createFreePageManager(), interiorFrameFactory, leafFrameFactory,
- cmpFactories, fieldCount);
+ return new BTree(bufferCache, fileMapProvider, freePageManagerFactory.createFreePageManager(),
+ interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount);
}
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java
index b643049..2f98b6f 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java
@@ -43,7 +43,8 @@
protected final IFileMapProvider fileMapProvider;
// baseDir should reflect dataset name and partition name.
- protected final String baseDir;
+ protected FileReference file;
+ protected String baseDir;
protected final Format formatter = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS");
protected final Comparator<String> cmp = new FileNameComparator();
protected final Comparator<ComparableFileName> recencyCmp = new RecencyComparator();
@@ -57,13 +58,14 @@
}
};
- public LSMTreeFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, String baseDir) {
+ public LSMTreeFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file) {
+ this.file = file;
+ this.baseDir = file.getFile().getPath();
if (!baseDir.endsWith(System.getProperty("file.separator"))) {
baseDir += System.getProperty("file.separator");
}
this.fileMapProvider = fileMapProvider;
this.ioManager = ioManager;
- this.baseDir = baseDir;
}
@Override
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java
index d5b5650..90ef20d 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java
@@ -16,29 +16,38 @@
package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManagerFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public abstract class TreeFactory<T extends ITreeIndex> {
- protected IBufferCache bufferCache;
- protected int fieldCount;
- protected IBinaryComparatorFactory[] cmpFactories;
- protected ITreeIndexFrameFactory interiorFrameFactory;
- protected ITreeIndexFrameFactory leafFrameFactory;
- protected LinkedListFreePageManagerFactory freePageManagerFactory;
+ protected final IBufferCache bufferCache;
- public TreeFactory(IBufferCache bufferCache, LinkedListFreePageManagerFactory freePageManagerFactory,
- IBinaryComparatorFactory[] cmpFactories, int fieldCount, ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory) {
+ protected final IFileMapProvider fileMapProvider;
+
+ protected final IFreePageManagerFactory freePageManagerFactory;
+
+ protected final ITreeIndexFrameFactory interiorFrameFactory;
+
+ protected final ITreeIndexFrameFactory leafFrameFactory;
+
+ protected final IBinaryComparatorFactory[] cmpFactories;
+
+ protected final int fieldCount;
+
+ public TreeFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ IFreePageManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
this.bufferCache = bufferCache;
- this.fieldCount = fieldCount;
- this.cmpFactories = cmpFactories;
+ this.fileMapProvider = fileMapProvider;
+ this.freePageManagerFactory = freePageManagerFactory;
this.interiorFrameFactory = interiorFrameFactory;
this.leafFrameFactory = leafFrameFactory;
- this.freePageManagerFactory = freePageManagerFactory;
+ this.cmpFactories = cmpFactories;
+ this.fieldCount = fieldCount;
}
public abstract T createIndexInstance();
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeIndexComponentFinalizer.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeIndexComponentFinalizer.java
index 34287cb..e6dc66b 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeIndexComponentFinalizer.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeIndexComponentFinalizer.java
@@ -40,10 +40,7 @@
ITreeIndex treeIndex = (ITreeIndex) lsmComponent;
IBufferCache bufferCache = treeIndex.getBufferCache();
FileReference fileRef = new FileReference(file);
- bufferCache.createFile(fileRef);
- int fileId = fileMapProvider.lookupFileId(fileRef);
- bufferCache.openFile(fileId);
- treeIndex.open(fileId);
+ treeIndex.open(fileRef);
try {
int metadataPage = treeIndex.getFreePageManager().getFirstMetadataPage();
ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory()
@@ -60,8 +57,6 @@
}
} finally {
treeIndex.close();
- bufferCache.closeFile(fileId);
- bufferCache.deleteFile(fileId, false);
}
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
index c7bd9a5..d311e56 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
@@ -95,15 +95,15 @@
InMemoryFreePageManager memFreePageManager = new LSMRTreeInMemoryFreePageManager(memNumPages,
metaDataFrameFactory);
- return createLSMTree(memBufferCache, memFreePageManager, ctx.getIOManager(), file.getFile().getPath(), opDesc
- .getStorageManager().getBufferCache(ctx), opDesc.getStorageManager().getFileMapProvider(ctx),
+ return createLSMTree(memBufferCache, memFreePageManager, ctx.getIOManager(), file, opDesc.getStorageManager()
+ .getBufferCache(ctx), opDesc.getStorageManager().getFileMapProvider(ctx),
treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc.getTreeIndexComparatorFactories(),
btreeComparatorFactories, valueProviderFactories, rtreePolicyType);
}
protected abstract ITreeIndex createLSMTree(IBufferCache memBufferCache,
- InMemoryFreePageManager memFreePageManager, IIOManager ioManager, String onDiskDir,
+ InMemoryFreePageManager memFreePageManager, IIOManager ioManager, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType)
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
index cf9ac0a..4023dd2 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
@@ -19,6 +19,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
@@ -56,12 +57,12 @@
@Override
protected ITreeIndex createLSMTree(IBufferCache memBufferCache, InMemoryFreePageManager memFreePageManager,
- IIOManager ioManager, String onDiskDir, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider,
+ IIOManager ioManager, FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider,
ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType) throws HyracksDataException {
try {
- return LSMRTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ioManager, onDiskDir,
+ return LSMRTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ioManager, file,
diskBufferCache, diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories,
valueProviderFactories, rtreePolicyType, flushController, mergePolicy, opTracker, ioScheduler);
} catch (TreeIndexException e) {
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
index 1371161..d32e691 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
@@ -19,6 +19,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
@@ -55,13 +56,13 @@
@Override
protected ITreeIndex createLSMTree(IBufferCache memBufferCache, InMemoryFreePageManager memFreePageManager,
- IIOManager ioManager, String onDiskDir, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider,
+ IIOManager ioManager, FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider,
ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType) throws HyracksDataException {
try {
return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(memBufferCache, memFreePageManager, ioManager,
- onDiskDir, diskBufferCache, diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories,
+ file, diskBufferCache, diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories,
valueProviderFactories, rtreePolicyType, flushController, mergePolicy, opTracker, ioScheduler);
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index d105cea..498db30 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -15,6 +15,7 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
+import java.io.File;
import java.util.LinkedList;
import java.util.List;
@@ -43,6 +44,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeFactory;
@@ -81,8 +83,8 @@
// In-memory components.
protected final LSMRTreeComponent memComponent;
protected final InMemoryFreePageManager memFreePageManager;
- protected final static int MEM_RTREE_FILE_ID = 0;
- protected final static int MEM_BTREE_FILE_ID = 1;
+ protected FileReference memRtreeFile = new FileReference(new File("memrtree"));
+ protected FileReference memBtreeFile = new FileReference(new File("membtree"));
// This is used to estimate number of tuples in the memory RTree and BTree
// for efficient memory allocation in the sort operation prior to flushing
@@ -119,11 +121,10 @@
IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMFlushController flushController,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOScheduler ioScheduler) {
- RTree memRTree = new RTree(memBufferCache, memFreePageManager, rtreeInteriorFrameFactory,
- rtreeLeafFrameFactory, rtreeCmpFactories, fieldCount);
- // TODO: Do we need another operation callback here?
- BTree memBTree = new BTree(memBufferCache, memFreePageManager, btreeInteriorFrameFactory,
- btreeLeafFrameFactory, btreeCmpFactories, fieldCount);
+ RTree memRTree = new RTree(memBufferCache, ((InMemoryBufferCache) memBufferCache).getFileMapProvider(),
+ memFreePageManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories, fieldCount);
+ BTree memBTree = new BTree(memBufferCache, ((InMemoryBufferCache) memBufferCache).getFileMapProvider(),
+ memFreePageManager, btreeInteriorFrameFactory, btreeLeafFrameFactory, btreeCmpFactories, fieldCount);
memComponent = new LSMRTreeComponent(memRTree, memBTree);
this.memFreePageManager = memFreePageManager;
this.diskBufferCache = diskRTreeFactory.getBufferCache();
@@ -144,16 +145,16 @@
}
@Override
- public void create(int indexFileId) throws HyracksDataException {
- memComponent.getRTree().create(MEM_RTREE_FILE_ID);
- memComponent.getBTree().create(MEM_BTREE_FILE_ID);
+ public void create(FileReference fileReference) throws HyracksDataException {
+ memComponent.getRTree().create(memRtreeFile);
+ memComponent.getBTree().create(memBtreeFile);
fileManager.createDirs();
}
@Override
- public void open(int indexFileId) throws HyracksDataException {
- memComponent.getRTree().open(MEM_RTREE_FILE_ID);
- memComponent.getBTree().open(MEM_BTREE_FILE_ID);
+ public void open(FileReference fileReference) throws HyracksDataException {
+ memComponent.getRTree().open(memRtreeFile);
+ memComponent.getBTree().open(memBtreeFile);
}
@Override
@@ -165,18 +166,13 @@
@SuppressWarnings("rawtypes")
protected ITreeIndex createDiskTree(TreeFactory diskTreeFactory, FileReference fileRef, boolean createTree)
throws HyracksDataException {
- // File will be deleted during cleanup of merge().
- diskBufferCache.createFile(fileRef);
- int diskTreeFileId = diskFileMapProvider.lookupFileId(fileRef);
- // File will be closed during cleanup of merge().
- diskBufferCache.openFile(diskTreeFileId);
// Create new tree instance.
ITreeIndex diskTree = diskTreeFactory.createIndexInstance();
if (createTree) {
- diskTree.create(diskTreeFileId);
+ diskTree.create(fileRef);
}
// Tree will be closed during cleanup of merge().
- diskTree.open(diskTreeFileId);
+ diskTree.open(fileRef);
return diskTree;
}
@@ -286,8 +282,8 @@
@Override
public void resetInMemoryComponent() throws HyracksDataException {
- memComponent.getRTree().create(MEM_RTREE_FILE_ID);
- memComponent.getBTree().create(MEM_BTREE_FILE_ID);
+ memComponent.getRTree().create(memRtreeFile);
+ memComponent.getBTree().create(memBtreeFile);
memFreePageManager.reset();
memRTreeTuples = 0;
memBTreeTuples = 0;
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 8d25eb0..4561615 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -28,13 +28,13 @@
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
@@ -83,15 +83,17 @@
* Opens LSMRTree, cleaning up invalid files from base dir, and registering
* all valid files as on-disk RTrees and BTrees.
*
- * @param indexFileId
+ * @param fileReference
* Dummy file id.
* @throws HyracksDataException
*/
@Override
- public void open(int indexFileId) throws HyracksDataException {
- super.open(indexFileId);
+ public void open(FileReference fileReference) throws HyracksDataException {
+ super.open(fileReference);
RTree dummyRTree = diskRTreeFactory.createIndexInstance();
+ dummyRTree.create(new FileReference(new File("dummyrtree")));
BTree dummyBTree = diskBTreeFactory.createIndexInstance();
+ dummyBTree.create(new FileReference(new File("dummybtree")));
LSMRTreeComponent dummyComponent = new LSMRTreeComponent(dummyRTree, dummyBTree);
List<Object> validFileNames = fileManager.cleanupAndGetValidFiles(dummyComponent, componentFinalizer);
for (Object o : validFileNames) {
@@ -111,11 +113,7 @@
LSMRTreeComponent diskComponent = (LSMRTreeComponent) o;
RTree rtree = diskComponent.getRTree();
BTree btree = diskComponent.getBTree();
- diskBufferCache.closeFile(rtree.getFileId());
- diskBufferCache.deleteFile(rtree.getFileId(), false);
rtree.close();
- diskBufferCache.closeFile(btree.getFileId());
- diskBufferCache.deleteFile(btree.getFileId(), false);
btree.close();
}
diskComponents.clear();
@@ -187,9 +185,9 @@
IBinaryComparatorFactory[] linearizerArray = { linearizer };
if (rTreeTupleSorter == null) {
- rTreeTupleSorter = new TreeTupleSorter(memRTreeTuples, MEM_RTREE_FILE_ID, linearizerArray,
- rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(), memComponent.getRTree()
- .getBufferCache(), comparatorFields);
+ rTreeTupleSorter = new TreeTupleSorter(memRTreeTuples, memComponent.getRTree().getFileId(),
+ linearizerArray, rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(),
+ memComponent.getRTree().getBufferCache(), comparatorFields);
} else {
rTreeTupleSorter.reset();
}
@@ -300,14 +298,10 @@
LSMRTreeComponent component = (LSMRTreeComponent) o;
BTree oldBTree = component.getBTree();
FileReference btreeFileRef = diskFileMapProvider.lookupFileName(oldBTree.getFileId());
- diskBufferCache.closeFile(oldBTree.getFileId());
- diskBufferCache.deleteFile(oldBTree.getFileId(), false);
oldBTree.close();
btreeFileRef.getFile().delete();
RTree oldRTree = component.getRTree();
FileReference rtreeFileRef = diskFileMapProvider.lookupFileName(oldRTree.getFileId());
- diskBufferCache.closeFile(oldRTree.getFileId());
- diskBufferCache.deleteFile(oldRTree.getFileId(), false);
oldRTree.close();
rtreeFileRef.getFile().delete();
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
index 6487397..31b7ba8 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
@@ -24,6 +24,7 @@
import java.util.List;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFinalizer;
@@ -40,15 +41,15 @@
return !name.startsWith(".") && name.endsWith(BTREE_STRING);
}
};
-
+
private static FilenameFilter rtreeFilter = new FilenameFilter() {
public boolean accept(File dir, String name) {
return !name.startsWith(".") && name.endsWith(RTREE_STRING);
}
};
-
- public LSMRTreeFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, String baseDir) {
- super(ioManager, fileMapProvider, baseDir);
+
+ public LSMRTreeFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file) {
+ super(ioManager, fileMapProvider, file);
}
@Override
@@ -67,14 +68,15 @@
}
@Override
- public List<Object> cleanupAndGetValidFiles(Object lsmComponent, ILSMComponentFinalizer componentFinalizer) throws HyracksDataException {
+ public List<Object> cleanupAndGetValidFiles(Object lsmComponent, ILSMComponentFinalizer componentFinalizer)
+ throws HyracksDataException {
List<Object> validFiles = new ArrayList<Object>();
ArrayList<ComparableFileName> allRTreeFiles = new ArrayList<ComparableFileName>();
ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<ComparableFileName>();
LSMRTreeComponent component = (LSMRTreeComponent) lsmComponent;
-
+
// Gather files from all IODeviceHandles.
- for (IODeviceHandle dev : ioManager.getIODevices()) {
+ for (IODeviceHandle dev : ioManager.getIODevices()) {
cleanupAndGetValidFilesInternal(dev, btreeFilter, component.getBTree(), componentFinalizer, allBTreeFiles);
HashSet<String> btreeFilesSet = new HashSet<String>();
for (ComparableFileName cmpFileName : allBTreeFiles) {
@@ -83,7 +85,8 @@
}
// List of valid RTree files that may or may not have a BTree buddy. Will check for buddies below.
ArrayList<ComparableFileName> tmpAllRTreeFiles = new ArrayList<ComparableFileName>();
- cleanupAndGetValidFilesInternal(dev, rtreeFilter, component.getRTree(), componentFinalizer, tmpAllRTreeFiles);
+ cleanupAndGetValidFilesInternal(dev, rtreeFilter, component.getRTree(), componentFinalizer,
+ tmpAllRTreeFiles);
// Look for buddy BTrees for all valid RTrees.
// If no buddy is found, delete the file, otherwise add the RTree to allRTreeFiles.
for (ComparableFileName cmpFileName : tmpAllRTreeFiles) {
@@ -103,7 +106,7 @@
if (allRTreeFiles.size() != allBTreeFiles.size()) {
throw new HyracksDataException("Unequal number of valid RTree and BTree files found. Aborting cleanup.");
}
-
+
// Trivial cases.
if (allRTreeFiles.isEmpty() || allBTreeFiles.isEmpty()) {
return validFiles;
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeInMemoryBufferCache.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeInMemoryBufferCache.java
index 39065ad..d08bcae 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeInMemoryBufferCache.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeInMemoryBufferCache.java
@@ -21,11 +21,12 @@
import edu.uci.ics.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+import edu.uci.ics.hyracks.storage.common.smi.TransientFileMapManager;
public class LSMRTreeInMemoryBufferCache extends InMemoryBufferCache {
public LSMRTreeInMemoryBufferCache(ICacheMemoryAllocator allocator, int pageSize, int numPages) {
- super(allocator, pageSize, numPages);
+ super(allocator, pageSize, numPages, new TransientFileMapManager());
}
@Override
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index 1fb77a9..3ec91b1 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -28,6 +28,7 @@
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
@@ -35,7 +36,6 @@
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
@@ -87,14 +87,15 @@
* Opens LSMRTree, cleaning up invalid files from base dir, and registering
* all valid files as on-disk RTrees and BTrees.
*
- * @param indexFileId
+ * @param fileReference
* Dummy file id.
* @throws HyracksDataException
*/
@Override
- public void open(int indexFileId) throws HyracksDataException {
- super.open(indexFileId);
+ public void open(FileReference fileReference) throws HyracksDataException {
+ super.open(fileReference);
RTree dummyRTree = diskRTreeFactory.createIndexInstance();
+ dummyRTree.create(new FileReference(new File("dummyrtree")));
List<Object> validFileNames = fileManager.cleanupAndGetValidFiles(dummyRTree, componentFinalizer);
for (Object o : validFileNames) {
String fileName = (String) o;
@@ -108,8 +109,6 @@
public void close() throws HyracksDataException {
for (Object o : diskComponents) {
RTree rtree = (RTree) o;
- diskBufferCache.closeFile(rtree.getFileId());
- diskBufferCache.deleteFile(rtree.getFileId(), false);
rtree.close();
}
diskComponents.clear();
@@ -196,13 +195,13 @@
// Since the LSM-RTree is used as a secondary assumption, the
// primary key will be the last comparator in the BTree comparators
if (rTreeTupleSorter == null) {
- rTreeTupleSorter = new TreeTupleSorter(memRTreeTuples, MEM_RTREE_FILE_ID, linearizerArray,
- rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(), memComponent.getRTree()
- .getBufferCache(), comparatorFields);
+ rTreeTupleSorter = new TreeTupleSorter(memRTreeTuples, memComponent.getRTree().getFileId(),
+ linearizerArray, rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(),
+ memComponent.getRTree().getBufferCache(), comparatorFields);
- bTreeTupleSorter = new TreeTupleSorter(memBTreeTuples, MEM_BTREE_FILE_ID, linearizerArray,
- btreeLeafFrameFactory.createFrame(), btreeLeafFrameFactory.createFrame(), memComponent.getBTree()
- .getBufferCache(), comparatorFields);
+ bTreeTupleSorter = new TreeTupleSorter(memBTreeTuples, memComponent.getBTree().getFileId(),
+ linearizerArray, btreeLeafFrameFactory.createFrame(), btreeLeafFrameFactory.createFrame(),
+ memComponent.getBTree().getBufferCache(), comparatorFields);
} else {
rTreeTupleSorter.reset();
bTreeTupleSorter.reset();
@@ -301,8 +300,6 @@
for (Object o : mergedComponents) {
RTree oldRTree = (RTree) o;
FileReference fileRef = diskFileMapProvider.lookupFileName(oldRTree.getFileId());
- diskBufferCache.closeFile(oldRTree.getFileId());
- diskBufferCache.deleteFile(oldRTree.getFileId(), false);
oldRTree.close();
fileRef.getFile().delete();
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
index f1dbd43..c6ee511 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/RTreeFactory.java
@@ -16,23 +16,26 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManagerFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeFactory;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class RTreeFactory extends TreeFactory<RTree> {
- public RTreeFactory(IBufferCache bufferCache, LinkedListFreePageManagerFactory freePageManagerFactory, IBinaryComparatorFactory[] cmpFactories,
- int fieldCount, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
- super(bufferCache, freePageManagerFactory, cmpFactories, fieldCount, interiorFrameFactory, leafFrameFactory);
+ public RTreeFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ IFreePageManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
+ super(bufferCache, fileMapProvider, freePageManagerFactory, interiorFrameFactory, leafFrameFactory,
+ cmpFactories, fieldCount);
}
@Override
public RTree createIndexInstance() {
- return new RTree(bufferCache, freePageManagerFactory.createFreePageManager(), interiorFrameFactory, leafFrameFactory,
- cmpFactories, fieldCount);
+ return new RTree(bufferCache, fileMapProvider, freePageManagerFactory.createFreePageManager(),
+ interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount);
}
}
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
index f4680f9..b48190e 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
@@ -18,6 +18,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
@@ -55,11 +56,12 @@
public class LSMRTreeUtils {
public static LSMRTree createLSMTree(IBufferCache memBufferCache, InMemoryFreePageManager memFreePageManager,
- IIOManager ioManager, String onDiskDir, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider,
- ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
- IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTracker opTracker, ILSMIOScheduler ioScheduler) throws TreeIndexException {
+ IIOManager ioManager, FileReference file, IBufferCache diskBufferCache,
+ IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
+ IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
+ ILSMFlushController flushController, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOScheduler ioScheduler) throws TreeIndexException {
LSMTypeAwareTupleWriterFactory rtreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
LSMTypeAwareTupleWriterFactory btreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
@@ -75,27 +77,26 @@
LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
metaFrameFactory);
- RTreeFactory diskRTreeFactory = new RTreeFactory(diskBufferCache, freePageManagerFactory, rtreeCmpFactories,
- typeTraits.length, rtreeInteriorFrameFactory, rtreeLeafFrameFactory);
- // TODO: Do we need another operation callback here?
- BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, freePageManagerFactory, btreeCmpFactories,
- typeTraits.length, btreeInteriorFrameFactory, btreeLeafFrameFactory);
+ RTreeFactory diskRTreeFactory = new RTreeFactory(diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+ rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories, typeTraits.length);
+ BTreeFactory diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+ btreeInteriorFrameFactory, btreeLeafFrameFactory, btreeCmpFactories, typeTraits.length);
ILinearizeComparatorFactory linearizer = proposeBestLinearizer(typeTraits, rtreeCmpFactories.length);
int[] comparatorFields = { 0 };
IBinaryComparatorFactory[] linearizerArray = { linearizer };
- ILSMFileManager fileNameManager = new LSMRTreeFileManager(ioManager, diskFileMapProvider, onDiskDir);
+ ILSMFileManager fileNameManager = new LSMRTreeFileManager(ioManager, diskFileMapProvider, file);
LSMRTree lsmTree = new LSMRTree(memBufferCache, memFreePageManager, rtreeInteriorFrameFactory,
rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager,
diskRTreeFactory, diskBTreeFactory, diskFileMapProvider, typeTraits.length, rtreeCmpFactories,
- btreeCmpFactories, linearizer, comparatorFields, linearizerArray, flushController, mergePolicy, opTracker,
- ioScheduler);
+ btreeCmpFactories, linearizer, comparatorFields, linearizerArray, flushController, mergePolicy,
+ opTracker, ioScheduler);
return lsmTree;
}
public static LSMRTreeWithAntiMatterTuples createLSMTreeWithAntiMatterTuples(IBufferCache memBufferCache,
- InMemoryFreePageManager memFreePageManager, IIOManager ioManager, String onDiskDir,
+ InMemoryFreePageManager memFreePageManager, IIOManager ioManager, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
@@ -122,11 +123,12 @@
LinkedListFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
metaFrameFactory);
- RTreeFactory diskRTreeFactory = new RTreeFactory(diskBufferCache, freePageManagerFactory, rtreeCmpFactories,
- typeTraits.length, rtreeInteriorFrameFactory, copyTupleLeafFrameFactory);
+ RTreeFactory diskRTreeFactory = new RTreeFactory(diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+ rtreeInteriorFrameFactory, copyTupleLeafFrameFactory, rtreeCmpFactories, typeTraits.length);
- RTreeFactory bulkLoadRTreeFactory = new RTreeFactory(diskBufferCache, freePageManagerFactory,
- rtreeCmpFactories, typeTraits.length, rtreeInteriorFrameFactory, rtreeLeafFrameFactory);
+ RTreeFactory bulkLoadRTreeFactory = new RTreeFactory(diskBufferCache, diskFileMapProvider,
+ freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories,
+ typeTraits.length);
ILinearizeComparatorFactory linearizer = proposeBestLinearizer(typeTraits, rtreeCmpFactories.length);
// The first field is for the sorted curve (e.g. Hilbert curve), and the
@@ -134,7 +136,7 @@
int[] comparatorFields = { 0, btreeCmpFactories.length - 1 };
IBinaryComparatorFactory[] linearizerArray = { linearizer, btreeCmpFactories[btreeCmpFactories.length - 1] };
- ILSMFileManager fileNameManager = new LSMTreeFileManager(ioManager, diskFileMapProvider, onDiskDir);
+ ILSMFileManager fileNameManager = new LSMTreeFileManager(ioManager, diskFileMapProvider, file);
LSMRTreeWithAntiMatterTuples lsmTree = new LSMRTreeWithAntiMatterTuples(memBufferCache, memFreePageManager,
rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory,
fileNameManager, diskRTreeFactory, bulkLoadRTreeFactory, diskFileMapProvider, typeTraits.length,
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java
index 843c472..c69ab9f 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java
@@ -38,8 +38,8 @@
@Override
public ITreeIndex createIndexInstance() throws HyracksDataException {
- return RTreeUtils.createRTree(treeOpDesc.getStorageManager().getBufferCache(ctx),
- treeOpDesc.getTreeIndexTypeTraits(), valueProviderFactories,
- treeOpDesc.getTreeIndexComparatorFactories(), rtreePolicyType);
+ return RTreeUtils.createRTree(treeOpDesc.getStorageManager().getBufferCache(ctx), treeOpDesc
+ .getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
+ valueProviderFactories, treeOpDesc.getTreeIndexComparatorFactories(), rtreePolicyType);
}
}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
index 6a69d9b..6e18ad4 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
@@ -54,16 +54,18 @@
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class RTree extends AbstractTreeIndex {
// Global node sequence number used for the concurrency control protocol
private final AtomicLong globalNsn;
- public RTree(IBufferCache bufferCache, IFreePageManager freePageManager,
+ public RTree(IBufferCache bufferCache, IFileMapProvider fileMapProvider, IFreePageManager freePageManager,
ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
- super(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount);
+ super(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
+ fieldCount);
globalNsn = new AtomicLong();
}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/util/RTreeUtils.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/util/RTreeUtils.java
index cd7c6f7..306d907 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/util/RTreeUtils.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/util/RTreeUtils.java
@@ -34,11 +34,12 @@
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class RTreeUtils {
- public static RTree createRTree(IBufferCache bufferCache, ITypeTraits[] typeTraits,
- IPrimitiveValueProviderFactory[] valueProviderFactories, IBinaryComparatorFactory[] cmpFactories,
- RTreePolicyType rtreePolicyType) {
+ public static RTree createRTree(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ ITypeTraits[] typeTraits, IPrimitiveValueProviderFactory[] valueProviderFactories,
+ IBinaryComparatorFactory[] cmpFactories, RTreePolicyType rtreePolicyType) {
RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(typeTraits);
ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(tupleWriterFactory,
@@ -48,8 +49,8 @@
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
- RTree rtree = new RTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
- typeTraits.length);
+ RTree rtree = new RTree(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory,
+ cmpFactories, typeTraits.length);
return rtree;
}
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexExamplesTest.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
index 827e6d0..cf5f69c 100644
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
@@ -24,6 +24,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
@@ -36,10 +37,10 @@
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
@@ -54,11 +55,10 @@
protected abstract ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories)
throws TreeIndexException;
- protected abstract int getIndexFileId();
+ protected abstract FileReference getFileReference();
/**
* Fixed-Length Key,Value Example.
- *
* Create a tree index with one fixed-length key field and one fixed-length value
* field. Fill index with random values using insertions (not bulk load).
* Perform scans and range search.
@@ -83,10 +83,10 @@
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- int indexFileId = getIndexFileId();
+ FileReference file = getFileReference();
ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories);
- treeIndex.create(indexFileId);
- treeIndex.open(indexFileId);
+ treeIndex.create(file);
+ treeIndex.open(file);
long start = System.currentTimeMillis();
if (LOGGER.isLoggable(Level.INFO)) {
@@ -136,7 +136,6 @@
/**
* Composite Key Example (Non-Unique Index).
- *
* Create a tree index with two fixed-length key fields and one fixed-length
* value field. Fill index with random values using insertions (not bulk
* load) Perform scans and range search.
@@ -163,10 +162,10 @@
cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
cmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- int indexFileId = getIndexFileId();
+ FileReference file = getFileReference();
ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories);
- treeIndex.create(indexFileId);
- treeIndex.open(indexFileId);
+ treeIndex.create(file);
+ treeIndex.open(file);
long start = System.currentTimeMillis();
if (LOGGER.isLoggable(Level.INFO)) {
@@ -241,10 +240,10 @@
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
cmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
- int indexFileId = getIndexFileId();
+ FileReference file = getFileReference();
ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories);
- treeIndex.create(indexFileId);
- treeIndex.open(indexFileId);
+ treeIndex.create(file);
+ treeIndex.open(file);
long start = System.currentTimeMillis();
if (LOGGER.isLoggable(Level.INFO)) {
@@ -296,7 +295,6 @@
/**
* Deletion Example.
- *
* Create a BTree with one variable-length key field and one variable-length
* value field. Fill B-tree with random values using insertions, then delete
* entries one-by-one. Repeat procedure a few times on same BTree.
@@ -321,10 +319,10 @@
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
cmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
- int indexFileId = getIndexFileId();
+ FileReference file = getFileReference();
ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories);
- treeIndex.create(indexFileId);
- treeIndex.open(indexFileId);
+ treeIndex.create(file);
+ treeIndex.open(file);
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
@@ -398,7 +396,6 @@
/**
* Update example.
- *
* Create a BTree with one variable-length key field and one variable-length
* value field. Fill B-tree with random values using insertions, then update
* entries one-by-one. Repeat procedure a few times on same BTree.
@@ -423,10 +420,10 @@
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
cmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
- int indexFileId = getIndexFileId();
+ FileReference file = getFileReference();
ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories);
- treeIndex.create(indexFileId);
- treeIndex.open(indexFileId);
+ treeIndex.create(file);
+ treeIndex.open(file);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Inserting into tree...");
@@ -485,10 +482,8 @@
/**
* Bulk load example.
- *
* Load a tree with 100,000 tuples. BTree has a composite key to "simulate"
* non-unique index creation.
- *
*/
@Test
public void bulkLoadExample() throws Exception {
@@ -511,10 +506,10 @@
cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
cmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- int indexFileId = getIndexFileId();
+ FileReference file = getFileReference();
ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories);
- treeIndex.create(indexFileId);
- treeIndex.open(indexFileId);
+ treeIndex.create(file);
+ treeIndex.open(file);
// Load sorted records.
int ins = 100000;
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexMultiThreadTest.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexMultiThreadTest.java
index 0401038..7744faa 100644
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexMultiThreadTest.java
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexMultiThreadTest.java
@@ -26,44 +26,46 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
import edu.uci.ics.hyracks.storage.am.common.ITreeIndexTestWorkerFactory;
+import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
import edu.uci.ics.hyracks.storage.am.common.TestWorkloadConf;
import edu.uci.ics.hyracks.storage.am.common.TreeIndexMultiThreadTestDriver;
-import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.config.AccessMethodTestsConfig;
@SuppressWarnings("rawtypes")
-public abstract class OrderedIndexMultiThreadTest {
-
+public abstract class OrderedIndexMultiThreadTest {
+
protected final Logger LOGGER = Logger.getLogger(OrderedIndexMultiThreadTest.class.getName());
-
+
// Machine-specific number of threads to use for testing.
protected final int REGULAR_NUM_THREADS = Runtime.getRuntime().availableProcessors();
// Excessive number of threads for testing.
protected final int EXCESSIVE_NUM_THREADS = Runtime.getRuntime().availableProcessors() * 4;
protected final int NUM_OPERATIONS = AccessMethodTestsConfig.BTREE_MULTITHREAD_NUM_OPERATIONS;
-
- protected ArrayList<TestWorkloadConf> workloadConfs = getTestWorkloadConf();
-
- protected abstract void setUp() throws HyracksException;
-
- protected abstract void tearDown() throws HyracksDataException;
- protected abstract ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories) throws TreeIndexException;
-
- protected abstract int getFileId();
-
+ protected ArrayList<TestWorkloadConf> workloadConfs = getTestWorkloadConf();
+
+ protected abstract void setUp() throws HyracksException;
+
+ protected abstract void tearDown() throws HyracksDataException;
+
+ protected abstract ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories)
+ throws TreeIndexException;
+
protected abstract ITreeIndexTestWorkerFactory getWorkerFactory();
-
+
protected abstract ArrayList<TestWorkloadConf> getTestWorkloadConf();
-
+
protected abstract String getIndexTypeName();
-
+
+ protected abstract FileReference getFileReference();
+
protected static float[] getUniformOpProbs(TestOperation[] ops) {
float[] opProbs = new float[ops.length];
for (int i = 0; i < ops.length; i++) {
@@ -71,54 +73,59 @@
}
return opProbs;
}
-
- protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, int numThreads, TestWorkloadConf conf, String dataMsg) throws InterruptedException, TreeIndexException, HyracksException {
+
+ protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, int numThreads, TestWorkloadConf conf,
+ String dataMsg) throws InterruptedException, TreeIndexException, HyracksException {
setUp();
-
+
if (LOGGER.isLoggable(Level.INFO)) {
- String indexTypeName = getIndexTypeName();
- LOGGER.info(indexTypeName + " MultiThread Test:\nData: " + dataMsg + "; Threads: " + numThreads + "; Workload: " + conf.toString() + ".");
+ String indexTypeName = getIndexTypeName();
+ LOGGER.info(indexTypeName + " MultiThread Test:\nData: " + dataMsg + "; Threads: " + numThreads
+ + "; Workload: " + conf.toString() + ".");
}
-
+
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
- IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeys);
-
+ IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeys);
+
ITreeIndex index = createTreeIndex(typeTraits, cmpFactories);
ITreeIndexTestWorkerFactory workerFactory = getWorkerFactory();
-
+
// 4 batches per thread.
int batchSize = (NUM_OPERATIONS / numThreads) / 4;
-
- TreeIndexMultiThreadTestDriver driver = new TreeIndexMultiThreadTestDriver(index, workerFactory, fieldSerdes, conf.ops, conf.opProbs);
- driver.init(getFileId());
+
+ TreeIndexMultiThreadTestDriver driver = new TreeIndexMultiThreadTestDriver(index, workerFactory, fieldSerdes,
+ conf.ops, conf.opProbs);
+ driver.init(getFileReference());
long[] times = driver.run(numThreads, 1, NUM_OPERATIONS, batchSize);
driver.deinit();
-
+
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("BTree MultiThread Test Time: " + times[0] + "ms");
}
-
+
tearDown();
}
-
+
@Test
- public void oneIntKeyAndValue() throws InterruptedException, TreeIndexException, HyracksException {
- ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+ public void oneIntKeyAndValue() throws InterruptedException, TreeIndexException, HyracksException {
+ ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE };
int numKeys = 1;
String dataMsg = "One Int Key And Value";
-
+
for (TestWorkloadConf conf : workloadConfs) {
runTest(fieldSerdes, numKeys, REGULAR_NUM_THREADS, conf, dataMsg);
runTest(fieldSerdes, numKeys, EXCESSIVE_NUM_THREADS, conf, dataMsg);
}
}
-
+
@Test
- public void oneStringKeyAndValue() throws InterruptedException, TreeIndexException, HyracksException {
- ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE };
+ public void oneStringKeyAndValue() throws InterruptedException, TreeIndexException, HyracksException {
+ ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE };
int numKeys = 1;
String dataMsg = "One String Key And Value";
-
+
for (TestWorkloadConf conf : workloadConfs) {
runTest(fieldSerdes, numKeys, REGULAR_NUM_THREADS, conf, dataMsg);
runTest(fieldSerdes, numKeys, EXCESSIVE_NUM_THREADS, conf, dataMsg);
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TreeIndexMultiThreadTestDriver.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TreeIndexMultiThreadTestDriver.java
index 8c1d06f..12596f3 100644
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TreeIndexMultiThreadTestDriver.java
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TreeIndexMultiThreadTestDriver.java
@@ -17,6 +17,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
@@ -27,25 +28,26 @@
private static final int RANDOM_SEED = 50;
// Means no additional payload. Only the specified fields.
private static final int PAYLOAD_SIZE = 0;
- private final TestOperationSelector opSelector;
+ private final TestOperationSelector opSelector;
private final ISerializerDeserializer[] fieldSerdes;
private final ITreeIndex index;
private final ITreeIndexTestWorkerFactory workerFactory;
-
+
public TreeIndexMultiThreadTestDriver(ITreeIndex index, ITreeIndexTestWorkerFactory workerFactory,
ISerializerDeserializer[] fieldSerdes, TestOperation[] ops, float[] opProbs) {
this.index = index;
this.workerFactory = workerFactory;
this.fieldSerdes = fieldSerdes;
this.opSelector = new TestOperationSelector(ops, opProbs);
- }
-
- public void init(int fileId) throws HyracksDataException {
- index.create(fileId);
- index.open(fileId);
}
-
- public long[] run(int numThreads, int numRepeats, int numOps, int batchSize) throws InterruptedException, TreeIndexException {
+
+ public void init(FileReference file) throws HyracksDataException {
+ index.create(file);
+ index.open(file);
+ }
+
+ public long[] run(int numThreads, int numRepeats, int numOps, int batchSize) throws InterruptedException,
+ TreeIndexException {
int numBatches = numOps / batchSize;
int threadNumBatches = numBatches / numThreads;
if (threadNumBatches <= 0) {
@@ -59,7 +61,7 @@
while (dataGen.tupleBatchQueue.remainingCapacity() != 0 && dataGen.tupleBatchQueue.size() != numBatches) {
Thread.sleep(10);
}
-
+
// Start worker threads.
AbstractTreeIndexTestWorker[] workers = new AbstractTreeIndexTestWorker[numThreads];
long start = System.currentTimeMillis();
@@ -68,7 +70,7 @@
workers[j].start();
}
// Join worker threads.
- for (int j = 0; j < numThreads; j++) {
+ for (int j = 0; j < numThreads; j++) {
workers[j].join();
}
long end = System.currentTimeMillis();
@@ -76,13 +78,14 @@
}
return times;
}
-
+
public void deinit() throws HyracksDataException {
- index.close();
+ index.close();
}
-
+
// To allow subclasses to override the data gen params.
public DataGenThread createDatagenThread(int numThreads, int numBatches, int batchSize) {
- return new DataGenThread(numThreads, numBatches, batchSize, fieldSerdes, PAYLOAD_SIZE, RANDOM_SEED, 2*numThreads, false);
+ return new DataGenThread(numThreads, numBatches, batchSize, fieldSerdes, PAYLOAD_SIZE, RANDOM_SEED,
+ 2 * numThreads, false);
}
}
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TreeIndexTestUtils.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TreeIndexTestUtils.java
index 47673c9..000f7d0 100644
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TreeIndexTestUtils.java
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TreeIndexTestUtils.java
@@ -243,7 +243,6 @@
ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
// Perform bulk load.
- // IIndexBulkLoadContext bulkLoadCtx = ctx.getIndex().beginBulkLoad(0.7f);
IIndexBulkLoader bulkLoader = ctx.getIndex().createBulkLoader(0.7f);
int c = 1;
for (CheckTuple checkTuple : checkTuples) {
@@ -253,11 +252,9 @@
}
}
createTupleFromCheckTuple(checkTuple, tupleBuilder, tuple, ctx.getFieldSerdes());
- // ctx.getIndex().bulkLoadAddTuple(tuple, bulkLoadCtx);
bulkLoader.add(tuple);
c++;
}
- // ctx.getIndex().endBulkLoad(bulkLoadCtx);
bulkLoader.end();
}
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java
index 73ada0e..994db2a 100644
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/config/AccessMethodTestsConfig.java
@@ -19,52 +19,52 @@
* Global parameters used for executing access method JUnit tests.
*/
public class AccessMethodTestsConfig {
- // Test params for RTree, LSMRTree and LSMRTreeWithAntiMatterTuples.
- public static final int RTREE_NUM_TUPLES_TO_INSERT = 10000;
- public static final int RTREE_NUM_INSERT_ROUNDS = 2;
- public static final int RTREE_NUM_DELETE_ROUNDS = 2;
- public static final int RTREE_MULTITHREAD_NUM_OPERATIONS = 10000;
- // Test params for LSMRTree and LSMRTreeWithAntiMatterTuples.
- public static final int LSM_RTREE_BULKLOAD_ROUNDS = 5;
- public static final int LSM_RTREE_MAX_TREES_TO_MERGE = 3;
+ // Test params for RTree, LSMRTree and LSMRTreeWithAntiMatterTuples.
+ public static final int RTREE_NUM_TUPLES_TO_INSERT = 10000;
+ public static final int RTREE_NUM_INSERT_ROUNDS = 2;
+ public static final int RTREE_NUM_DELETE_ROUNDS = 2;
+ public static final int RTREE_MULTITHREAD_NUM_OPERATIONS = 10000;
+ // Test params for LSMRTree and LSMRTreeWithAntiMatterTuples.
+ public static final int LSM_RTREE_BULKLOAD_ROUNDS = 5;
+ public static final int LSM_RTREE_MAX_TREES_TO_MERGE = 3;
- // Test params for BTree, LSMBTree.
- public static final int BTREE_NUM_TUPLES_TO_INSERT = 10000;
- public static final int BTREE_NUM_INSERT_ROUNDS = 3;
- public static final int BTREE_NUM_DELETE_ROUNDS = 3;
- public static final int BTREE_NUM_UPDATE_ROUNDS = 3;
- public static final int BTREE_MULTITHREAD_NUM_OPERATIONS = 10000;
- // Test params for LSMBTree only.
- public static final int LSM_BTREE_BULKLOAD_ROUNDS = 5;
- public static final int LSM_BTREE_MAX_TREES_TO_MERGE = 10;
+ // Test params for BTree, LSMBTree.
+ public static final int BTREE_NUM_TUPLES_TO_INSERT = 10000;
+ public static final int BTREE_NUM_INSERT_ROUNDS = 3;
+ public static final int BTREE_NUM_DELETE_ROUNDS = 3;
+ public static final int BTREE_NUM_UPDATE_ROUNDS = 3;
+ public static final int BTREE_MULTITHREAD_NUM_OPERATIONS = 10000;
+ // Test params for LSMBTree only.
+ public static final int LSM_BTREE_BULKLOAD_ROUNDS = 5;
+ public static final int LSM_BTREE_MAX_TREES_TO_MERGE = 10;
- // Mem configuration for RTree.
- public static final int RTREE_PAGE_SIZE = 512;
- public static final int RTREE_NUM_PAGES = 1000;
- public static final int RTREE_MAX_OPEN_FILES = 10;
- public static final int RTREE_HYRACKS_FRAME_SIZE = 128;
+ // Mem configuration for RTree.
+ public static final int RTREE_PAGE_SIZE = 512;
+ public static final int RTREE_NUM_PAGES = 1000;
+ public static final int RTREE_MAX_OPEN_FILES = 10;
+ public static final int RTREE_HYRACKS_FRAME_SIZE = 128;
- // Mem configuration for LSMRTree and LSMRTreeWithAntiMatterTuples.
- public static final int LSM_RTREE_DISK_PAGE_SIZE = 256;
- public static final int LSM_RTREE_DISK_NUM_PAGES = 1000;
- public static final int LSM_RTREE_DISK_MAX_OPEN_FILES = 2000;
- public static final int LSM_RTREE_MEM_PAGE_SIZE = 256;
- public static final int LSM_RTREE_MEM_NUM_PAGES = 1000;
- public static final int LSM_RTREE_HYRACKS_FRAME_SIZE = 128;
+ // Mem configuration for LSMRTree and LSMRTreeWithAntiMatterTuples.
+ public static final int LSM_RTREE_DISK_PAGE_SIZE = 256;
+ public static final int LSM_RTREE_DISK_NUM_PAGES = 1000;
+ public static final int LSM_RTREE_DISK_MAX_OPEN_FILES = 2000;
+ public static final int LSM_RTREE_MEM_PAGE_SIZE = 256;
+ public static final int LSM_RTREE_MEM_NUM_PAGES = 1000;
+ public static final int LSM_RTREE_HYRACKS_FRAME_SIZE = 128;
- // Mem configuration for BTree.
- public static final int BTREE_PAGE_SIZE = 256;
- public static final int BTREE_NUM_PAGES = 100;
- public static final int BTREE_MAX_OPEN_FILES = 10;
- public static final int BTREE_HYRACKS_FRAME_SIZE = 128;
+ // Mem configuration for BTree.
+ public static final int BTREE_PAGE_SIZE = 256;
+ public static final int BTREE_NUM_PAGES = 100;
+ public static final int BTREE_MAX_OPEN_FILES = 10;
+ public static final int BTREE_HYRACKS_FRAME_SIZE = 128;
- // Mem configuration for LSMBTree.
- public static final int LSM_BTREE_DISK_PAGE_SIZE = 256;
- public static final int LSM_BTREE_DISK_NUM_PAGES = 1000;
- public static final int LSM_BTREE_DISK_MAX_OPEN_FILES = 200;
- public static final int LSM_BTREE_MEM_PAGE_SIZE = 256;
- public static final int LSM_BTREE_MEM_NUM_PAGES = 100;
- public static final int LSM_BTREE_HYRACKS_FRAME_SIZE = 128;
+ // Mem configuration for LSMBTree.
+ public static final int LSM_BTREE_DISK_PAGE_SIZE = 256;
+ public static final int LSM_BTREE_DISK_NUM_PAGES = 1000;
+ public static final int LSM_BTREE_DISK_MAX_OPEN_FILES = 200;
+ public static final int LSM_BTREE_MEM_PAGE_SIZE = 256;
+ public static final int LSM_BTREE_MEM_NUM_PAGES = 100;
+ public static final int LSM_BTREE_HYRACKS_FRAME_SIZE = 128;
}
/* ORIGINAL TEST PARAMETERS: DO NOT EDIT!
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java
index 0d86b78..f5753f1 100644
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java
@@ -24,6 +24,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
@@ -34,10 +35,10 @@
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
@@ -57,11 +58,10 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType)
throws TreeIndexException;
- protected abstract int getIndexFileId();
+ protected abstract FileReference getFileReference();
/**
* Two Dimensions Example.
- *
* Create an RTree index of two dimensions, where they keys are of type
* integer, and the payload is two integer values. Fill index with random
* values using insertions (not bulk load). Perform scans and range search.
@@ -109,11 +109,11 @@
IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
rtreeCmpFactories.length, IntegerPointable.FACTORY);
- int indexFileId = getIndexFileId();
+ FileReference file = getFileReference();
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories,
valueProviderFactories, RTreePolicyType.RTREE);
- treeIndex.create(indexFileId);
- treeIndex.open(indexFileId);
+ treeIndex.create(file);
+ treeIndex.open(file);
long start = System.currentTimeMillis();
if (LOGGER.isLoggable(Level.INFO)) {
@@ -166,7 +166,6 @@
/**
* Two Dimensions Example.
- *
* Create an RTree index of three dimensions, where they keys are of type
* double, and the payload is one double value. Fill index with random
* values using insertions (not bulk load). Perform scans and range search.
@@ -218,11 +217,11 @@
IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
rtreeCmpFactories.length, DoublePointable.FACTORY);
- int indexFileId = getIndexFileId();
+ FileReference file = getFileReference();
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories,
valueProviderFactories, RTreePolicyType.RTREE);
- treeIndex.create(indexFileId);
- treeIndex.open(indexFileId);
+ treeIndex.create(file);
+ treeIndex.open(file);
long start = System.currentTimeMillis();
if (LOGGER.isLoggable(Level.INFO)) {
@@ -277,7 +276,6 @@
/**
* Deletion Example.
- *
* Create an RTree index of two dimensions, where they keys are of type
* integer, and the payload is one integer value. Fill index with random
* values using insertions, then delete entries one-by-one. Repeat procedure
@@ -319,11 +317,11 @@
IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
rtreeCmpFactories.length, IntegerPointable.FACTORY);
- int indexFileId = getIndexFileId();
+ FileReference file = getFileReference();
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories,
valueProviderFactories, RTreePolicyType.RTREE);
- treeIndex.create(indexFileId);
- treeIndex.open(indexFileId);
+ treeIndex.create(file);
+ treeIndex.open(file);
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
@@ -409,9 +407,7 @@
/**
* Bulk load example.
- *
* Load a tree with 10,000 tuples.
- *
*/
@Test
public void bulkLoadExample() throws Exception {
@@ -453,11 +449,11 @@
IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
rtreeCmpFactories.length, IntegerPointable.FACTORY);
- int indexFileId = getIndexFileId();
+ FileReference file = getFileReference();
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories,
valueProviderFactories, RTreePolicyType.RTREE);
- treeIndex.create(indexFileId);
- treeIndex.open(indexFileId);
+ treeIndex.create(file);
+ treeIndex.open(file);
// Load records.
int numInserts = 10000;
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeMultiThreadTest.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeMultiThreadTest.java
index 9aafc48..e2cff56 100644
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeMultiThreadTest.java
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeMultiThreadTest.java
@@ -26,6 +26,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
@@ -64,14 +65,14 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType)
throws TreeIndexException;
- protected abstract int getFileId();
-
protected abstract ITreeIndexTestWorkerFactory getWorkerFactory();
protected abstract ArrayList<TestWorkloadConf> getTestWorkloadConf();
protected abstract String getIndexTypeName();
+ protected abstract FileReference getFileReference();
+
protected static float[] getUniformOpProbs(TestOperation[] ops) {
float[] opProbs = new float[ops.length];
for (int i = 0; i < ops.length; i++) {
@@ -106,7 +107,7 @@
TreeIndexMultiThreadTestDriver driver = new TreeIndexMultiThreadTestDriver(index, workerFactory, fieldSerdes,
conf.ops, conf.opProbs);
- driver.init(getFileId());
+ driver.init(getFileReference());
long[] times = driver.run(numThreads, 1, NUM_OPERATIONS, batchSize);
driver.deinit();
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeBulkLoadTest.java
index 11c47c7..cb76cc8 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeBulkLoadTest.java
@@ -22,8 +22,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexBulkLoadTest;
-import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexTestContext;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
@@ -50,8 +48,8 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
- return BTreeTestContext.create(harness.getBufferCache(), harness.getBTreeFileId(), fieldSerdes, numKeys,
- leafType);
+ return BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, numKeys, leafType);
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeDeleteTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeDeleteTest.java
index 0205540..596b136 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeDeleteTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeDeleteTest.java
@@ -22,8 +22,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexDeleteTest;
-import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexTestContext;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
@@ -50,8 +48,8 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
- return BTreeTestContext.create(harness.getBufferCache(), harness.getBTreeFileId(), fieldSerdes, numKeys,
- leafType);
+ return BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, numKeys, leafType);
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java
index a89239c..2d1b46c 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java
@@ -21,7 +21,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexExamplesTest;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
@@ -40,12 +40,15 @@
public void tearDown() throws HyracksDataException {
harness.tearDown();
}
-
- protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories) throws TreeIndexException {
- return BTreeUtils.createBTree(harness.getBufferCache(), typeTraits, cmpFactories, BTreeLeafFrameType.REGULAR_NSM);
+
+ protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories)
+ throws TreeIndexException {
+ return BTreeUtils.createBTree(harness.getBufferCache(), harness.getFileMapProvider(), typeTraits, cmpFactories,
+ BTreeLeafFrameType.REGULAR_NSM);
}
-
- protected int getIndexFileId() {
- return harness.getBTreeFileId();
+
+ @Override
+ protected FileReference getFileReference() {
+ return harness.getFileReference();
}
}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeInsertTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeInsertTest.java
index 0b6cf4d..14ae6f7 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeInsertTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeInsertTest.java
@@ -22,8 +22,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexInsertTest;
-import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexTestContext;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
@@ -31,12 +29,10 @@
/**
* Tests the BTree insert operation with strings and integer fields using
* various numbers of key and payload fields.
- *
* Each tests first fills a BTree with randomly generated tuples. We compare the
* following operations against expected results: 1. Point searches for all
* tuples. 2. Ordered scan. 3. Disk-order scan. 4. Range search (and prefix
* search for composite keys).
- *
*/
@SuppressWarnings("rawtypes")
public class BTreeInsertTest extends OrderedIndexInsertTest {
@@ -60,8 +56,8 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
- return BTreeTestContext.create(harness.getBufferCache(), harness.getBTreeFileId(), fieldSerdes, numKeys,
- leafType);
+ return BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, numKeys, leafType);
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeSearchCursorTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeSearchCursorTest.java
index e283f52..55df7b8 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeSearchCursorTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeSearchCursorTest.java
@@ -85,8 +85,7 @@
}
IBufferCache bufferCache = harness.getBufferCache();
- int btreeFileId = harness.getBTreeFileId();
-
+
// declare keys
int keyFieldCount = 1;
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
@@ -100,14 +99,16 @@
IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
+ BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory,
+ leafFrameFactory, cmpFactories, fieldCount);
+ btree.create(harness.getFileReference());
+ btree.open(harness.getFileReference());
ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
- ITreeIndexAccessor indexAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ ITreeIndexAccessor indexAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
// generate keys
int numKeys = 50;
@@ -157,8 +158,7 @@
}
IBufferCache bufferCache = harness.getBufferCache();
- int btreeFileId = harness.getBTreeFileId();
-
+
// declare keys
int keyFieldCount = 2;
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
@@ -173,14 +173,16 @@
IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
+ BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory,
+ leafFrameFactory, cmpFactories, fieldCount);
+ btree.create(harness.getFileReference());
+ btree.open(harness.getFileReference());
ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
- ITreeIndexAccessor indexAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ ITreeIndexAccessor indexAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
// generate keys
int numKeys = 50;
@@ -227,8 +229,7 @@
}
IBufferCache bufferCache = harness.getBufferCache();
- int btreeFileId = harness.getBTreeFileId();
-
+
// declare keys
int keyFieldCount = 2;
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
@@ -243,14 +244,16 @@
IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
+ BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory,
+ leafFrameFactory, cmpFactories, fieldCount);
+ btree.create(harness.getFileReference());
+ btree.open(harness.getFileReference());
ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
- ITreeIndexAccessor indexAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ ITreeIndexAccessor indexAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
// generate keys
int numKeys = 50;
@@ -290,8 +293,8 @@
btree.close();
}
- public RangePredicate createRangePredicate(int lk, int hk, boolean lowKeyInclusive,
- boolean highKeyInclusive) throws HyracksDataException {
+ public RangePredicate createRangePredicate(int lk, int hk, boolean lowKeyInclusive, boolean highKeyInclusive)
+ throws HyracksDataException {
// create tuplereferences for search keys
ITupleReference lowKey = TupleUtils.createIntegerTuple(lk);
@@ -301,8 +304,8 @@
searchCmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
MultiComparator searchCmp = new MultiComparator(searchCmps);
- RangePredicate rangePred = new RangePredicate(lowKey, highKey, lowKeyInclusive, highKeyInclusive,
- searchCmp, searchCmp);
+ RangePredicate rangePred = new RangePredicate(lowKey, highKey, lowKeyInclusive, highKeyInclusive, searchCmp,
+ searchCmp);
return rangePred;
}
@@ -345,9 +348,9 @@
int highKey = j;
ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor(leafFrame, false);
- RangePredicate rangePred = createRangePredicate(lowKey, highKey, lowKeyInclusive,
- highKeyInclusive);
- ITreeIndexAccessor indexAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ RangePredicate rangePred = createRangePredicate(lowKey, highKey, lowKeyInclusive, highKeyInclusive);
+ ITreeIndexAccessor indexAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
indexAccessor.search(rangeCursor, rangePred);
try {
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java
index 130302a..bb9fb5f 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java
@@ -1,7 +1,6 @@
package edu.uci.ics.hyracks.storage.am.btree;
import java.io.DataOutput;
-import java.io.File;
import java.nio.ByteBuffer;
import java.util.Random;
import java.util.logging.Level;
@@ -14,7 +13,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
@@ -58,12 +56,8 @@
public void test01() throws Exception {
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(harness.getFileName()));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
+ IBufferCache bufferCache = harness.getBufferCache();
+ IFileMapProvider fmp = harness.getFileMapProvider();
// declare fields
int fieldCount = 2;
@@ -87,10 +81,10 @@
IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
- fieldCount);
- btree.create(fileId);
- btree.open(fileId);
+ BTree btree = new BTree(bufferCache, fmp, freePageManager, interiorFrameFactory, leafFrameFactory,
+ cmpFactories, fieldCount);
+ btree.create(harness.getFileReference());
+ btree.open(harness.getFileReference());
Random rnd = new Random();
rnd.setSeed(50);
@@ -147,6 +141,7 @@
}
}
+ int fileId = fmp.lookupFileId(harness.getFileReference());
TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, freePageManager, fileId,
btree.getRootPageId());
TreeIndexStats stats = statsGatherer.gatherStats(leafFrame, interiorFrame, metaFrame);
@@ -159,7 +154,6 @@
bufferCacheWarmup.warmup(leafFrame, metaFrame, new int[] { 1, 2 }, new int[] { 2, 5 });
btree.close();
- bufferCache.closeFile(fileId);
bufferCache.close();
}
}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateSearchTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
index 15b72f8..998a7b5 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
@@ -41,7 +41,6 @@
@Test
public void test01() throws Exception {
IBufferCache bufferCache = harness.getBufferCache();
- int btreeFileId = harness.getBTreeFileId();
// declare fields
int fieldCount = 2;
@@ -65,10 +64,10 @@
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
- fieldCount);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
+ BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory,
+ leafFrameFactory, cmpFactories, fieldCount);
+ btree.create(harness.getFileReference());
+ btree.open(harness.getFileReference());
Random rnd = new Random();
rnd.setSeed(50);
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateTest.java
index c3b56d5..5461cbd 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateTest.java
@@ -22,8 +22,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexTestContext;
-import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexUpdateTest;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
@@ -50,8 +48,8 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
- return BTreeTestContext.create(harness.getBufferCache(), harness.getBTreeFileId(), fieldSerdes, numKeys,
- leafType);
+ return BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, numKeys, leafType);
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpsertTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpsertTest.java
index 6e14607..4141832 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpsertTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpsertTest.java
@@ -29,12 +29,10 @@
/**
* Tests the BTree insert operation with strings and integer fields using
* various numbers of key and payload fields.
- *
* Each tests first fills a BTree with randomly generated tuples. We compare the
* following operations against expected results: 1. Point searches for all
* tuples. 2. Ordered scan. 3. Disk-order scan. 4. Range search (and prefix
* search for composite keys).
- *
*/
@SuppressWarnings("rawtypes")
public class BTreeUpsertTest extends OrderedIndexUpsertTest {
@@ -58,8 +56,8 @@
@Override
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
- return BTreeTestContext.create(harness.getBufferCache(), harness.getBTreeFileId(), fieldSerdes, numKeys,
- leafType);
+ return BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, numKeys, leafType);
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java
index d61d16a..9f7daf5 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java
@@ -48,6 +48,7 @@
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@SuppressWarnings("rawtypes")
public class FieldPrefixNSMTest extends AbstractBTreeTest {
@@ -57,10 +58,10 @@
private static final int MAX_OPEN_FILES = 10;
private static final int HYRACKS_FRAME_SIZE = 128;
- public FieldPrefixNSMTest() {
+ public FieldPrefixNSMTest() {
super(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES, HYRACKS_FRAME_SIZE);
}
-
+
private ITupleReference createTuple(IHyracksTaskContext ctx, int f0, int f1, int f2, boolean print)
throws HyracksDataException {
if (print) {
@@ -73,7 +74,7 @@
FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
ArrayTupleBuilder tb = new ArrayTupleBuilder(3);
DataOutput dos = tb.getDataOutput();
-
+
ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
@@ -98,8 +99,8 @@
}
@Test
- public void test01() throws Exception {
-
+ public void test01() throws Exception {
+
// declare fields
int fieldCount = 3;
ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
@@ -123,7 +124,10 @@
rnd.setSeed(50);
IBufferCache bufferCache = harness.getBufferCache();
- int btreeFileId = harness.getBTreeFileId();
+ IFileMapProvider fileMapProvider = harness.getFileMapProvider();
+ bufferCache.createFile(harness.getFileReference());
+ int btreeFileId = fileMapProvider.lookupFileId(harness.getFileReference());
+ bufferCache.openFile(btreeFileId);
IHyracksTaskContext ctx = harness.getHyracksTaskContext();
ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(btreeFileId, 0), false);
try {
@@ -220,6 +224,8 @@
} finally {
bufferCache.unpin(page);
+ bufferCache.closeFile(btreeFileId);
+ bufferCache.close();
}
}
}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java
index 9ec64b5..ef4e2a8 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java
@@ -27,9 +27,10 @@
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
import edu.uci.ics.hyracks.storage.common.sync.LatchType;
-public class StorageManagerTest extends AbstractBTreeTest {
+public class StorageManagerTest extends AbstractBTreeTest {
public class PinnedLatchedPage {
public final ICachedPage page;
public final LatchType latch;
@@ -43,7 +44,10 @@
}
public enum FileAccessType {
- FTA_READONLY, FTA_WRITEONLY, FTA_MIXED, FTA_UNLATCHED
+ FTA_READONLY,
+ FTA_WRITEONLY,
+ FTA_MIXED,
+ FTA_UNLATCHED
}
public class FileAccessWorker implements Runnable {
@@ -249,11 +253,17 @@
}
@Test
- public void oneThreadOneFileTest() throws Exception {
- Thread worker = new Thread(new FileAccessWorker(0,
- harness.getBufferCache(), FileAccessType.FTA_UNLATCHED,
- harness.getBTreeFileId(), 10, 10, 100, 10, 0));
+ public void oneThreadOneFileTest() throws Exception {
+ IBufferCache bufferCache = harness.getBufferCache();
+ IFileMapProvider fmp = harness.getFileMapProvider();
+ bufferCache.createFile(harness.getFileReference());
+ int btreeFileId = harness.getFileMapProvider().lookupFileId(harness.getFileReference());
+ bufferCache.openFile(btreeFileId);
+ Thread worker = new Thread(new FileAccessWorker(0, harness.getBufferCache(), FileAccessType.FTA_UNLATCHED,
+ btreeFileId, 10, 10, 100, 10, 0));
worker.start();
worker.join();
+ bufferCache.closeFile(btreeFileId);
+ bufferCache.close();
}
}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java
index 26919b0..9fd7c16 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java
@@ -20,22 +20,23 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexMultiThreadTest;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
import edu.uci.ics.hyracks.storage.am.common.ITreeIndexTestWorkerFactory;
-import edu.uci.ics.hyracks.storage.am.common.TestWorkloadConf;
import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
+import edu.uci.ics.hyracks.storage.am.common.TestWorkloadConf;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
public class BTreeMultiThreadTest extends OrderedIndexMultiThreadTest {
private BTreeTestHarness harness = new BTreeTestHarness();
-
+
private BTreeTestWorkerFactory workerFactory = new BTreeTestWorkerFactory();
-
+
@Override
protected void setUp() throws HyracksDataException {
harness.setUp();
@@ -47,8 +48,10 @@
}
@Override
- protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories) throws TreeIndexException {
- return BTreeUtils.createBTree(harness.getBufferCache(), typeTraits, cmpFactories, BTreeLeafFrameType.REGULAR_NSM);
+ protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories)
+ throws TreeIndexException {
+ return BTreeUtils.createBTree(harness.getBufferCache(), harness.getFileMapProvider(), typeTraits, cmpFactories,
+ BTreeLeafFrameType.REGULAR_NSM);
}
@Override
@@ -59,29 +62,34 @@
@Override
protected ArrayList<TestWorkloadConf> getTestWorkloadConf() {
ArrayList<TestWorkloadConf> workloadConfs = new ArrayList<TestWorkloadConf>();
-
+
// Insert only workload.
TestOperation[] insertOnlyOps = new TestOperation[] { TestOperation.INSERT };
workloadConfs.add(new TestWorkloadConf(insertOnlyOps, getUniformOpProbs(insertOnlyOps)));
-
+
// Inserts mixed with point searches and scans.
- TestOperation[] insertSearchOnlyOps = new TestOperation[] { TestOperation.INSERT, TestOperation.POINT_SEARCH, TestOperation.SCAN, TestOperation.DISKORDER_SCAN };
+ TestOperation[] insertSearchOnlyOps = new TestOperation[] { TestOperation.INSERT, TestOperation.POINT_SEARCH,
+ TestOperation.SCAN, TestOperation.DISKORDER_SCAN };
workloadConfs.add(new TestWorkloadConf(insertSearchOnlyOps, getUniformOpProbs(insertSearchOnlyOps)));
-
+
// Inserts, updates, deletes, and upserts.
- TestOperation[] insertDeleteUpdateUpsertOps = new TestOperation[] { TestOperation.INSERT, TestOperation.DELETE, TestOperation.UPDATE, TestOperation.UPSERT };
- workloadConfs.add(new TestWorkloadConf(insertDeleteUpdateUpsertOps, getUniformOpProbs(insertDeleteUpdateUpsertOps)));
-
+ TestOperation[] insertDeleteUpdateUpsertOps = new TestOperation[] { TestOperation.INSERT, TestOperation.DELETE,
+ TestOperation.UPDATE, TestOperation.UPSERT };
+ workloadConfs.add(new TestWorkloadConf(insertDeleteUpdateUpsertOps,
+ getUniformOpProbs(insertDeleteUpdateUpsertOps)));
+
// All operations mixed.
- TestOperation[] allOps = new TestOperation[] { TestOperation.INSERT, TestOperation.DELETE, TestOperation.UPDATE, TestOperation.UPSERT, TestOperation.POINT_SEARCH, TestOperation.SCAN, TestOperation.DISKORDER_SCAN };
+ TestOperation[] allOps = new TestOperation[] { TestOperation.INSERT, TestOperation.DELETE,
+ TestOperation.UPDATE, TestOperation.UPSERT, TestOperation.POINT_SEARCH, TestOperation.SCAN,
+ TestOperation.DISKORDER_SCAN };
workloadConfs.add(new TestWorkloadConf(allOps, getUniformOpProbs(allOps)));
-
+
return workloadConfs;
}
-
+
@Override
- protected int getFileId() {
- return harness.getBTreeFileId();
+ protected FileReference getFileReference() {
+ return harness.getFileReference();
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/AbstractBTreeTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/AbstractBTreeTest.java
index f4eca1b..ef9a456 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/AbstractBTreeTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/AbstractBTreeTest.java
@@ -23,24 +23,25 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
public abstract class AbstractBTreeTest {
- protected final Logger LOGGER = Logger.getLogger(BTreeTestHarness.class.getName());
- protected final BTreeTestHarness harness;
-
- public AbstractBTreeTest() {
- harness = new BTreeTestHarness();
+ protected final Logger LOGGER = Logger.getLogger(BTreeTestHarness.class.getName());
+
+ protected final BTreeTestHarness harness;
+
+ public AbstractBTreeTest() {
+ harness = new BTreeTestHarness();
}
-
+
public AbstractBTreeTest(int pageSize, int numPages, int maxOpenFiles, int hyracksFrameSize) {
- harness = new BTreeTestHarness(pageSize, numPages, maxOpenFiles, hyracksFrameSize);
+ harness = new BTreeTestHarness(pageSize, numPages, maxOpenFiles, hyracksFrameSize);
}
-
- @Before
- public void setUp() throws HyracksDataException {
- harness.setUp();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
}
-
- @After
+
+ @After
public void tearDown() throws HyracksDataException {
- harness.tearDown();
+ harness.tearDown();
}
}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestContext.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestContext.java
index 8442131..7950084 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestContext.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestContext.java
@@ -18,16 +18,18 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexTestContext;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@SuppressWarnings("rawtypes")
public class BTreeTestContext extends OrderedIndexTestContext {
-
+
public BTreeTestContext(ISerializerDeserializer[] fieldSerdes, ITreeIndex treeIndex) {
super(fieldSerdes, treeIndex);
}
@@ -37,19 +39,21 @@
BTree btree = (BTree) treeIndex;
return btree.getComparatorFactories().length;
}
-
+
@Override
public IBinaryComparatorFactory[] getComparatorFactories() {
BTree btree = (BTree) treeIndex;
return btree.getComparatorFactories();
}
-
- public static BTreeTestContext create(IBufferCache bufferCache, int btreeFileId, ISerializerDeserializer[] fieldSerdes, int numKeyFields, BTreeLeafFrameType leafType) throws Exception {
+
+ public static BTreeTestContext create(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
+ FileReference file, ISerializerDeserializer[] fieldSerdes, int numKeyFields, BTreeLeafFrameType leafType)
+ throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
- BTree btree = BTreeUtils.createBTree(bufferCache, typeTraits, cmpFactories, leafType);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
+ BTree btree = BTreeUtils.createBTree(bufferCache, fileMapProvider, typeTraits, cmpFactories, leafType);
+ btree.create(file);
+ btree.open(file);
BTreeTestContext testCtx = new BTreeTestContext(fieldSerdes, btree);
return testCtx;
}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestHarness.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestHarness.java
index 80d2be1..f1e3e6c 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestHarness.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestHarness.java
@@ -37,25 +37,36 @@
private static final long RANDOM_SEED = 50;
protected final int pageSize;
+
protected final int numPages;
+
protected final int maxOpenFiles;
+
protected final int hyracksFrameSize;
protected IHyracksTaskContext ctx;
+
protected IBufferCache bufferCache;
- protected int btreeFileId;
+
+ protected IFileMapProvider fileMapProvider;
+
+ protected FileReference file;
protected final Random rnd = new Random();
+
protected final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+
protected final String tmpDir = System.getProperty("java.io.tmpdir");
+
protected final String sep = System.getProperty("file.separator");
+
protected String fileName;
public BTreeTestHarness() {
- this.pageSize = AccessMethodTestsConfig.BTREE_PAGE_SIZE;
- this.numPages = AccessMethodTestsConfig.BTREE_NUM_PAGES;
- this.maxOpenFiles = AccessMethodTestsConfig.BTREE_MAX_OPEN_FILES;
- this.hyracksFrameSize = AccessMethodTestsConfig.BTREE_HYRACKS_FRAME_SIZE;
+ this.pageSize = AccessMethodTestsConfig.BTREE_PAGE_SIZE;
+ this.numPages = AccessMethodTestsConfig.BTREE_NUM_PAGES;
+ this.maxOpenFiles = AccessMethodTestsConfig.BTREE_MAX_OPEN_FILES;
+ this.hyracksFrameSize = AccessMethodTestsConfig.BTREE_HYRACKS_FRAME_SIZE;
}
public BTreeTestHarness(int pageSize, int numPages, int maxOpenFiles, int hyracksFrameSize) {
@@ -70,19 +81,14 @@
ctx = TestUtils.create(getHyracksFrameSize());
TestStorageManagerComponentHolder.init(pageSize, numPages, maxOpenFiles);
bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- btreeFileId = fmp.lookupFileId(file);
- bufferCache.openFile(btreeFileId);
+ fileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ file = new FileReference(new File(fileName));
rnd.setSeed(RANDOM_SEED);
}
public void tearDown() throws HyracksDataException {
- bufferCache.closeFile(btreeFileId);
bufferCache.close();
- File f = new File(fileName);
- f.deleteOnExit();
+ file.delete();
}
public IHyracksTaskContext getHyracksTaskContext() {
@@ -93,8 +99,12 @@
return bufferCache;
}
- public int getBTreeFileId() {
- return btreeFileId;
+ public IFileMapProvider getFileMapProvider() {
+ return fileMapProvider;
+ }
+
+ public FileReference getFileReference() {
+ return file;
}
public String getFileName() {
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java
index ec7c76d..52b0925 100644
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java
@@ -137,7 +137,7 @@
protected IIndexCursor resultCursor;
protected abstract void setTokenizer();
-
+
/**
* Initialize members, generate data, and bulk load the inverted index.
*/
@@ -149,33 +149,27 @@
// --- BTREE ---
- bufferCache.createFile(btreeFile);
- btreeFileId = fmp.lookupFileId(btreeFile);
- bufferCache.openFile(btreeFileId);
-
btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
- btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory,
- btreeCmpFactories, btreeTypeTraits.length);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
+ btree = new BTree(bufferCache, fmp, freePageManager, interiorFrameFactory, leafFrameFactory, btreeCmpFactories,
+ btreeTypeTraits.length);
+ btree.create(btreeFile);
+ btree.open(btreeFile);
// --- INVERTED INDEX ---
setTokenizer();
-
- bufferCache.createFile(invListsFile);
- invListsFileId = fmp.lookupFileId(invListsFile);
- bufferCache.openFile(invListsFileId);
invListTypeTraits[0] = IntegerPointable.TYPE_TRAITS;
invListCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
- invIndex = new InvertedIndex(bufferCache, btree, invListTypeTraits, invListCmpFactories, invListBuilder, tokenizer);
- invIndex.open(invListsFileId);
+ invIndex = new InvertedIndex(bufferCache, btree, invListTypeTraits, invListCmpFactories, invListBuilder,
+ tokenizer, fmp);
+ invIndex.create(invListsFile);
+ invIndex.open(invListsFile);
rnd.setSeed(50);
}
@@ -185,8 +179,6 @@
AbstractInvIndexTest.tearDown();
btree.close();
invIndex.close();
- bufferCache.closeFile(btreeFileId);
- bufferCache.closeFile(invListsFileId);
bufferCache.close();
}
}
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java
index b81a814..731ebb1 100644
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java
@@ -96,14 +96,8 @@
// create file refs
FileReference btreeFile = new FileReference(new File(btreeFileName));
- bufferCache.createFile(btreeFile);
- int btreeFileId = fmp.lookupFileId(btreeFile);
- bufferCache.openFile(btreeFileId);
FileReference invListsFile = new FileReference(new File(invListsFileName));
- bufferCache.createFile(invListsFile);
- int invListsFileId = fmp.lookupFileId(invListsFile);
- bufferCache.openFile(invListsFileId);
// Declare token type traits, and compute BTree type traits.
ITypeTraits[] tokenTypeTraits = new ITypeTraits[] { UTF8StringPointable.TYPE_TRAITS };
@@ -123,10 +117,10 @@
IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
- btreeTypeTraits.length);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
+ BTree btree = new BTree(bufferCache, fmp, freePageManager, interiorFrameFactory, leafFrameFactory,
+ cmpFactories, btreeTypeTraits.length);
+ btree.create(btreeFile);
+ btree.open(btreeFile);
int invListFields = 1;
ITypeTraits[] invListTypeTraits = new ITypeTraits[invListFields];
@@ -138,8 +132,9 @@
IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
InvertedIndex invIndex = new InvertedIndex(bufferCache, btree, invListTypeTraits, invListCmpFactories,
- invListBuilder, null);
- invIndex.open(invListsFileId);
+ invListBuilder, null, fmp);
+ invIndex.create(invListsFile);
+ invIndex.open(invListsFile);
Random rnd = new Random();
rnd.setSeed(50);
@@ -214,8 +209,8 @@
MultiComparator btreeCmp = MultiComparator.create(cmpFactories);
RangePredicate btreePred = new RangePredicate(searchKey, searchKey, true, true, btreeCmp, btreeCmp);
- IInvertedListCursor invListCursor = new FixedSizeElementInvertedListCursor(bufferCache, invListsFileId,
- invListTypeTraits);
+ IInvertedListCursor invListCursor = new FixedSizeElementInvertedListCursor(bufferCache,
+ fmp.lookupFileId(invListsFile), invListTypeTraits);
ISerializerDeserializer[] tokenSerde = { UTF8StringSerializerDeserializer.INSTANCE };
RecordDescriptor tokenRecDesc = new RecordDescriptor(tokenSerde);
@@ -284,8 +279,6 @@
}
btree.close();
- bufferCache.closeFile(btreeFileId);
- bufferCache.closeFile(invListsFileId);
bufferCache.close();
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
index acce282..e10d30e 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
@@ -52,10 +52,9 @@
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(),
- harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFileId(),
- harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler());
+ harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
+ harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFlushController(),
+ harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
index 771696b..c75c2d2 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
@@ -52,10 +52,9 @@
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(),
- harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFileId(),
- harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler());
+ harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
+ harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFlushController(),
+ harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
index a6544e3..7a8929e 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexExamplesTest;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
@@ -35,16 +36,11 @@
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories)
throws TreeIndexException {
return LSMBTreeUtils.createLSMTree(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(),
+ harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), typeTraits, cmpFactories, harness.getFlushController(),
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler());
}
- @Override
- protected int getIndexFileId() {
- return harness.getFileId();
- }
-
@Before
public void setUp() throws HyracksException {
harness.setUp();
@@ -54,4 +50,9 @@
public void tearDown() throws HyracksDataException {
harness.tearDown();
}
+
+ @Override
+ protected FileReference getFileReference() {
+ return harness.getFileReference();
+ }
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
index 15fbfa1..3eba66c 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
@@ -52,10 +52,9 @@
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(),
- harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFileId(),
- harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler());
+ harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
+ harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFlushController(),
+ harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
index d2d3bd9..3cd7e42 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
@@ -51,10 +51,9 @@
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(),
- harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFileId(),
- harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler());
+ harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
+ harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFlushController(),
+ harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
index 9219676..1837e44 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
@@ -53,10 +53,9 @@
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(),
- harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFileId(),
- harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler());
+ harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
+ harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFlushController(),
+ harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
index bdefe39..62843a2 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
@@ -52,10 +52,9 @@
protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
BTreeLeafFrameType leafType) throws Exception {
return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(),
- harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFileId(),
- harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTracker(),
- harness.getIOScheduler());
+ harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
+ harness.getDiskFileMapProvider(), fieldSerdes, numKeys, harness.getFlushController(),
+ harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
index 394ffe9..0a16c4d 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
@@ -21,6 +21,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexMultiThreadTest;
import edu.uci.ics.hyracks.storage.am.common.ITreeIndexTestWorkerFactory;
import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
@@ -50,7 +51,7 @@
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories)
throws TreeIndexException {
return LSMBTreeUtils.createLSMTree(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(),
+ harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), typeTraits, cmpFactories, harness.getFlushController(),
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler());
}
@@ -102,12 +103,12 @@
}
@Override
- protected int getFileId() {
- return harness.getFileId();
+ protected String getIndexTypeName() {
+ return "LSMBTree";
}
@Override
- protected String getIndexTypeName() {
- return "LSMBTree";
+ protected FileReference getFileReference() {
+ return harness.getFileReference();
}
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/BTreeBulkLoadRunner.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/BTreeBulkLoadRunner.java
index 764faa5..8af29ed 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/BTreeBulkLoadRunner.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/BTreeBulkLoadRunner.java
@@ -35,7 +35,7 @@
@Override
public long runExperiment(DataGenThread dataGen, int numThreads) throws Exception {
- btree.create(btreeFileId);
+ btree.create(file);
long start = System.currentTimeMillis();
IIndexBulkLoader bulkLoader = btree.createBulkLoader(1.0f);
for (int i = 0; i < numBatches; i++) {
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/BTreeRunner.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/BTreeRunner.java
index ed0f373..ba0fd31 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/BTreeRunner.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/BTreeRunner.java
@@ -15,13 +15,10 @@
package edu.uci.ics.hyracks.storage.am.lsm.btree.perf;
-import java.io.File;
-
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
@@ -31,12 +28,13 @@
public class BTreeRunner extends InMemoryBTreeRunner {
protected static final int MAX_OPEN_FILES = 10;
- protected static final int HYRACKS_FRAME_SIZE = 128;
-
- public BTreeRunner(int numTuples, int pageSize, int numPages, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories) throws HyracksDataException, BTreeException {
+ protected static final int HYRACKS_FRAME_SIZE = 128;
+
+ public BTreeRunner(int numTuples, int pageSize, int numPages, ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] cmpFactories) throws HyracksDataException, BTreeException {
super(numTuples, pageSize, numPages, typeTraits, cmpFactories);
}
-
+
@Override
protected void init(int pageSize, int numPages, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories)
throws HyracksDataException, BTreeException {
@@ -44,10 +42,6 @@
TestStorageManagerComponentHolder.init(pageSize, numPages, MAX_OPEN_FILES);
bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- btreeFileId = fmp.lookupFileId(file);
- bufferCache.openFile(btreeFileId);
- btree = BTreeUtils.createBTree(bufferCache, typeTraits, cmpFactories, BTreeLeafFrameType.REGULAR_NSM);
+ btree = BTreeUtils.createBTree(bufferCache, fmp, typeTraits, cmpFactories, BTreeLeafFrameType.REGULAR_NSM);
}
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java
index 9ae149c..a23ad43 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java
@@ -15,12 +15,14 @@
package edu.uci.ics.hyracks.storage.am.lsm.btree.perf;
+import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Date;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
@@ -40,10 +42,11 @@
import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
+import edu.uci.ics.hyracks.storage.common.smi.TransientFileMapManager;
public class InMemoryBTreeRunner extends Thread implements IExperimentRunner {
protected IBufferCache bufferCache;
- protected int btreeFileId;
+ protected FileReference file;
protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
protected final static String tmpDir = System.getProperty("java.io.tmpdir");
@@ -57,22 +60,21 @@
IBinaryComparatorFactory[] cmpFactories) throws HyracksDataException, BTreeException {
this.numBatches = numBatches;
fileName = tmpDir + sep + simpleDateFormat.format(new Date());
+ file = new FileReference(new File(fileName));
init(pageSize, numPages, typeTraits, cmpFactories);
}
protected void init(int pageSize, int numPages, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories)
throws HyracksDataException, BTreeException {
ICacheMemoryAllocator allocator = new HeapBufferAllocator();
- bufferCache = new InMemoryBufferCache(allocator, pageSize, numPages);
- // Chose an aribtrary file id.
- btreeFileId = 0;
+ bufferCache = new InMemoryBufferCache(allocator, pageSize, numPages, new TransientFileMapManager());
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
IFreePageManager freePageManager = new InMemoryFreePageManager(bufferCache.getNumPages(), metaFrameFactory);
- btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
- typeTraits.length);
+ btree = new BTree(bufferCache, new TransientFileMapManager(), freePageManager, interiorFrameFactory,
+ leafFrameFactory, cmpFactories, typeTraits.length);
}
@Override
@@ -105,13 +107,12 @@
@Override
public void deinit() throws Exception {
- bufferCache.closeFile(btreeFileId);
bufferCache.close();
}
@Override
public void reset() throws Exception {
- btree.create(btreeFileId);
+ btree.create(file);
}
public class BTreeThread extends Thread {
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
index 62d9115..a1b5f8a 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
@@ -15,6 +15,7 @@
package edu.uci.ics.hyracks.storage.am.lsm.btree.perf;
+import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Date;
@@ -22,6 +23,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
@@ -42,6 +44,7 @@
import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.storage.common.smi.TransientFileMapManager;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
import edu.uci.ics.hyracks.test.support.TestUtils;
@@ -59,6 +62,7 @@
protected final static String sep = System.getProperty("file.separator");
protected final static String classDir = "/lsmtree/";
protected String onDiskDir;
+ protected FileReference file;
protected final int numBatches;
protected final LSMBTree lsmtree;
@@ -75,6 +79,7 @@
this.onDiskNumPages = onDiskNumPages;
onDiskDir = classDir + sep + simpleDateFormat.format(new Date()) + sep;
+ file = new FileReference(new File(onDiskDir));
ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
TestStorageManagerComponentHolder.init(this.onDiskPageSize, this.onDiskNumPages, MAX_OPEN_FILES);
@@ -83,12 +88,12 @@
IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
InMemoryBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), inMemPageSize,
- inMemNumPages);
+ inMemNumPages, new TransientFileMapManager());
InMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(inMemNumPages,
new LIFOMetaDataFrameFactory());
this.ioScheduler = ImmediateScheduler.INSTANCE;
- lsmtree = LSMBTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ioManager, onDiskDir, bufferCache,
- fmp, typeTraits, cmpFactories, new FlushController(), NoMergePolicy.INSTANCE,
+ lsmtree = LSMBTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ioManager, file, bufferCache, fmp,
+ typeTraits, cmpFactories, new FlushController(), NoMergePolicy.INSTANCE,
new RefCountingOperationTracker(), ioScheduler);
}
@@ -122,7 +127,7 @@
@Override
public void reset() throws Exception {
- lsmtree.create(lsmtreeFileId);
+ lsmtree.create(file);
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
index 8d5c94d..556a960 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
@@ -20,6 +20,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexTestContext;
@@ -63,17 +64,17 @@
}
public static LSMBTreeTestContext create(InMemoryBufferCache memBufferCache,
- InMemoryFreePageManager memFreePageManager, IOManager ioManager, String onDiskDir,
+ InMemoryFreePageManager memFreePageManager, IOManager ioManager, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ISerializerDeserializer[] fieldSerdes,
- int numKeyFields, int fileId, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
+ int numKeyFields, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
ILSMOperationTracker opTracker, ILSMIOScheduler ioScheduler) throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
- LSMBTree lsmTree = LSMBTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ioManager, onDiskDir,
+ LSMBTree lsmTree = LSMBTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ioManager, file,
diskBufferCache, diskFileMapProvider, typeTraits, cmpFactories, flushController, mergePolicy,
opTracker, ioScheduler);
- lsmTree.create(fileId);
- lsmTree.open(fileId);
+ lsmTree.create(file);
+ lsmTree.open(file);
LSMBTreeTestContext testCtx = new LSMBTreeTestContext(fieldSerdes, lsmTree);
return testCtx;
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
index b05ded0..a2a1753 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
@@ -25,6 +25,7 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
@@ -43,6 +44,7 @@
import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.storage.common.smi.TransientFileMapManager;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
import edu.uci.ics.hyracks.test.support.TestUtils;
@@ -52,7 +54,6 @@
public static final BTreeLeafFrameType[] LEAF_FRAMES_TO_TEST = new BTreeLeafFrameType[] { BTreeLeafFrameType.REGULAR_NSM };
private static final long RANDOM_SEED = 50;
- private static final int DUMMY_FILE_ID = -1;
protected final int diskPageSize;
protected final int diskNumPages;
@@ -76,6 +77,7 @@
protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
protected final static String sep = System.getProperty("file.separator");
protected String onDiskDir;
+ protected FileReference file;
public LSMBTreeTestHarness() {
this.diskPageSize = AccessMethodTestsConfig.LSM_BTREE_DISK_PAGE_SIZE;
@@ -106,11 +108,13 @@
public void setUp() throws HyracksException {
onDiskDir = "lsm_btree_" + simpleDateFormat.format(new Date()) + sep;
+ file = new FileReference(new File(onDiskDir));
ctx = TestUtils.create(getHyracksFrameSize());
TestStorageManagerComponentHolder.init(diskPageSize, diskNumPages, diskMaxOpenFiles);
diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
diskFileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), memPageSize, memNumPages);
+ memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), memPageSize, memNumPages,
+ new TransientFileMapManager());
memFreePageManager = new InMemoryFreePageManager(memNumPages, new LIFOMetaDataFrameFactory());
ioManager = TestStorageManagerComponentHolder.getIOManager();
rnd.setSeed(RANDOM_SEED);
@@ -161,10 +165,6 @@
return hyracksFrameSize;
}
- public int getFileId() {
- return DUMMY_FILE_ID;
- }
-
public IOManager getIOManager() {
return ioManager;
}
@@ -189,8 +189,8 @@
return ctx;
}
- public String getOnDiskDir() {
- return onDiskDir;
+ public FileReference getFileReference() {
+ return file;
}
public Random getRandom() {
diff --git a/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/InMemoryBufferCacheTest.java b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/InMemoryBufferCacheTest.java
index 2238e64..1b923e1 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/InMemoryBufferCacheTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/InMemoryBufferCacheTest.java
@@ -26,15 +26,17 @@
import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+import edu.uci.ics.hyracks.storage.common.smi.TransientFileMapManager;
-public class InMemoryBufferCacheTest{
+public class InMemoryBufferCacheTest {
private static final int PAGE_SIZE = 256;
private static final int NUM_PAGES = 100;
private HashSet<ICachedPage> pinnedPages = new HashSet<ICachedPage>();
-
+
@Test
public void test01() throws Exception {
- InMemoryBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), PAGE_SIZE, NUM_PAGES);
+ InMemoryBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), PAGE_SIZE, NUM_PAGES,
+ new TransientFileMapManager());
int dummyFileId = 0;
// Pin all pages, and make sure they return unique ICachedPages.
// We expect no overflow pages.
diff --git a/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/LSMTreeFileManagerTest.java b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/LSMTreeFileManagerTest.java
index 6c78afb..a87db81 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/LSMTreeFileManagerTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/common/LSMTreeFileManagerTest.java
@@ -53,9 +53,10 @@
protected IOManager ioManager;
protected IFileMapProvider fileMapProvider;
protected String baseDir;
+ protected FileReference file;
// Dummy finalizer that always considers components valid. Just for testing.
protected ILSMComponentFinalizer componentFinalizer = new DummyLSMComponentFinalizer();
-
+
@Before
public void setUp() throws HyracksException {
TestStorageManagerComponentHolder.init(DEFAULT_PAGE_SIZE, DEFAULT_NUM_PAGES, DEFAULT_MAX_OPEN_FILES);
@@ -64,8 +65,9 @@
baseDir = "lsm_tree" + simpleDateFormat.format(new Date()) + sep;
File f = new File(baseDir);
f.mkdirs();
+ file = new FileReference(f);
}
-
+
@After
public void tearDown() throws HyracksDataException {
File f = new File(baseDir);
@@ -73,7 +75,7 @@
}
public void sortOrderTest(boolean testFlushFileName) throws InterruptedException, HyracksDataException {
- ILSMFileManager fileManager = new LSMTreeFileManager(ioManager, fileMapProvider, baseDir);
+ ILSMFileManager fileManager = new LSMTreeFileManager(ioManager, fileMapProvider, file);
LinkedList<String> fileNames = new LinkedList<String>();
int numFileNames = 100;
@@ -109,14 +111,14 @@
sortOrderTest(true);
sortOrderTest(false);
}
-
+
public void cleanInvalidFilesTest(IOManager ioManager) throws InterruptedException, IOException {
- ILSMFileManager fileManager = new LSMTreeFileManager(ioManager, fileMapProvider, baseDir);
+ ILSMFileManager fileManager = new LSMTreeFileManager(ioManager, fileMapProvider, file);
fileManager.createDirs();
-
+
List<FileReference> flushFiles = new ArrayList<FileReference>();
List<FileReference> allFiles = new ArrayList<FileReference>();
-
+
int numFileNames = 100;
long sleepTime = 5;
// Generate a bunch of flush files.
@@ -124,10 +126,10 @@
String relFlushFileName = (String) fileManager.getRelFlushFileName();
FileReference flushFile = fileManager.createFlushFile(relFlushFileName);
flushFiles.add(flushFile);
- Thread.sleep(sleepTime);
+ Thread.sleep(sleepTime);
}
allFiles.addAll(flushFiles);
-
+
// Simulate merging some of the flush files.
// Merge range 0 to 4.
FileReference mergeFile1 = simulateMerge(fileManager, flushFiles.get(0), flushFiles.get(4));
@@ -144,19 +146,19 @@
// Merge range 50 to 79.
FileReference mergeFile5 = simulateMerge(fileManager, flushFiles.get(50), flushFiles.get(79));
allFiles.add(mergeFile5);
-
+
// Simulate merging of merge files.
FileReference mergeFile6 = simulateMerge(fileManager, mergeFile1, mergeFile2);
allFiles.add(mergeFile6);
FileReference mergeFile7 = simulateMerge(fileManager, mergeFile3, mergeFile4);
allFiles.add(mergeFile7);
-
+
// Create all files and set delete on exit for all files.
- for (FileReference fileRef : allFiles) {
+ for (FileReference fileRef : allFiles) {
fileRef.getFile().createNewFile();
fileRef.getFile().deleteOnExit();
}
-
+
// Populate expected valid flush files.
List<String> expectedValidFiles = new ArrayList<String>();
for (int i = 30; i < 50; i++) {
@@ -165,7 +167,7 @@
for (int i = 80; i < 100; i++) {
expectedValidFiles.add(flushFiles.get(i).getFile().getName());
}
-
+
// Populate expected valid merge files.
expectedValidFiles.add(mergeFile5.getFile().getName());
expectedValidFiles.add(mergeFile6.getFile().getName());
@@ -173,10 +175,10 @@
// Sort expected files.
Collections.sort(expectedValidFiles, fileManager.getFileNameComparator());
-
+
// Pass null and a dummy component finalizer. We don't test for physical consistency in this test.
List<Object> validFiles = fileManager.cleanupAndGetValidFiles(null, componentFinalizer);
-
+
// Check actual files against expected files.
assertEquals(expectedValidFiles.size(), validFiles.size());
for (int i = 0; i < expectedValidFiles.size(); i++) {
@@ -184,10 +186,10 @@
File f = new File(fileName);
assertEquals(expectedValidFiles.get(i), f.getName());
}
-
+
// Make sure invalid files were removed from all IODevices.
ArrayList<String> remainingFiles = new ArrayList<String>();
- for(IODeviceHandle dev : ioManager.getIODevices()) {
+ for (IODeviceHandle dev : ioManager.getIODevices()) {
File dir = new File(dev.getPath(), baseDir);
FilenameFilter filter = new FilenameFilter() {
public boolean accept(File dir, String name) {
@@ -200,38 +202,38 @@
remainingFiles.add(f.getName());
}
}
-
+
Collections.sort(remainingFiles, fileManager.getFileNameComparator());
// Check actual files in directory against expected files.
assertEquals(expectedValidFiles.size(), remainingFiles.size());
for (int i = 0; i < expectedValidFiles.size(); i++) {
- assertEquals(expectedValidFiles.get(i), remainingFiles.get(i));
+ assertEquals(expectedValidFiles.get(i), remainingFiles.get(i));
}
}
-
+
@Test
public void singleIODeviceTest() throws InterruptedException, IOException {
IOManager singleDeviceIOManager = createIOManager(1);
cleanInvalidFilesTest(singleDeviceIOManager);
cleanDirs(singleDeviceIOManager);
}
-
+
@Test
public void twoIODevicesTest() throws InterruptedException, IOException {
IOManager twoDevicesIOManager = createIOManager(2);
cleanInvalidFilesTest(twoDevicesIOManager);
cleanDirs(twoDevicesIOManager);
}
-
+
@Test
public void fourIODevicesTest() throws InterruptedException, IOException {
IOManager fourDevicesIOManager = createIOManager(4);
cleanInvalidFilesTest(fourDevicesIOManager);
cleanDirs(fourDevicesIOManager);
}
-
+
private void cleanDirs(IOManager ioManager) {
- for(IODeviceHandle dev : ioManager.getIODevices()) {
+ for (IODeviceHandle dev : ioManager.getIODevices()) {
File dir = new File(dev.getPath(), baseDir);
FilenameFilter filter = new FilenameFilter() {
public boolean accept(File dir, String name) {
@@ -245,7 +247,7 @@
}
}
}
-
+
private IOManager createIOManager(int numDevices) throws HyracksException {
List<IODeviceHandle> devices = new ArrayList<IODeviceHandle>();
for (int i = 0; i < numDevices; i++) {
@@ -254,14 +256,17 @@
}
return new IOManager(devices, Executors.newCachedThreadPool());
}
-
- private FileReference simulateMerge(ILSMFileManager fileManager, FileReference a, FileReference b) throws HyracksDataException {
- String relMergeFileName = (String) fileManager.getRelMergeFileName(a.getFile().getName(), b.getFile().getName());
+
+ private FileReference simulateMerge(ILSMFileManager fileManager, FileReference a, FileReference b)
+ throws HyracksDataException {
+ String relMergeFileName = (String) fileManager
+ .getRelMergeFileName(a.getFile().getName(), b.getFile().getName());
FileReference mergeFile = fileManager.createMergeFile(relMergeFileName);
return mergeFile;
}
-
- private String getMergeFileName(ILSMFileManager fileNameManager, String firstFile, String lastFile) throws HyracksDataException {
+
+ private String getMergeFileName(ILSMFileManager fileNameManager, String firstFile, String lastFile)
+ throws HyracksDataException {
File f1 = new File(firstFile);
File f2 = new File(lastFile);
return (String) fileNameManager.getRelMergeFileName(f1.getName(), f2.getName());
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
index 0ecd7da..da9ef05 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeBulkLoadTest.java
@@ -54,10 +54,10 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
return LSMRTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(),
+ harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType,
- harness.getFileId(), harness.getFlushController(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler());
+ harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTracker(),
+ harness.getIOScheduler());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
index 4202fcd..efd93ae 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeDeleteTest.java
@@ -50,10 +50,10 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
return LSMRTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(),
+ harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType,
- harness.getFileId(), harness.getFlushController(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler());
+ harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTracker(),
+ harness.getIOScheduler());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
index 8cb9172..377e630 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
@@ -38,17 +39,12 @@
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType) throws TreeIndexException {
return LSMRTreeUtils.createLSMTree(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(),
+ harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), typeTraits, rtreeCmpFactories, btreeCmpFactories,
valueProviderFactories, rtreePolicyType, harness.getFlushController(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler());
}
- @Override
- protected int getIndexFileId() {
- return harness.getFileId();
- }
-
@Before
public void setUp() throws HyracksException {
harness.setUp();
@@ -58,4 +54,10 @@
public void tearDown() throws HyracksDataException {
harness.tearDown();
}
+
+ @Override
+ protected FileReference getFileReference() {
+ return harness.getFileReference();
+ }
+
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
index 6b9fe6d..f6ac75a 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeInsertTest.java
@@ -50,10 +50,10 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
return LSMRTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(),
+ harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType,
- harness.getFileId(), harness.getFlushController(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler());
+ harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTracker(),
+ harness.getIOScheduler());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
index 06fa2f9..7e88133 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMergeTest.java
@@ -49,10 +49,10 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
return LSMRTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(),
+ harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType,
- harness.getFileId(), harness.getFlushController(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler());
+ harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTracker(),
+ harness.getIOScheduler());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMultiBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMultiBulkLoadTest.java
index 2d84e8f..710b51a 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMultiBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeMultiBulkLoadTest.java
@@ -54,10 +54,10 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
return LSMRTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(),
+ harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType,
- harness.getFileId(), harness.getFlushController(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler());
+ harness.getFlushController(), harness.getMergePolicy(), harness.getOperationTracker(),
+ harness.getIOScheduler());
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
index b1bc1a3..632b8e4 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesBulkLoadTest.java
@@ -54,9 +54,9 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getMemBufferCache(),
- harness.getMemFreePageManager(), harness.getIOManager(), harness.getOnDiskDir(),
+ harness.getMemFreePageManager(), harness.getIOManager(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
- numKeys, rtreePolicyType, harness.getFileId(), harness.getFlushController(), harness.getMergePolicy(),
+ numKeys, rtreePolicyType, harness.getFlushController(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
index 04b7ea6..6440c3a 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesDeleteTest.java
@@ -50,9 +50,9 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getMemBufferCache(),
- harness.getMemFreePageManager(), harness.getIOManager(), harness.getOnDiskDir(),
+ harness.getMemFreePageManager(), harness.getIOManager(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
- numKeys, rtreePolicyType, harness.getFileId(), harness.getFlushController(), harness.getMergePolicy(),
+ numKeys, rtreePolicyType, harness.getFlushController(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
index 8c13a00..ddfd314 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
@@ -38,17 +39,12 @@
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType) throws TreeIndexException {
return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getMemBufferCache(),
- harness.getMemFreePageManager(), harness.getIOManager(), harness.getOnDiskDir(),
+ harness.getMemFreePageManager(), harness.getIOManager(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, rtreeCmpFactories,
btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getFlushController(),
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler());
}
- @Override
- protected int getIndexFileId() {
- return harness.getFileId();
- }
-
@Before
public void setUp() throws HyracksException {
harness.setUp();
@@ -58,4 +54,9 @@
public void tearDown() throws HyracksDataException {
harness.tearDown();
}
+
+ @Override
+ protected FileReference getFileReference() {
+ return harness.getFileReference();
+ }
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
index e938109..3d266c9 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesInsertTest.java
@@ -50,9 +50,9 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getMemBufferCache(),
- harness.getMemFreePageManager(), harness.getIOManager(), harness.getOnDiskDir(),
+ harness.getMemFreePageManager(), harness.getIOManager(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
- numKeys, rtreePolicyType, harness.getFileId(), harness.getFlushController(), harness.getMergePolicy(),
+ numKeys, rtreePolicyType, harness.getFlushController(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
index 9350679..8531bb5 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMergeTest.java
@@ -49,9 +49,9 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getMemBufferCache(),
- harness.getMemFreePageManager(), harness.getIOManager(), harness.getOnDiskDir(),
+ harness.getMemFreePageManager(), harness.getIOManager(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
- numKeys, rtreePolicyType, harness.getFileId(), harness.getFlushController(), harness.getMergePolicy(),
+ numKeys, rtreePolicyType, harness.getFlushController(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMultiBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMultiBulkLoadTest.java
index 20d1ef1..c0db1cd 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMultiBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesMultiBulkLoadTest.java
@@ -54,9 +54,9 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
return LSMRTreeWithAntiMatterTuplesTestContext.create(harness.getMemBufferCache(),
- harness.getMemFreePageManager(), harness.getIOManager(), harness.getOnDiskDir(),
+ harness.getMemFreePageManager(), harness.getIOManager(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes, valueProviderFactories,
- numKeys, rtreePolicyType, harness.getFileId(), harness.getFlushController(), harness.getMergePolicy(),
+ numKeys, rtreePolicyType, harness.getFlushController(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler());
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
index 1a8779d..6d662cf 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
@@ -21,6 +21,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.common.ITreeIndexTestWorkerFactory;
import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
import edu.uci.ics.hyracks.storage.am.common.TestWorkloadConf;
@@ -53,7 +54,7 @@
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType) throws TreeIndexException {
return LSMRTreeUtils.createLSMTree(harness.getMemBufferCache(), harness.getMemFreePageManager(),
- harness.getIOManager(), harness.getOnDiskDir(), harness.getDiskBufferCache(),
+ harness.getIOManager(), harness.getFileReference(), harness.getDiskBufferCache(),
harness.getDiskFileMapProvider(), typeTraits, rtreeCmpFactories, btreeCmpFactories,
valueProviderFactories, rtreePolicyType, harness.getFlushController(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler());
@@ -103,13 +104,13 @@
}
@Override
- protected int getFileId() {
- return harness.getFileId();
+ protected String getIndexTypeName() {
+ return "LSMRTree";
}
@Override
- protected String getIndexTypeName() {
- return "LSMRTree";
+ protected FileReference getFileReference() {
+ return harness.getFileReference();
}
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
index 8616946..eab866c 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
@@ -21,6 +21,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.common.ITreeIndexTestWorkerFactory;
import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
import edu.uci.ics.hyracks.storage.am.common.TestWorkloadConf;
@@ -53,7 +54,7 @@
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType) throws TreeIndexException {
return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getMemBufferCache(),
- harness.getMemFreePageManager(), harness.getIOManager(), harness.getOnDiskDir(),
+ harness.getMemFreePageManager(), harness.getIOManager(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, rtreeCmpFactories,
btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getFlushController(),
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler());
@@ -104,13 +105,13 @@
}
@Override
- protected int getFileId() {
- return harness.getFileId();
+ protected String getIndexTypeName() {
+ return "LSMRTree";
}
@Override
- protected String getIndexTypeName() {
- return "LSMRTree";
+ protected FileReference getFileReference() {
+ return harness.getFileReference();
}
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
index 2bbff9e..7b92cce 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
@@ -20,6 +20,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
@@ -67,21 +68,21 @@
}
public static LSMRTreeTestContext create(InMemoryBufferCache memBufferCache,
- InMemoryFreePageManager memFreePageManager, IOManager ioManager, String onDiskDir,
+ InMemoryFreePageManager memFreePageManager, IOManager ioManager, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeyFields, RTreePolicyType rtreePolicyType,
- int fileId, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTracker opTracker, ILSMIOScheduler ioScheduler) throws Exception {
+ ILSMFlushController flushController, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOScheduler ioScheduler) throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] rtreeCmpFactories = SerdeUtils
.serdesToComparatorFactories(fieldSerdes, numKeyFields);
IBinaryComparatorFactory[] btreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes,
fieldSerdes.length);
- LSMRTree lsmTree = LSMRTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ioManager, onDiskDir,
+ LSMRTree lsmTree = LSMRTreeUtils.createLSMTree(memBufferCache, memFreePageManager, ioManager, file,
diskBufferCache, diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories,
valueProviderFactories, rtreePolicyType, flushController, mergePolicy, opTracker, ioScheduler);
- lsmTree.create(fileId);
- lsmTree.open(fileId);
+ lsmTree.create(file);
+ lsmTree.open(file);
LSMRTreeTestContext testCtx = new LSMRTreeTestContext(fieldSerdes, lsmTree);
return testCtx;
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
index 405f791..0456879 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestHarness.java
@@ -25,6 +25,7 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
@@ -75,6 +76,7 @@
protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
protected final static String sep = System.getProperty("file.separator");
protected String onDiskDir;
+ protected FileReference file;
public LSMRTreeTestHarness() {
this.diskPageSize = AccessMethodTestsConfig.LSM_RTREE_DISK_PAGE_SIZE;
@@ -105,6 +107,7 @@
public void setUp() throws HyracksException {
onDiskDir = "lsm_rtree_" + simpleDateFormat.format(new Date()) + sep;
+ file = new FileReference(new File(onDiskDir));
ctx = TestUtils.create(getHyracksFrameSize());
TestStorageManagerComponentHolder.init(diskPageSize, diskNumPages, diskMaxOpenFiles);
diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
@@ -160,10 +163,6 @@
return hyracksFrameSize;
}
- public int getFileId() {
- return DUMMY_FILE_ID;
- }
-
public IOManager getIOManager() {
return ioManager;
}
@@ -192,6 +191,10 @@
return onDiskDir;
}
+ public FileReference getFileReference() {
+ return file;
+ }
+
public Random getRandom() {
return rnd;
}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
index 4b2e010..15a06c5 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
@@ -20,6 +20,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
@@ -67,22 +68,22 @@
}
public static LSMRTreeWithAntiMatterTuplesTestContext create(InMemoryBufferCache memBufferCache,
- InMemoryFreePageManager memFreePageManager, IOManager ioManager, String onDiskDir,
+ InMemoryFreePageManager memFreePageManager, IOManager ioManager, FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeyFields, RTreePolicyType rtreePolicyType,
- int fileId, ILSMFlushController flushController, ILSMMergePolicy mergePolicy,
- ILSMOperationTracker opTracker, ILSMIOScheduler ioScheduler) throws Exception {
+ ILSMFlushController flushController, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOScheduler ioScheduler) throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] rtreeCmpFactories = SerdeUtils
.serdesToComparatorFactories(fieldSerdes, numKeyFields);
IBinaryComparatorFactory[] btreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes,
fieldSerdes.length);
LSMRTreeWithAntiMatterTuples lsmTree = LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(memBufferCache,
- memFreePageManager, ioManager, onDiskDir, diskBufferCache, diskFileMapProvider, typeTraits,
+ memFreePageManager, ioManager, file, diskBufferCache, diskFileMapProvider, typeTraits,
rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, flushController,
mergePolicy, opTracker, ioScheduler);
- lsmTree.create(fileId);
- lsmTree.open(fileId);
+ lsmTree.create(file);
+ lsmTree.open(file);
LSMRTreeWithAntiMatterTuplesTestContext testCtx = new LSMRTreeWithAntiMatterTuplesTestContext(fieldSerdes,
lsmTree);
return testCtx;
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeBulkLoadTest.java
index da38708..d2081a7 100644
--- a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeBulkLoadTest.java
@@ -50,8 +50,8 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return RTreeTestContext.create(harness.getBufferCache(), harness.getTreeFileId(), fieldSerdes,
- valueProviderFactories, numKeys, rtreePolicyType);
+ return RTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType);
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeDeleteTest.java b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeDeleteTest.java
index 4d0bad4..c5ba01dc 100644
--- a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeDeleteTest.java
+++ b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeDeleteTest.java
@@ -46,8 +46,8 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return RTreeTestContext.create(harness.getBufferCache(), harness.getTreeFileId(), fieldSerdes,
- valueProviderFactories, numKeys, rtreePolicyType);
+ return RTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType);
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeExamplesTest.java b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeExamplesTest.java
index 99b496f..e6900e5 100644
--- a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeExamplesTest.java
+++ b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeExamplesTest.java
@@ -21,6 +21,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
@@ -45,12 +46,12 @@
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType) throws TreeIndexException {
- return RTreeUtils.createRTree(harness.getBufferCache(), typeTraits, valueProviderFactories, rtreeCmpFactories,
- rtreePolicyType);
+ return RTreeUtils.createRTree(harness.getBufferCache(), harness.getFileMapProvider(), typeTraits,
+ valueProviderFactories, rtreeCmpFactories, rtreePolicyType);
}
@Override
- protected int getIndexFileId() {
- return harness.getTreeFileId();
+ protected FileReference getFileReference() {
+ return harness.getFileReference();
}
}
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeInsertTest.java b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeInsertTest.java
index 193e067..b5e01b7 100644
--- a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeInsertTest.java
+++ b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeInsertTest.java
@@ -46,8 +46,8 @@
protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
throws Exception {
- return RTreeTestContext.create(harness.getBufferCache(), harness.getTreeFileId(), fieldSerdes,
- valueProviderFactories, numKeys, rtreePolicyType);
+ return RTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(),
+ harness.getFileReference(), fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType);
}
@Override
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeSearchCursorTest.java b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
index 7f73e33..8be3e8c 100644
--- a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
+++ b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
@@ -78,7 +78,6 @@
}
IBufferCache bufferCache = harness.getBufferCache();
- int rtreeFileId = harness.getTreeFileId();
// Declare fields.
int fieldCount = 5;
@@ -117,14 +116,15 @@
IRTreeLeafFrame leafFrame = (IRTreeLeafFrame) leafFrameFactory.createFrame();
IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
- RTree rtree = new RTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
- fieldCount);
- rtree.create(rtreeFileId);
- rtree.open(rtreeFileId);
+ RTree rtree = new RTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory,
+ leafFrameFactory, cmpFactories, fieldCount);
+ rtree.create(harness.getFileReference());
+ rtree.open(harness.getFileReference());
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
- ITreeIndexAccessor indexAccessor = rtree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ ITreeIndexAccessor indexAccessor = rtree.createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
int numInserts = 10000;
ArrayList<RTreeCheckTuple> checkTuples = new ArrayList<RTreeCheckTuple>();
for (int i = 0; i < numInserts; i++) {
@@ -133,8 +133,7 @@
int p2x = rnd.nextInt();
int p2y = rnd.nextInt();
- int pk = rnd.nextInt();
- ;
+ int pk = rnd.nextInt();;
TupleUtils.createIntegerTuple(tb, tuple, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x),
Math.max(p1y, p2y), pk);
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java
index 2e8c28f..8ffd1b3 100644
--- a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java
+++ b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java
@@ -20,6 +20,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.common.ITreeIndexTestWorkerFactory;
import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
import edu.uci.ics.hyracks.storage.am.common.TestWorkloadConf;
@@ -51,8 +52,8 @@
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType) throws TreeIndexException {
- return RTreeUtils.createRTree(harness.getBufferCache(), typeTraits, valueProviderFactories, rtreeCmpFactories,
- rtreePolicyType);
+ return RTreeUtils.createRTree(harness.getBufferCache(), harness.getFileMapProvider(), typeTraits,
+ valueProviderFactories, rtreeCmpFactories, rtreePolicyType);
}
@@ -87,12 +88,12 @@
}
@Override
- protected int getFileId() {
- return harness.getTreeFileId();
+ protected String getIndexTypeName() {
+ return "RTree";
}
@Override
- protected String getIndexTypeName() {
- return "RTree";
+ protected FileReference getFileReference() {
+ return harness.getFileReference();
}
}
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/utils/RTreeTestContext.java b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/utils/RTreeTestContext.java
index 4591398..de69e72 100644
--- a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/utils/RTreeTestContext.java
+++ b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/utils/RTreeTestContext.java
@@ -18,6 +18,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
@@ -26,6 +27,7 @@
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@SuppressWarnings("rawtypes")
public class RTreeTestContext extends AbstractRTreeTestContext {
@@ -46,15 +48,15 @@
return rtree.getComparatorFactories();
}
- public static RTreeTestContext create(IBufferCache bufferCache, int rtreeFileId,
+ public static RTreeTestContext create(IBufferCache bufferCache, IFileMapProvider fileMapProvider, FileReference file,
ISerializerDeserializer[] fieldSerdes, IPrimitiveValueProviderFactory[] valueProviderFactories,
int numKeyFields, RTreePolicyType rtreePolicyType) throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
- RTree rtree = RTreeUtils.createRTree(bufferCache, typeTraits, valueProviderFactories, cmpFactories,
- rtreePolicyType);
- rtree.create(rtreeFileId);
- rtree.open(rtreeFileId);
+ RTree rtree = RTreeUtils.createRTree(bufferCache, fileMapProvider, typeTraits, valueProviderFactories,
+ cmpFactories, rtreePolicyType);
+ rtree.create(file);
+ rtree.open(file);
RTreeTestContext testCtx = new RTreeTestContext(fieldSerdes, rtree);
return testCtx;
}
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/utils/RTreeTestHarness.java b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/utils/RTreeTestHarness.java
index e463f57..210c2b0 100644
--- a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/utils/RTreeTestHarness.java
+++ b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/utils/RTreeTestHarness.java
@@ -40,6 +40,7 @@
protected IHyracksTaskContext ctx;
protected IBufferCache bufferCache;
+ protected IFileMapProvider fileMapProvider;
protected int treeFileId;
protected final Random rnd = new Random();
@@ -47,10 +48,11 @@
protected final String tmpDir = System.getProperty("java.io.tmpdir");
protected final String sep = System.getProperty("file.separator");
protected String fileName;
+ protected FileReference file;
public RTreeTestHarness() {
this.pageSize = AccessMethodTestsConfig.RTREE_PAGE_SIZE;
- this.numPages = AccessMethodTestsConfig.RTREE_NUM_PAGES;
+ this.numPages = AccessMethodTestsConfig.RTREE_NUM_PAGES;
this.maxOpenFiles = AccessMethodTestsConfig.RTREE_MAX_OPEN_FILES;
this.hyracksFrameSize = AccessMethodTestsConfig.RTREE_HYRACKS_FRAME_SIZE;
}
@@ -64,13 +66,14 @@
public void setUp() throws HyracksDataException {
fileName = tmpDir + sep + simpleDateFormat.format(new Date());
+ file = new FileReference(new File(fileName));
ctx = TestUtils.create(getHyracksFrameSize());
TestStorageManagerComponentHolder.init(pageSize, numPages, maxOpenFiles);
bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ fileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
FileReference file = new FileReference(new File(fileName));
bufferCache.createFile(file);
- treeFileId = fmp.lookupFileId(file);
+ treeFileId = fileMapProvider.lookupFileId(file);
bufferCache.openFile(treeFileId);
rnd.setSeed(RANDOM_SEED);
}
@@ -90,8 +93,8 @@
return bufferCache;
}
- public int getTreeFileId() {
- return treeFileId;
+ public IFileMapProvider getFileMapProvider() {
+ return fileMapProvider;
}
public String getFileName() {
@@ -117,4 +120,8 @@
public int getMaxOpenFiles() {
return maxOpenFiles;
}
+
+ public FileReference getFileReference() {
+ return file;
+ }
}