ASTERIXDB-1273: Get file references from components to be merged
This change includes the following:
- Get file references from components to be merged instead of file map manager.
- Fix an issue with passing inverted index files to replication job.
Change-Id: I7d0f06c002db86a2a34e6393d06c49fb528c22b7
Reviewed-on: https://asterix-gerrit.ics.uci.edu/596
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
Reviewed-by: Ian Maxon <imaxon@apache.org>
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index 1942623..5183e1d 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -198,8 +198,8 @@
ITreeIndexCursor cursor = new LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples);
BTree firstBTree = ((LSMBTreeDiskComponent) mergingComponents.get(0)).getBTree();
BTree lastBTree = ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1)).getBTree();
- FileReference firstFile = diskFileMapProvider.lookupFileName(firstBTree.getFileId());
- FileReference lastFile = diskFileMapProvider.lookupFileName(lastBTree.getFileId());
+ FileReference firstFile = firstBTree.getFileReference();
+ FileReference lastFile = lastBTree.getFileReference();
LSMComponentFileReferences relMergeFileRefs = fileManager
.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
ILSMIndexAccessorInternal accessor = new LSMBTreeAccessor(lsmHarness, opCtx);
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index cdd797b..a4f355f 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -287,8 +287,7 @@
public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
boolean checkIfEmptyIndex) throws TreeIndexException {
try {
- return new LSMTwoPCBTreeWithBuddyBulkLoader(fillLevel, verifyInput, 0, checkIfEmptyIndex,
- false);
+ return new LSMTwoPCBTreeWithBuddyBulkLoader(fillLevel, verifyInput, 0, checkIfEmptyIndex, false);
} catch (HyracksDataException e) {
throw new TreeIndexException(e);
}
@@ -336,8 +335,8 @@
BTree firstTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(0)).getBTree();
BTree lastTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1))
.getBTree();
- FileReference firstFile = diskFileMapProvider.lookupFileName(firstTree.getFileId());
- FileReference lastFile = diskFileMapProvider.lookupFileName(lastTree.getFileId());
+ FileReference firstFile = firstTree.getFileReference();
+ FileReference lastFile = lastTree.getFileReference();
LSMComponentFileReferences fileRefs = fileManager.getRelMergeFileReference(firstFile.getFile().getName(),
lastFile.getFile().getName());
return fileRefs;
@@ -358,8 +357,8 @@
// which list to check against and we need to synchronize for this
boolean keepDeleteTuples = false;
if (version == 0) {
- keepDeleteTuples = mergingComponents.get(mergingComponents.size() - 1) != diskComponents.get(diskComponents
- .size() - 1);
+ keepDeleteTuples = mergingComponents.get(mergingComponents.size() - 1) != diskComponents
+ .get(diskComponents.size() - 1);
} else {
keepDeleteTuples = mergingComponents.get(mergingComponents.size() - 1) != secondDiskComponents
.get(secondDiskComponents.size() - 1);
@@ -373,7 +372,8 @@
// This method creates the appropriate opContext for the targeted version
public ExternalBTreeWithBuddyOpContext createOpContext(ISearchOperationCallback searchCallback, int targetVersion) {
return new ExternalBTreeWithBuddyOpContext(btreeCmpFactories, buddyBtreeCmpFactories, searchCallback,
- targetVersion, lsmHarness, btreeInteriorFrameFactory, btreeLeafFrameFactory, buddyBtreeLeafFrameFactory);
+ targetVersion, lsmHarness, btreeInteriorFrameFactory, btreeLeafFrameFactory,
+ buddyBtreeLeafFrameFactory);
}
@Override
@@ -405,8 +405,8 @@
long numElements = 0L;
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
- numElements += ((LSMBTreeWithBuddyDiskComponent) mergeOp.getMergingComponents().get(i))
- .getBloomFilter().getNumElements();
+ numElements += ((LSMBTreeWithBuddyDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter()
+ .getNumElements();
}
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
@@ -487,7 +487,7 @@
public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException {
LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) lsmComponent;
// Flush the bloom filter first.
- markAsValidInternal(component.getBTree().getBufferCache(),component.getBloomFilter());
+ markAsValidInternal(component.getBTree().getBufferCache(), component.getBloomFilter());
markAsValidInternal(component.getBTree());
markAsValidInternal(component.getBuddyBTree());
}
@@ -599,9 +599,8 @@
FileReference insertFileRef, FileReference deleteFileRef, FileReference bloomFilterFileRef,
boolean createComponent) throws HyracksDataException, IndexException {
// Create new instance.
- LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) factory
- .createLSMComponentInstance(new LSMComponentFileReferences(insertFileRef, deleteFileRef,
- bloomFilterFileRef));
+ LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) factory.createLSMComponentInstance(
+ new LSMComponentFileReferences(insertFileRef, deleteFileRef, bloomFilterFileRef));
if (createComponent) {
component.getBloomFilter().create();
}
@@ -670,7 +669,7 @@
// Create the three loaders
btreeBulkLoader = (BTreeBulkLoader) ((LSMBTreeWithBuddyDiskComponent) component).getBTree()
- .createBulkLoader(fillFactor, verifyInput, numElementsHint, false,true);
+ .createBulkLoader(fillFactor, verifyInput, numElementsHint, false, true);
buddyBtreeBulkLoader = (BTreeBulkLoader) ((LSMBTreeWithBuddyDiskComponent) component).getBuddyBTree()
.createBulkLoader(fillFactor, verifyInput, numElementsHint, false, true);
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
@@ -864,25 +863,22 @@
@Override
public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
Set<String> files = new HashSet<String>();
-
LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) lsmComponent;
- files.add(component.getBTree().getFileReference().toString());
- files.add(component.getBuddyBTree().getFileReference().toString());
- files.add(component.getBloomFilter().getFileReference().toString());
-
+ files.add(component.getBTree().getFileReference().getFile().getAbsolutePath());
+ files.add(component.getBuddyBTree().getFileReference().getFile().getAbsolutePath());
+ files.add(component.getBloomFilter().getFileReference().getFile().getAbsolutePath());
return files;
}
@Override
public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
boolean checkIfEmptyIndex, boolean appendOnly) throws IndexException {
- if(!appendOnly){
+ if (!appendOnly) {
throw new IndexException("LSM Indices do not support in-place inserts");
+ } else {
+ return createBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex);
}
- else{
- return createBulkLoader(fillFactor, verifyInput,numElementsHint,checkIfEmptyIndex);
- }
- }
+ }
@Override
public void allocateMemoryComponents() throws HyracksDataException {
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 841dbc4..ae3aa52 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -488,8 +488,8 @@
ITreeIndexCursor cursor = new LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples);
BTree firstBTree = ((LSMBTreeDiskComponent) mergingComponents.get(0)).getBTree();
BTree lastBTree = ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1)).getBTree();
- FileReference firstFile = diskFileMapProvider.lookupFileName(firstBTree.getFileId());
- FileReference lastFile = diskFileMapProvider.lookupFileName(lastBTree.getFileId());
+ FileReference firstFile = firstBTree.getFileReference();
+ FileReference lastFile = lastBTree.getFileReference();
LSMComponentFileReferences relMergeFileRefs = fileManager
.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
ILSMIndexAccessorInternal accessor = new LSMBTreeAccessor(lsmHarness, opCtx);
@@ -659,7 +659,7 @@
}
}
- protected void cleanupArtifacts() throws HyracksDataException, IndexException {
+ protected void cleanupArtifacts() throws HyracksDataException {
if (!cleanedUpArtifacts) {
cleanedUpArtifacts = true;
if (!endedBloomFilterLoad) {
@@ -835,10 +835,8 @@
public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
Set<String> files = new HashSet<String>();
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) lsmComponent;
-
- files.add(component.getBTree().getFileReference().toString());
- files.add(component.getBloomFilter().getFileReference().toString());
-
+ files.add(component.getBTree().getFileReference().getFile().getAbsolutePath());
+ files.add(component.getBloomFilter().getFileReference().getFile().getAbsolutePath());
return files;
}
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 1f8feca..284fc1b 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -38,7 +38,16 @@
import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
import org.apache.hyracks.storage.am.btree.util.BTreeUtils;
-import org.apache.hyracks.storage.am.common.api.*;
+import org.apache.hyracks.storage.am.common.api.ICursorInitialState;
+import org.apache.hyracks.storage.am.common.api.IIndexAccessor;
+import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
+import org.apache.hyracks.storage.am.common.api.IIndexCursor;
+import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
+import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
+import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
+import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
+import org.apache.hyracks.storage.am.common.api.IVirtualMetaDataPageManager;
+import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.common.exceptions.TreeIndexDuplicateKeyException;
import org.apache.hyracks.storage.am.common.impls.AbstractSearchPredicate;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
@@ -128,7 +137,7 @@
InMemoryInvertedIndex memInvIndex = createInMemoryInvertedIndex(virtualBufferCache,
new VirtualMetaDataPageManager(virtualBufferCache.getNumPages()), i);
BTree deleteKeysBTree = BTreeUtils.createBTree(virtualBufferCache, new VirtualMetaDataPageManager(
- virtualBufferCache.getNumPages()), ((IVirtualBufferCache) virtualBufferCache).getFileMapProvider(),
+ virtualBufferCache.getNumPages()), virtualBufferCache.getFileMapProvider(),
invListTypeTraits, invListCmpFactories, BTreeLeafFrameType.REGULAR_NSM, new FileReference(new File(
fileManager.getBaseDir() + "_virtual_del_" + i)));
LSMInvertedIndexMemoryComponent mutableComponent = new LSMInvertedIndexMemoryComponent(memInvIndex,
@@ -874,12 +883,7 @@
public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException {
LSMInvertedIndexDiskComponent invIndexComponent = (LSMInvertedIndexDiskComponent) lsmComponent;
OnDiskInvertedIndex invIndex = (OnDiskInvertedIndex) invIndexComponent.getInvIndex();
- // Flush the bloom filter first.
- int fileId = invIndexComponent.getBloomFilter().getFileId();
IBufferCache bufferCache = invIndex.getBufferCache();
- int startPage = 0;
- int maxPage = invIndexComponent.getBloomFilter().getNumPages();
-
markAsValidInternal(invIndex.getBufferCache(),invIndexComponent.getBloomFilter());
// Flush inverted index second.
@@ -919,15 +923,12 @@
@Override
public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
Set<String> files = new HashSet<String>();
-
LSMInvertedIndexDiskComponent invIndexComponent = (LSMInvertedIndexDiskComponent) lsmComponent;
OnDiskInvertedIndex invIndex = (OnDiskInvertedIndex) invIndexComponent.getInvIndex();
-
- files.add(invIndex.getInvListsFile().toString());
- files.add(invIndex.getBTree().toString());
- files.add(invIndexComponent.getBloomFilter().getFileReference().toString());
- files.add(invIndexComponent.getDeletedKeysBTree().getFileReference().toString());
-
+ files.add(invIndex.getInvListsFile().getFile().getAbsolutePath());
+ files.add(invIndex.getBTree().getFileReference().getFile().getAbsolutePath());
+ files.add(invIndexComponent.getBloomFilter().getFileReference().getFile().getAbsolutePath());
+ files.add(invIndexComponent.getDeletedKeysBTree().getFileReference().getFile().getAbsolutePath());
return files;
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index a03d251..61284f0 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -267,8 +267,8 @@
RTree firstTree = ((LSMRTreeDiskComponent) mergingDiskComponents.get(0)).getRTree();
RTree lastTree = ((LSMRTreeDiskComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1))
.getRTree();
- FileReference firstFile = diskFileMapProvider.lookupFileName(firstTree.getFileId());
- FileReference lastFile = diskFileMapProvider.lookupFileName(lastTree.getFileId());
+ FileReference firstFile = firstTree.getFileReference();
+ FileReference lastFile = lastTree.getFileReference();
LSMComponentFileReferences fileRefs = fileManager.getRelMergeFileReference(firstFile.getFile().getName(),
lastFile.getFile().getName());
return fileRefs;
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index f177ce0..f839e15 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -639,13 +639,10 @@
@Override
public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
Set<String> files = new HashSet<String>();
-
LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) lsmComponent;
-
- files.add(component.getBTree().getFileReference().toString());
- files.add(component.getRTree().getFileReference().toString());
- files.add(component.getBloomFilter().getFileReference().toString());
-
+ files.add(component.getBTree().getFileReference().getFile().getAbsolutePath());
+ files.add(component.getRTree().getFileReference().getFile().getAbsolutePath());
+ files.add(component.getBloomFilter().getFileReference().getFile().getAbsolutePath());
return files;
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index 2388cff..daebb5f 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -471,10 +471,8 @@
@Override
public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
Set<String> files = new HashSet<String>();
-
RTree rtree = ((LSMRTreeDiskComponent) lsmComponent).getRTree();
- files.add(rtree.getFileReference().toString());
-
+ files.add(rtree.getFileReference().getFile().getAbsolutePath());
return files;
}
}