Fixed various bugs that prevented an lsm index from being partitioned accross iodevices.
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
index e46efff..5bd3298 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
@@ -15,6 +15,7 @@
package edu.uci.ics.hyracks.storage.am.common.dataflow;
+import java.io.File;
import java.io.IOException;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
@@ -37,6 +38,7 @@
protected final ResourceIdFactory resourceIdFactory;
protected final FileReference file;
protected final int partition;
+ protected final int ioDeviceId;
protected IIndex index;
@@ -47,7 +49,10 @@
this.localResourceRepository = opDesc.getStorageManager().getLocalResourceRepository(ctx);
this.resourceIdFactory = opDesc.getStorageManager().getResourceIdFactory(ctx);
this.partition = partition;
- this.file = opDesc.getFileSplitProvider().getFileSplits()[partition].getLocalFile();
+ this.ioDeviceId = opDesc.getFileSplitProvider().getFileSplits()[partition].getIODeviceId();
+ this.file = new FileReference(new File(opDesc.getFileSplitProvider().getFileSplits()[partition].getLocalFile()
+ .getFile().getPath()
+ + "_" + ioDeviceId));
}
protected abstract IIndex createIndexInstance() throws HyracksDataException;
@@ -70,7 +75,7 @@
// any physical artifact that the LocalResourceRepository is managing (e.g. a file containing the resource ID).
// Once the index has been created, a new resource ID can be generated.
if (resourceID != -1) {
- localResourceRepository.deleteResourceByName(file.getFile().getPath());
+ localResourceRepository.deleteResourceByName(file.getFile().getPath(), ioDeviceId);
}
index.create();
try {
@@ -78,8 +83,9 @@
resourceID = resourceIdFactory.createId();
ILocalResourceFactory localResourceFactory = opDesc.getLocalResourceFactoryProvider()
.getLocalResourceFactory();
- localResourceRepository.insert(localResourceFactory.createLocalResource(resourceID, file.getFile()
- .getPath(), partition));
+ localResourceRepository.insert(
+ localResourceFactory.createLocalResource(resourceID, file.getFile().getPath(), partition),
+ ioDeviceId);
} catch (IOException e) {
throw new HyracksDataException(e);
}
@@ -121,7 +127,7 @@
}
if (resourceID != -1) {
- localResourceRepository.deleteResourceByName(file.getFile().getPath());
+ localResourceRepository.deleteResourceByName(file.getFile().getPath(), ioDeviceId);
}
index.destroy();
}
@@ -143,4 +149,4 @@
public IHyracksTaskContext getTaskContext() {
return ctx;
}
-}
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
index 6a894e9..0080c42 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
@@ -27,7 +27,6 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IIOManager;
-import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
@@ -80,17 +79,16 @@
ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<ComparableFileName>();
ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<ComparableFileName>();
- // Gather files from all IODeviceHandles.
- for (IODeviceHandle dev : ioManager.getIODevices()) {
- // List of valid BTree files.
- cleanupAndGetValidFilesInternal(dev, btreeFilter, btreeFactory, allBTreeFiles);
- HashSet<String> btreeFilesSet = new HashSet<String>();
- for (ComparableFileName cmpFileName : allBTreeFiles) {
- int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
- btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
- }
- validateFiles(dev, btreeFilesSet, allBloomFilterFiles, bloomFilterFilter, null);
+ // Gather files from the IODeviceHandle.
+
+ // List of valid BTree files.
+ cleanupAndGetValidFilesInternal(dev, btreeFilter, btreeFactory, allBTreeFiles);
+ HashSet<String> btreeFilesSet = new HashSet<String>();
+ for (ComparableFileName cmpFileName : allBTreeFiles) {
+ int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
+ btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
}
+ validateFiles(dev, btreeFilesSet, allBloomFilterFiles, bloomFilterFilter, null);
// Sanity check.
if (allBTreeFiles.size() != allBloomFilterFiles.size()) {
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
index ca147d6..99b62d8 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
@@ -44,7 +44,6 @@
protected static final String SPLIT_STRING = "_";
protected static final String BLOOM_FILTER_STRING = "f";
- protected final IIOManager ioManager;
protected final IFileMapProvider fileMapProvider;
protected final IODeviceHandle dev;
@@ -63,7 +62,6 @@
baseDir += System.getProperty("file.separator");
}
this.fileMapProvider = fileMapProvider;
- this.ioManager = ioManager;
this.treeFactory = treeFactory;
this.dev = ioManager.getIODevices().get(ioDeviceId);
}
@@ -131,18 +129,14 @@
@Override
public void createDirs() {
- for (IODeviceHandle dev : ioManager.getIODevices()) {
- File f = new File(dev.getPath(), baseDir);
- f.mkdirs();
- }
+ File f = new File(dev.getPath(), baseDir);
+ f.mkdirs();
}
@Override
public void deleteDirs() {
- for (IODeviceHandle dev : ioManager.getIODevices()) {
- File f = new File(dev.getPath(), baseDir);
- delete(f);
- }
+ File f = new File(dev.getPath(), baseDir);
+ delete(f);
}
private void delete(File f) {
diff --git a/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/ILocalResourceRepository.java b/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/ILocalResourceRepository.java
index ab9ec41..36aa088 100644
--- a/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/ILocalResourceRepository.java
+++ b/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/ILocalResourceRepository.java
@@ -19,16 +19,16 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
public interface ILocalResourceRepository {
-
+
public LocalResource getResourceById(long id) throws HyracksDataException;
public LocalResource getResourceByName(String name) throws HyracksDataException;
- public void insert(LocalResource resource) throws HyracksDataException;
+ public void insert(LocalResource resource, int ioDeviceId) throws HyracksDataException;
- public void deleteResourceById(long id) throws HyracksDataException;
+ public void deleteResourceById(long id, int ioDeviceId) throws HyracksDataException;
- public void deleteResourceByName(String name) throws HyracksDataException;
+ public void deleteResourceByName(String name, int ioDeviceId) throws HyracksDataException;
public List<LocalResource> getAllResources() throws HyracksDataException;
}
diff --git a/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/TransientLocalResourceRepository.java b/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/TransientLocalResourceRepository.java
index 55bd807..f853a36 100644
--- a/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/TransientLocalResourceRepository.java
+++ b/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/TransientLocalResourceRepository.java
@@ -37,7 +37,7 @@
}
@Override
- public synchronized void insert(LocalResource resource) throws HyracksDataException {
+ public synchronized void insert(LocalResource resource, int ioDeviceId) throws HyracksDataException {
long id = resource.getResourceId();
if (id2ResourceMap.containsKey(id)) {
@@ -48,7 +48,7 @@
}
@Override
- public synchronized void deleteResourceById(long id) throws HyracksDataException {
+ public synchronized void deleteResourceById(long id, int ioDeviceId) throws HyracksDataException {
LocalResource resource = id2ResourceMap.get(id);
if (resource == null) {
throw new HyracksDataException("Resource doesn't exist");
@@ -58,7 +58,7 @@
}
@Override
- public synchronized void deleteResourceByName(String name) throws HyracksDataException {
+ public synchronized void deleteResourceByName(String name, int ioDeviceId) throws HyracksDataException {
LocalResource resource = name2ResourceMap.get(name);
if (resource == null) {
throw new HyracksDataException("Resource doesn't exist");