checkpoint of the local resource repository
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_lsm_tree@1925 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/RuntimeContext.java b/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/RuntimeContext.java
index a0f2ff0..b37368e 100644
--- a/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/RuntimeContext.java
+++ b/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/RuntimeContext.java
@@ -17,6 +17,7 @@
import edu.uci.ics.hyracks.api.application.INCApplicationContext;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManager;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexLifecycleManager;
import edu.uci.ics.hyracks.storage.common.TransientIndexArtifactMap;
@@ -29,21 +30,25 @@
import edu.uci.ics.hyracks.storage.common.file.IFileMapManager;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
import edu.uci.ics.hyracks.storage.common.file.IIndexArtifactMap;
+import edu.uci.ics.hyracks.storage.common.file.ILocalResourceRepository;
+import edu.uci.ics.hyracks.storage.common.file.ILocalResourceRepositoryFactory;
+import edu.uci.ics.hyracks.storage.common.file.IndexLocalResourceRepositoryFactory;
import edu.uci.ics.hyracks.storage.common.file.TransientFileMapManager;
public class RuntimeContext {
private IBufferCache bufferCache;
private IFileMapManager fileMapManager;
- private IIndexArtifactMap indexArtifactMap;
+ private ILocalResourceRepository localResourceRepository;
private IIndexLifecycleManager lcManager;
- public RuntimeContext(INCApplicationContext appCtx) {
+ public RuntimeContext(INCApplicationContext appCtx) throws HyracksDataException {
fileMapManager = new TransientFileMapManager();
ICacheMemoryAllocator allocator = new HeapBufferAllocator();
IPageReplacementStrategy prs = new ClockPageReplacementStrategy();
bufferCache = new BufferCache(appCtx.getRootContext().getIOManager(), allocator, prs, fileMapManager, 32768,
50, 100);
- indexArtifactMap = new TransientIndexArtifactMap();
+ ILocalResourceRepositoryFactory indexLocalResourceRepositoryFactory = new IndexLocalResourceRepositoryFactory(appCtx.getRootContext().getIOManager());
+ localResourceRepository = indexLocalResourceRepositoryFactory.createRepository();
lcManager = new IndexLifecycleManager();
}
@@ -63,8 +68,8 @@
return (RuntimeContext) ctx.getJobletContext().getApplicationContext().getApplicationObject();
}
- public IIndexArtifactMap getIndexArtifactMap() {
- return indexArtifactMap;
+ public ILocalResourceRepository getLocalResourceRepository() {
+ return localResourceRepository;
}
public IIndexLifecycleManager getIndexLifecycleManager() {
diff --git a/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/StorageManagerInterface.java b/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/StorageManagerInterface.java
index a1e94d2..953f7c1 100644
--- a/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/StorageManagerInterface.java
+++ b/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/StorageManagerInterface.java
@@ -19,7 +19,7 @@
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-import edu.uci.ics.hyracks.storage.common.file.IIndexArtifactMap;
+import edu.uci.ics.hyracks.storage.common.file.ILocalResourceRepository;
public class StorageManagerInterface implements IStorageManagerInterface {
private static final long serialVersionUID = 1L;
@@ -40,7 +40,7 @@
}
@Override
- public IIndexArtifactMap getIndexArtifactMap(IHyracksTaskContext ctx) {
- return RuntimeContext.get(ctx).getIndexArtifactMap();
+ public ILocalResourceRepository getLocalResourceRepository(IHyracksTaskContext ctx) {
+ return RuntimeContext.get(ctx).getLocalResourceRepository();
}
}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexDataflowHelper.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexDataflowHelper.java
index 9a43ea9..39b4553 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexDataflowHelper.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexDataflowHelper.java
@@ -17,7 +17,7 @@
public FileReference getFileReference();
- public long getResourceID();
+ public long getResourceID() throws HyracksDataException;
public IHyracksTaskContext getTaskContext();
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
index 4d0905e..3cdba2f 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
@@ -23,14 +23,16 @@
import edu.uci.ics.hyracks.storage.am.common.api.IIndex;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexDataflowHelper;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManager;
-import edu.uci.ics.hyracks.storage.common.file.IIndexArtifactMap;
+import edu.uci.ics.hyracks.storage.common.file.ILocalResourceRepository;
+import edu.uci.ics.hyracks.storage.common.file.IndexLocalResource;
+import edu.uci.ics.hyracks.storage.common.file.LSMBTreeLocalResourceClass;
public abstract class IndexDataflowHelper implements IIndexDataflowHelper {
protected final IIndexOperatorDescriptor opDesc;
protected final IHyracksTaskContext ctx;
protected final IIndexLifecycleManager lcManager;
- protected final IIndexArtifactMap indexArtifactMap;
+ protected final ILocalResourceRepository localResourceRepository;
protected final FileReference file;
protected IIndex index;
@@ -39,7 +41,7 @@
this.opDesc = opDesc;
this.ctx = ctx;
this.lcManager = opDesc.getLifecycleManagerProvider().getLifecycleManager(ctx);
- this.indexArtifactMap = opDesc.getStorageManager().getIndexArtifactMap(ctx);
+ this.localResourceRepository = opDesc.getStorageManager().getLocalResourceRepository(ctx);
this.file = opDesc.getFileSplitProvider().getFileSplits()[partition].getLocalFile();
}
@@ -63,11 +65,12 @@
// any physical artifact that the IIndexArtifactMap is managing (e.g. a file containing the resource ID).
// Once the index has been created, a new resource ID can be generated.
if (resourceID != -1) {
- indexArtifactMap.delete(file.getFile().getPath(), ctx.getIOManager().getIODevices());
+ localResourceRepository.deleteResourceByName(file.getFile().getPath());
}
index.create();
try {
- resourceID = indexArtifactMap.create(file.getFile().getPath(), ctx.getIOManager().getIODevices());
+ //TODO Create ResourceIdFactory and LocalResourceFactory
+ localResourceRepository.insert(new IndexLocalResource(resourceID, file.getFile().getPath(), object, resourceClass));
} catch (IOException e) {
throw new HyracksDataException(e);
}
@@ -92,7 +95,7 @@
}
}
- public void close() {
+ public void close() throws HyracksDataException {
synchronized (lcManager) {
lcManager.close(getResourceID());
}
@@ -109,7 +112,7 @@
}
if (resourceID != -1) {
- indexArtifactMap.delete(file.getFile().getPath(), ctx.getIOManager().getIODevices());
+ localResourceRepository.deleteResourceByName(file.getFile().getPath());
}
index.destroy();
}
@@ -119,8 +122,8 @@
return file;
}
- public long getResourceID() {
- return indexArtifactMap.get(file.getFile().getPath());
+ public long getResourceID() throws HyracksDataException {
+ return localResourceRepository.getResourceByName(file.getFile().getPath()).getResourceId();
}
public IHyracksTaskContext getTaskContext() {
diff --git a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/IStorageManagerInterface.java b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/IStorageManagerInterface.java
index aae9d67..9e3616a 100644
--- a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/IStorageManagerInterface.java
+++ b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/IStorageManagerInterface.java
@@ -20,11 +20,12 @@
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
import edu.uci.ics.hyracks.storage.common.file.IIndexArtifactMap;
+import edu.uci.ics.hyracks.storage.common.file.ILocalResourceRepository;
public interface IStorageManagerInterface extends Serializable {
public IBufferCache getBufferCache(IHyracksTaskContext ctx);
public IFileMapProvider getFileMapProvider(IHyracksTaskContext ctx);
- public IIndexArtifactMap getIndexArtifactMap(IHyracksTaskContext ctx);
+ public ILocalResourceRepository getLocalResourceRepository(IHyracksTaskContext ctx);
}
\ No newline at end of file
diff --git a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/ILocalResourceRepositoryFactory.java b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/ILocalResourceRepositoryFactory.java
index 6fd4b2c..4b3124a 100644
--- a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/ILocalResourceRepositoryFactory.java
+++ b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/ILocalResourceRepositoryFactory.java
@@ -14,8 +14,8 @@
*/
package edu.uci.ics.hyracks.storage.common.file;
-import java.util.List;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
public interface ILocalResourceRepositoryFactory {
- public ILocalResourceRepository createRepository();
+ public ILocalResourceRepository createRepository() throws HyracksDataException;
}
diff --git a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/IndexLocalResource.java b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/IndexLocalResource.java
index b5babd5..f5dfca8 100644
--- a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/IndexLocalResource.java
+++ b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/IndexLocalResource.java
@@ -47,5 +47,4 @@
public ILocalResourceClass getResourceClass() {
return resourceClass;
}
-
}
diff --git a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/IndexLocalResourceRepository.java b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/IndexLocalResourceRepository.java
index 6934124..9090aad 100644
--- a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/IndexLocalResourceRepository.java
+++ b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/IndexLocalResourceRepository.java
@@ -33,7 +33,6 @@
private final List<ILocalResourceClass> resourceClasses;
private final List<String> mountPoints;
- private final String rootDir;
private static final String METADATA_FILE_NAME = ".metadata";
private Map<String, ILocalResource> name2ResourceMap = new HashMap<String, ILocalResource>();
private Map<Long, ILocalResource> id2ResourceMap = new HashMap<Long, ILocalResource>();
@@ -42,7 +41,6 @@
String rootDir) throws HyracksDataException {
this.resourceClasses = resourceClasses;
this.mountPoints = mountPoints;
- this.rootDir = rootDir;
File rootFile = new File(this.mountPoints.get(0), rootDir);
if (!rootFile.exists()) {
@@ -59,9 +57,8 @@
}
};
- String[] fileNameList = rootFile.list();
- for (String fileName : fileNameList) {
- File childFile = new File(rootFile, fileName);
+ File[] childFileList = rootFile.listFiles();
+ for (File childFile : childFileList) {
if (childFile.isDirectory()) {
File[] targetFileList = childFile.listFiles(filter);
for (File targetFile : targetFileList) {
diff --git a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/IndexLocalResourceRepositoryFactory.java b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/IndexLocalResourceRepositoryFactory.java
index 4125d85..4707361 100644
--- a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/IndexLocalResourceRepositoryFactory.java
+++ b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/IndexLocalResourceRepositoryFactory.java
@@ -14,12 +14,33 @@
*/
package edu.uci.ics.hyracks.storage.common.file;
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.IIOManager;
+import edu.uci.ics.hyracks.api.io.IODeviceHandle;
+
public class IndexLocalResourceRepositoryFactory implements ILocalResourceRepositoryFactory {
-
- @Override
- public ILocalResourceRepository createRepository() {
- // TODO Auto-generated method stub
- return null;
+ private static final String rootDir = "";
+ private IIOManager ioManager;
+
+ public IndexLocalResourceRepositoryFactory(IIOManager ioManager) {
+ this.ioManager = ioManager;
}
-
+
+ @Override
+ public ILocalResourceRepository createRepository() throws HyracksDataException {
+ List<ILocalResourceClass> resourceClasses = new ArrayList<ILocalResourceClass>();
+ resourceClasses.add(LSMBTreeLocalResourceClass.getInstance());
+ resourceClasses.add(LSMRTreeLocalResourceClass.getInstance());
+ resourceClasses.add(LSMInvertedIndexLocalResourceClass.getInstance());
+
+ List<String> mountPoints = new ArrayList<String>();
+ List<IODeviceHandle> devices = ioManager.getIODevices();
+ for (IODeviceHandle dev : devices) {
+ mountPoints.add(dev.getPath().getPath());
+ }
+ return new IndexLocalResourceRepository(resourceClasses, mountPoints, rootDir);
+ }
}
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerComponentHolder.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerComponentHolder.java
index ea7b453..791eb84 100644
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerComponentHolder.java
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerComponentHolder.java
@@ -25,7 +25,6 @@
import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManager;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexLifecycleManager;
-import edu.uci.ics.hyracks.storage.common.TransientIndexArtifactMap;
import edu.uci.ics.hyracks.storage.common.buffercache.BufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ClockPageReplacementStrategy;
import edu.uci.ics.hyracks.storage.common.buffercache.HeapBufferAllocator;
@@ -34,14 +33,16 @@
import edu.uci.ics.hyracks.storage.common.buffercache.IPageReplacementStrategy;
import edu.uci.ics.hyracks.storage.common.file.IFileMapManager;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-import edu.uci.ics.hyracks.storage.common.file.IIndexArtifactMap;
+import edu.uci.ics.hyracks.storage.common.file.ILocalResourceRepository;
+import edu.uci.ics.hyracks.storage.common.file.ILocalResourceRepositoryFactory;
+import edu.uci.ics.hyracks.storage.common.file.IndexLocalResourceRepositoryFactory;
import edu.uci.ics.hyracks.storage.common.file.TransientFileMapManager;
public class TestStorageManagerComponentHolder {
private static IBufferCache bufferCache;
private static IFileMapProvider fileMapProvider;
private static IOManager ioManager;
- private static IIndexArtifactMap indexArtifactMap;
+ private static ILocalResourceRepository localResourceRepository;
private static IIndexLifecycleManager lcManager;
private static int pageSize;
@@ -54,7 +55,7 @@
TestStorageManagerComponentHolder.maxOpenFiles = maxOpenFiles;
bufferCache = null;
fileMapProvider = null;
- indexArtifactMap = null;
+ localResourceRepository = null;
lcManager = null;
}
@@ -92,10 +93,17 @@
return ioManager;
}
- public synchronized static IIndexArtifactMap getIndexArtifactMap(IHyracksTaskContext ctx) {
- if (indexArtifactMap == null) {
- indexArtifactMap = new TransientIndexArtifactMap();
+ public synchronized static ILocalResourceRepository getLocalResourceRepository(IHyracksTaskContext ctx) {
+ if (localResourceRepository == null) {
+ try {
+ ILocalResourceRepositoryFactory indexLocalResourceRepositoryFactory = new IndexLocalResourceRepositoryFactory(
+ getIOManager());
+ localResourceRepository = indexLocalResourceRepositoryFactory.createRepository();
+ } catch (HyracksException e) {
+ //In order not to change the IStorageManagerInterface due to the test code, throw runtime exception.
+ throw new IllegalArgumentException();
+ }
}
- return indexArtifactMap;
+ return localResourceRepository;
}
}
\ No newline at end of file
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerInterface.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerInterface.java
index 3ee93a4..a01ac97 100644
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerInterface.java
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerInterface.java
@@ -15,10 +15,11 @@
package edu.uci.ics.hyracks.test.support;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-import edu.uci.ics.hyracks.storage.common.file.IIndexArtifactMap;
+import edu.uci.ics.hyracks.storage.common.file.ILocalResourceRepository;
public class TestStorageManagerInterface implements IStorageManagerInterface {
private static final long serialVersionUID = 1L;
@@ -34,7 +35,7 @@
}
@Override
- public IIndexArtifactMap getIndexArtifactMap(IHyracksTaskContext ctx) {
- return TestStorageManagerComponentHolder.getIndexArtifactMap(ctx);
+ public ILocalResourceRepository getLocalResourceRepository(IHyracksTaskContext ctx) {
+ return TestStorageManagerComponentHolder.getLocalResourceRepository(ctx);
}
}
\ No newline at end of file