Finished readthrough and cosmetic cleanup of lsm-common package:
- Applied code formatting profile to all files
- Updated copyrights to reflect the most recent year
- Added copyrights where missing
- Fixed typos in comments
- Added comments where I thought them to be helpful
- Renamed misleading method name
- Organized imports
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_lsm_tree@1238 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/IExperimentRunner.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/IExperimentRunner.java
index 8b9d6f3..94d8d1e 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/IExperimentRunner.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/IExperimentRunner.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2009-2010 by The Regents of the University of California
+ * Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* you may obtain a copy of the License from
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFinalizer.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFinalizer.java
index f241fd6..b88560d 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFinalizer.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFinalizer.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2009-2010 by The Regents of the University of California
+ * Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* you may obtain a copy of the License from
@@ -20,17 +20,16 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
public interface ILSMComponentFinalizer {
-
+
/**
- * @return Checks whether the given file is valid with respect to the given LSM component. Used for guaranteeing
- * atomicity of LSM component writes.
+ * Checks whether the given file is valid with respect to the given LSM component.
+ * Used for guaranteeing the atomicity of LSM component writes.
*/
public boolean isValid(File file, Object lsmComponent) throws HyracksDataException;
-
+
/**
* Marks the given LSM component as physically valid, synchronously forcing
- * the necessary information to disk.
- *
+ * the necessary information to disk.
*/
public void finalize(Object lsmComponent) throws HyracksDataException;
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFileManager.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFileManager.java
index 9150b01..7ba0198 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFileManager.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMFileManager.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2009-2010 by The Regents of the University of California
+ * Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* you may obtain a copy of the License from
@@ -25,31 +25,30 @@
/**
* Provides file names for LSM on-disk components. Also cleans up invalid files.
*
- * There are separate methods to get file names for merge and flush, because we
+ * There are separate methods to get file names for merge and flush because we
* need to guarantee the correct order of on-disk components (i.e., the
* components produced by flush are always newer than those produced by a
* merge).
- *
- *
*/
public interface ILSMFileManager {
- public void createDirs();
-
- public FileReference createFlushFile(String relFlushFileName);
+ public void createDirs();
- public FileReference createMergeFile(String relMergeFileName);
+ public FileReference createFlushFile(String relFlushFileName);
+
+ public FileReference createMergeFile(String relMergeFileName);
public Object getRelFlushFileName();
-
- public Object getRelMergeFileName(String firstFileName, String lastFileName) throws HyracksDataException;
-
- public String getBaseDir();
-
- // Deletes invalid files, and returns list of valid files from baseDir.
- // The returned valid files are correctly sorted (based on the recency of data).
- public List<Object> cleanupAndGetValidFiles(Object lsmComponent, ILSMComponentFinalizer componentFinalizer) throws HyracksDataException;
-
- public Comparator<String> getFileNameComparator();
-
- public IOManager getIOManager();
+
+ public Object getRelMergeFileName(String firstFileName, String lastFileName) throws HyracksDataException;
+
+ public String getBaseDir();
+
+ // Deletes invalid files, and returns list of valid files from baseDir.
+ // The returned valid files are correctly sorted (based on the recency of data).
+ public List<Object> cleanupAndGetValidFiles(Object lsmComponent, ILSMComponentFinalizer componentFinalizer)
+ throws HyracksDataException;
+
+ public Comparator<String> getFileNameComparator();
+
+ public IOManager getIOManager();
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndex.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndex.java
index 362fa7c..0c163eb 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndex.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndex.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2009-2010 by The Regents of the University of California
+ * Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* you may obtain a copy of the License from
@@ -26,22 +26,22 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.freepage.InMemoryFreePageManager;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMHarness;
/**
- * Methods to be implemented by an LSM index, which are called from LSMHarness.
+ * Methods to be implemented by an LSM index, which are called from {@link LSMHarness}.
* The implementations of the methods below should be thread agnostic.
* Synchronization of LSM operations like updates/searches/flushes/merges are
- * done by the LSMHarness. For example, a flush() implementation should only
+ * done by the {@link LSMHarness}. For example, a flush() implementation should only
* create and return the new on-disk component, ignoring the fact that
* concurrent searches/updates/merges may be ongoing.
- *
*/
public interface ILSMIndex extends IIndex {
public boolean insertUpdateOrDelete(ITupleReference tuple, IIndexOpContext ictx) throws HyracksDataException,
IndexException;
- public void search(IIndexCursor cursor, List<Object> diskComponents, ISearchPredicate pred,
- IIndexOpContext ictx, boolean includeMemComponent, AtomicInteger searcherRefCount) throws HyracksDataException, IndexException;
+ public void search(IIndexCursor cursor, List<Object> diskComponents, ISearchPredicate pred, IIndexOpContext ictx,
+ boolean includeMemComponent, AtomicInteger searcherRefCount) throws HyracksDataException, IndexException;
public Object merge(List<Object> mergedComponents) throws HyracksDataException, IndexException;
@@ -58,6 +58,6 @@
public void resetInMemoryComponent() throws HyracksDataException;
public List<Object> getDiskComponents();
-
+
public ILSMComponentFinalizer getComponentFinalizer();
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
index e762bb5..a688b5b 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2009-2010 by The Regents of the University of California
+ * Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* you may obtain a copy of the License from
@@ -22,19 +22,19 @@
/**
* Client handle for performing operations
- * (insert/delete/update/search/diskorderscan/merge/flush) on an ILSMTree. An
- * ILSMTreeIndexAccessor is not thread safe, but different ILSMTreeIndexAccessor
- * can concurrently operate on the same ILSMTree (i.e., the ILSMTree must allow
+ * (insert/delete/update/search/diskorderscan/merge/flush) on an {@link ILSMIndex}.
+ * An {@link ILSMIndexAccessor} is not thread safe, but different {@link ILSMIndexAccessor}s
+ * can concurrently operate on the same {@link ILSMIndex} (i.e., the {@link ILSMIndex} must allow
* concurrent operations).
*/
public interface ILSMIndexAccessor extends IIndexAccessor {
- /**
- * Force a flush of the in-memory component.
- *
- * @throws HyracksDataException
- * @throws TreeIndexException
- */
- public void flush() throws HyracksDataException, IndexException;
+ /**
+ * Force a flush of the in-memory component.
+ *
+ * @throws HyracksDataException
+ * @throws TreeIndexException
+ */
+ public void flush() throws HyracksDataException, IndexException;
/**
* Merge all on-disk components.
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryBufferCache.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryBufferCache.java
index 303369e..f554778 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryBufferCache.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryBufferCache.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2009-2010 by The Regents of the University of California
+ * Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* you may obtain a copy of the License from
@@ -34,45 +34,45 @@
protected final int pageSize;
protected final CachedPage[] pages;
protected final List<CachedPage> overflowPages = new ArrayList<CachedPage>();
-
- public InMemoryBufferCache(ICacheMemoryAllocator allocator, int pageSize, int numPages){
+
+ public InMemoryBufferCache(ICacheMemoryAllocator allocator, int pageSize, int numPages) {
this.allocator = allocator;
- this.pageSize = pageSize;
- ByteBuffer[] buffers = allocator.allocate(pageSize, numPages);
- pages = new CachedPage[buffers.length];
+ this.pageSize = pageSize;
+ ByteBuffer[] buffers = allocator.allocate(pageSize, numPages);
+ pages = new CachedPage[buffers.length];
for (int i = 0; i < buffers.length; ++i) {
pages[i] = new CachedPage(i, buffers[i]);
}
- }
+ }
- @Override
- public ICachedPage pin(long dpid, boolean newPage) {
- int pageId = BufferedFileHandle.getPageId(dpid);
- if (pageId < pages.length) {
- // Common case: Return regular page.
- return pages[pageId];
- } else {
- // Rare case: Return overflow page, possibly expanding overflow array.
- synchronized(overflowPages) {
- int numNewPages = pageId - pages.length - overflowPages.size() + 1;
- if (numNewPages > 0) {
- ByteBuffer[] buffers = allocator.allocate(pageSize, numNewPages);
- for (int i = 0; i < numNewPages; i++) {
- CachedPage overflowPage = new CachedPage(pages.length + overflowPages.size(), buffers[i]);
- overflowPages.add(overflowPage);
- }
- }
- return overflowPages.get(pageId - pages.length);
- }
- }
- }
+ @Override
+ public ICachedPage pin(long dpid, boolean newPage) {
+ int pageId = BufferedFileHandle.getPageId(dpid);
+ if (pageId < pages.length) {
+ // Common case: Return regular page.
+ return pages[pageId];
+ } else {
+ // Rare case: Return overflow page, possibly expanding overflow array.
+ synchronized (overflowPages) {
+ int numNewPages = pageId - pages.length - overflowPages.size() + 1;
+ if (numNewPages > 0) {
+ ByteBuffer[] buffers = allocator.allocate(pageSize, numNewPages);
+ for (int i = 0; i < numNewPages; i++) {
+ CachedPage overflowPage = new CachedPage(pages.length + overflowPages.size(), buffers[i]);
+ overflowPages.add(overflowPage);
+ }
+ }
+ return overflowPages.get(pageId - pages.length);
+ }
+ }
+ }
- @Override
+ @Override
public ICachedPage tryPin(long dpid) throws HyracksDataException {
return pin(dpid, false);
}
-
- @Override
+
+ @Override
public int getPageSize() {
return pageSize;
}
@@ -86,12 +86,12 @@
public ICachedPageInternal getPage(int cpid) {
return pages[cpid];
}
-
+
public int getNumOverflowPages() {
return overflowPages.size();
}
-
- @Override
+
+ @Override
public void createFile(FileReference fileRef) throws HyracksDataException {
// Do nothing.
}
@@ -110,17 +110,17 @@
public void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException {
// Do nothing.
}
-
- @Override
- public void unpin(ICachedPage page) throws HyracksDataException {
- // Do Nothing.
- }
- @Override
- public void close() {
- // Do nothing.
- }
-
+ @Override
+ public void unpin(ICachedPage page) throws HyracksDataException {
+ // Do Nothing.
+ }
+
+ @Override
+ public void close() {
+ // Do nothing.
+ }
+
public class CachedPage implements ICachedPageInternal {
private final int cpid;
private final ByteBuffer buffer;
@@ -139,14 +139,14 @@
@Override
public Object getReplacementStrategyObject() {
- // Do nothing.
+ // Do nothing.
return null;
}
@Override
public boolean pinIfGoodVictim() {
- // Do nothing.
- return false;
+ // Do nothing.
+ return false;
}
@Override
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryFreePageManager.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryFreePageManager.java
index dfd437b..0b8c32d 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryFreePageManager.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/freepage/InMemoryFreePageManager.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2009-2010 by The Regents of the University of California
+ * Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* you may obtain a copy of the License from
@@ -38,7 +38,7 @@
@Override
public int getFreePage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
- // The very call returns page id 2 because the BTree uses
+ // The very first call returns page id 2 because the BTree uses
// the first page as metadata page, and the second page as root page.
return currentPageId.incrementAndGet();
}
@@ -61,7 +61,7 @@
public int getCapacity() {
return capacity - 2;
}
-
+
public void reset() {
currentPageId.set(1);
}
@@ -76,7 +76,7 @@
@Override
public byte getMetaPageLevelIndicator() {
- return 0;
+ return 0;
}
@Override
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BTreeFactory.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
index c5a6a1a..b51b84b 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2009-2010 by The Regents of the University of California
+ * Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* you may obtain a copy of the License from
@@ -23,8 +23,9 @@
public class BTreeFactory extends TreeFactory<BTree> {
- public BTreeFactory(IBufferCache bufferCache, LinkedListFreePageManagerFactory freePageManagerFactory, IBinaryComparatorFactory[] cmpFactories,
- int fieldCount, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
+ public BTreeFactory(IBufferCache bufferCache, LinkedListFreePageManagerFactory freePageManagerFactory,
+ IBinaryComparatorFactory[] cmpFactories, int fieldCount, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory leafFrameFactory) {
super(bufferCache, freePageManagerFactory, cmpFactories, fieldCount, interiorFrameFactory, leafFrameFactory);
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 846bacf..b3e5b2b 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2009-2010 by The Regents of the University of California
+ * Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* you may obtain a copy of the License from
@@ -32,28 +32,23 @@
/**
* Common code for synchronizing LSM operations like
- * updates/searches/flushes/merges on any ILSMTree. This class only deals with
+ * updates/searches/flushes/merges on any {@link ILSMIndex}. This class only deals with
* synchronizing LSM operations, and delegates the concrete implementations of
- * actual operations to ILSMTree (passed in the c'tor).
- *
+ * actual operations to {@link ILSMIndex} (passed in the constructor).
* Concurrency behavior:
- *
* All operations except merge (insert/update/delete/search) are blocked during a flush.
- *
* During a merge, all operations (except another merge) can proceed concurrently.
- *
* A merge and a flush can proceed concurrently.
- *
*/
public class LSMHarness {
- protected final Logger LOGGER = Logger.getLogger(LSMHarness.class.getName());
- protected static final long AFTER_MERGE_CLEANUP_SLEEP = 100;
-
- private ILSMIndex lsmIndex;
-
- // All accesses to the LSM-Tree's on-disk components are synchronized on diskComponentsSync.
- private Object diskComponentsSync = new Object();
-
+ protected final Logger LOGGER = Logger.getLogger(LSMHarness.class.getName());
+ protected static final long AFTER_MERGE_CLEANUP_SLEEP = 100;
+
+ private ILSMIndex lsmIndex;
+
+ // All accesses to the LSM-Tree's on-disk components are synchronized on diskComponentsSync.
+ private Object diskComponentsSync = new Object();
+
// For synchronizing all operations with flushes.
// Currently, all operations block during a flush.
private int threadRefCount;
@@ -63,13 +58,14 @@
private AtomicBoolean isMerging = new AtomicBoolean(false);
private AtomicInteger searcherRefCountA = new AtomicInteger(0);
private AtomicInteger searcherRefCountB = new AtomicInteger(0);
+
// Represents the current number of searcher threads that are operating on
// the unmerged on-disk Trees.
// We alternate between searcherRefCountA and searcherRefCountB.
private AtomicInteger searcherRefCount = searcherRefCountA;
-
+
public LSMHarness(ILSMIndex lsmIndex) {
- this.lsmIndex = lsmIndex;
+ this.lsmIndex = lsmIndex;
this.threadRefCount = 0;
this.flushFlag = false;
}
@@ -77,14 +73,16 @@
public void threadEnter() {
threadRefCount++;
}
-
+
public void threadExit() throws HyracksDataException, IndexException {
synchronized (this) {
threadRefCount--;
+
// Check if we've reached or exceeded the maximum number of pages.
if (!flushFlag && lsmIndex.getInMemoryFreePageManager().isFull()) {
flushFlag = true;
}
+
// Flush will only be handled by last exiting thread.
if (flushFlag && threadRefCount == 0) {
flush();
@@ -92,61 +90,74 @@
}
}
}
-
- public void insertUpdateOrDelete(ITupleReference tuple, IIndexOpContext ctx) throws HyracksDataException, IndexException {
- boolean waitForFlush = true;
- do {
- // Wait for ongoing flush to complete.
- synchronized (this) {
- if (!flushFlag) {
- // Increments threadRefCount, to force a flush to wait for this operation to finish.
- // (a flush can only begin once threadRefCount == 0).
- threadEnter();
- // Proceed with operation.
- waitForFlush = false;
- }
- }
- } while (waitForFlush);
-
- boolean operationComplete = true;
- try {
- do {
- operationComplete = lsmIndex.insertUpdateOrDelete(tuple, ctx);
- } while (!operationComplete);
- } finally {
- threadExit();
- }
- }
+
+ public void insertUpdateOrDelete(ITupleReference tuple, IIndexOpContext ctx) throws HyracksDataException,
+ IndexException {
+ boolean waitForFlush = true;
+ do {
+ synchronized (this) {
+ // flushFlag may be set to true even though the flush has not occurred yet.
+ // If flushFlag is set, then the flush is queued to occur by the last exiting thread.
+ // This operation should wait for that flush to occur before proceeding.
+ if (!flushFlag) {
+ // Increment the threadRefCount in order to block the possibility of a concurrent flush.
+ // The corresponding threadExit() call is in LSMTreeRangeSearchCursor.close()
+ threadEnter();
+
+ // A flush is not pending, so proceed with the operation.
+ waitForFlush = false;
+ }
+ }
+ } while (waitForFlush);
+
+ // It is possible, due to concurrent execution of operations, that an operation will
+ // fail. In such a case, simply retry the operation. Refer to the specific LSMIndex code
+ // to see exactly why an operation might fail.
+ boolean operationComplete = true;
+ try {
+ do {
+ operationComplete = lsmIndex.insertUpdateOrDelete(tuple, ctx);
+ } while (!operationComplete);
+ } finally {
+ threadExit();
+ }
+ }
public void flush() throws HyracksDataException, IndexException {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Flushing LSM-Tree.");
}
Object newComponent = lsmIndex.flush();
-
+
// The implementation of this call must take any necessary steps to make
// the new component permanent, and mark it as valid (usually this means
// forcing all pages of the tree to disk, possibly with some extra
// information to mark the tree as valid).
lsmIndex.getComponentFinalizer().finalize(newComponent);
-
+
lsmIndex.resetInMemoryComponent();
synchronized (diskComponentsSync) {
lsmIndex.addFlushedComponent(newComponent);
}
}
-
- public List<Object> search(IIndexCursor cursor, ISearchPredicate pred, IIndexOpContext ctx, boolean includeMemComponent) throws HyracksDataException, IndexException {
+
+ public List<Object> search(IIndexCursor cursor, ISearchPredicate pred, IIndexOpContext ctx,
+ boolean includeMemComponent) throws HyracksDataException, IndexException {
// If the search doesn't include the in-memory component, then we don't have
// to synchronize with a flush.
if (includeMemComponent) {
boolean waitForFlush = true;
do {
synchronized (this) {
+ // flushFlag may be set to true even though the flush has not occurred yet.
+ // If flushFlag is set, then the flush is queued to occur by the last exiting thread.
+ // This operation should wait for that flush to occur before proceeding.
if (!flushFlag) {
- // The corresponding threadExit() is in
- // LSMTreeRangeSearchCursor.close().
+ // Increment the threadRefCount in order to block the possibility of a concurrent flush.
+ // The corresponding threadExit() call is in LSMTreeRangeSearchCursor.close()
threadEnter();
+
+ // A flush is not pending, so proceed with the operation.
waitForFlush = false;
}
}
@@ -165,47 +176,49 @@
synchronized (diskComponentsSync) {
diskComponentSnapshot.addAll(lsmIndex.getDiskComponents());
localSearcherRefCount = searcherRefCount;
- localSearcherRefCount.incrementAndGet();
+ localSearcherRefCount.incrementAndGet();
}
-
+
lsmIndex.search(cursor, diskComponentSnapshot, pred, ctx, includeMemComponent, localSearcherRefCount);
return diskComponentSnapshot;
}
- public void merge() throws HyracksDataException, IndexException {
+ public void merge() throws HyracksDataException, IndexException {
if (!isMerging.compareAndSet(false, true)) {
- throw new LSMMergeInProgressException("Merge already in progress in LSMTree. Only one concurrent merge allowed.");
+ throw new LSMMergeInProgressException(
+ "Merge already in progress. Only one merge process allowed at a time.");
}
-
+
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Merging LSM-Tree.");
}
-
+
// Point to the current searcher ref count, so we can wait for it later
// (after we swap the searcher ref count).
AtomicInteger localSearcherRefCount = searcherRefCount;
-
+
List<Object> mergedComponents = new ArrayList<Object>();
Object newComponent = lsmIndex.merge(mergedComponents);
+
// No merge happened.
if (newComponent == null) {
isMerging.set(false);
return;
}
-
+
// Remove the old Trees from the list, and add the new merged Tree(s).
// Also, swap the searchRefCount.
synchronized (diskComponentsSync) {
- lsmIndex.addMergedComponent(newComponent, mergedComponents);
+ lsmIndex.addMergedComponent(newComponent, mergedComponents);
// Swap the searcher ref count reference, and reset it to zero.
- if (searcherRefCount == searcherRefCountA) {
+ if (searcherRefCount == searcherRefCountA) {
searcherRefCount = searcherRefCountB;
} else {
searcherRefCount = searcherRefCountA;
}
searcherRefCount.set(0);
}
-
+
// Wait for all searchers that are still accessing the old on-disk
// Trees, then perform the final cleanup of the old Trees.
while (localSearcherRefCount.get() > 0) {
@@ -217,20 +230,21 @@
throw new HyracksDataException(e);
}
}
-
+
// The implementation of this call must take any necessary steps to make
// the new component permanent, and mark it as valid (usually this means
// forcing all pages of the tree to disk, possibly with some extra
// information to mark the tree as valid).
lsmIndex.getComponentFinalizer().finalize(newComponent);
-
+
// Cleanup. At this point we have guaranteed that no searchers are
// touching the old on-disk Trees (localSearcherRefCount == 0).
lsmIndex.cleanUpAfterMerge(mergedComponents);
isMerging.set(false);
}
-
- public void closeSearchCursor(AtomicInteger searcherRefCount, boolean includeMemComponent) throws HyracksDataException {
+
+ public void closeSearchCursor(AtomicInteger searcherRefCount, boolean includeMemComponent)
+ throws HyracksDataException {
// If the in-memory Tree was not included in the search, then we don't
// need to synchronize with a flush.
if (includeMemComponent) {
@@ -240,10 +254,12 @@
throw new HyracksDataException(e);
}
}
- // Synchronize with ongoing merges.
+ // A merge may be waiting on this searcher to finish searching the on-disk components.
+ // Decrement the searcherRefCount so that the merge process is able to cleanup any old
+ // on-disk components.
searcherRefCount.decrementAndGet();
}
-
+
public void addBulkLoadedComponent(Object index) throws HyracksDataException {
// The implementation of this call must take any necessary steps to make
// the new component permanent, and mark it as valid (usually this means
@@ -251,7 +267,7 @@
// information to mark the tree as valid).
lsmIndex.getComponentFinalizer().finalize(index);
synchronized (diskComponentsSync) {
- lsmIndex.addFlushedComponent(index);
- }
+ lsmIndex.addFlushedComponent(index);
+ }
}
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMMergeInProgressException.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMMergeInProgressException.java
index 9834fae..ee239d3 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMMergeInProgressException.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMMergeInProgressException.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2009-2010 by The Regents of the University of California
+ * Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* you may obtain a copy of the License from
@@ -20,8 +20,8 @@
public class LSMMergeInProgressException extends TreeIndexException {
private static final long serialVersionUID = 1L;
-
- public LSMMergeInProgressException(Exception e) {
+
+ public LSMMergeInProgressException(Exception e) {
super(e);
}
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java
index bafdbf0..0334a8e 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeFileManager.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2009-2010 by The Regents of the University of California
+ * Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* you may obtain a copy of the License from
@@ -36,24 +36,27 @@
public class LSMTreeFileManager implements ILSMFileManager {
protected static final String SPLIT_STRING = "_";
-
- // Currently uses all IODevices registered in ioManager in a round-robin fashion.
+
+ // Use all IODevices registered in ioManager in a round-robin fashion to choose
+ // where to flush and merge
protected final IOManager ioManager;
- protected final IFileMapProvider fileMapProvider;
- // baseDir should reflect dataset name, and partition name.
+ protected final IFileMapProvider fileMapProvider;
+
+ // baseDir should reflect dataset name and partition name.
protected final String baseDir;
- protected final Format formatter = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS");
+ protected final Format formatter = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS");
protected final Comparator<String> cmp = new FileNameComparator();
protected final Comparator<ComparableFileName> recencyCmp = new RecencyComparator();
- // To implement round-robin assignment of files onto I/O devices.
+
+ // The current index for the round-robin file assignment
private int ioDeviceIndex = 0;
-
+
private static FilenameFilter fileNameFilter = new FilenameFilter() {
public boolean accept(File dir, String name) {
return !name.startsWith(".");
}
};
-
+
public LSMTreeFileManager(IOManager ioManager, IFileMapProvider fileMapProvider, String baseDir) {
if (!baseDir.endsWith(System.getProperty("file.separator"))) {
baseDir += System.getProperty("file.separator");
@@ -63,31 +66,31 @@
this.baseDir = baseDir;
createDirs();
}
-
+
@Override
public void createDirs() {
- for(IODeviceHandle dev : ioManager.getIODevices()) {
+ for (IODeviceHandle dev : ioManager.getIODevices()) {
File f = new File(dev.getPath(), baseDir);
f.mkdirs();
}
}
-
- public FileReference createFlushFile(String relFlushFileName) {
+
+ public FileReference createFlushFile(String relFlushFileName) {
// Assigns new files to I/O devices in round-robin fashion.
IODeviceHandle dev = ioManager.getIODevices().get(ioDeviceIndex);
ioDeviceIndex = (ioDeviceIndex + 1) % ioManager.getIODevices().size();
return dev.createFileReference(relFlushFileName);
}
-
+
public FileReference createMergeFile(String relMergeFileName) {
return createFlushFile(relMergeFileName);
}
-
+
@Override
public Object getRelFlushFileName() {
Date date = new Date();
String ts = formatter.format(date);
- // Begin timestamp and end timestamp are identical.
+ // Begin timestamp and end timestamp are identical since it is a flush
return baseDir + ts + SPLIT_STRING + ts;
}
@@ -95,7 +98,7 @@
public Object getRelMergeFileName(String firstFileName, String lastFileName) throws HyracksDataException {
String[] firstTimestampRange = firstFileName.split(SPLIT_STRING);
String[] lastTimestampRange = lastFileName.split(SPLIT_STRING);
- // Enclosing timestamp range.
+ // Get the range of timestamps by taking the earliest and the latest timestamps
return baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1];
}
@@ -107,12 +110,9 @@
/**
* Sorts strings in reverse lexicographical order. The way we construct the
* file names above guarantees that:
- *
- * 1. Flushed files (sort lower than merged files
- *
+ * 1. Flushed files sort lower than merged files
* 2. Flushed files are sorted from newest to oldest (based on the timestamp
* string)
- *
*/
private class FileNameComparator implements Comparator<String> {
@Override
@@ -127,7 +127,7 @@
return baseDir;
}
- protected void getValidFiles(IODeviceHandle dev, FilenameFilter filter, Object lsmComponent,
+ protected void cleanupAndGetValidFilesInternal(IODeviceHandle dev, FilenameFilter filter, Object lsmComponent,
ILSMComponentFinalizer componentFinalizer, ArrayList<ComparableFileName> allFiles)
throws HyracksDataException {
File dir = new File(dev.getPath(), baseDir);
@@ -141,62 +141,75 @@
}
}
}
-
+
@Override
- public List<Object> cleanupAndGetValidFiles(Object lsmComponent, ILSMComponentFinalizer componentFinalizer) throws HyracksDataException {
+ public List<Object> cleanupAndGetValidFiles(Object lsmComponent, ILSMComponentFinalizer componentFinalizer)
+ throws HyracksDataException {
List<Object> validFiles = new ArrayList<Object>();
ArrayList<ComparableFileName> allFiles = new ArrayList<ComparableFileName>();
- // Gather files from all IODeviceHandles.
- for(IODeviceHandle dev : ioManager.getIODevices()) {
- getValidFiles(dev, fileNameFilter, lsmComponent, componentFinalizer, allFiles);
+
+ // Gather files from all IODeviceHandles and delete invalid files
+ // There are two types of invalid files:
+ // (1) The isValid flag is not set
+ // (2) The file's interval is contained by some other file
+ // Here, we only filter out (1).
+ for (IODeviceHandle dev : ioManager.getIODevices()) {
+ cleanupAndGetValidFilesInternal(dev, fileNameFilter, lsmComponent, componentFinalizer, allFiles);
}
- // Trivial cases.
+
if (allFiles.isEmpty()) {
return validFiles;
}
+
if (allFiles.size() == 1) {
validFiles.add(allFiles.get(0).fullPath);
return validFiles;
}
-
+
// Sorts files names from earliest to latest timestamp.
Collections.sort(allFiles);
-
+
List<ComparableFileName> validComparableFiles = new ArrayList<ComparableFileName>();
ComparableFileName last = allFiles.get(0);
validComparableFiles.add(last);
for (int i = 1; i < allFiles.size(); i++) {
ComparableFileName current = allFiles.get(i);
- // Current start timestamp is greater than last stop timestamp.
+ // The current start timestamp is greater than last stop timestamp so current is valid.
if (current.interval[0].compareTo(last.interval[1]) > 0) {
validComparableFiles.add(current);
- last = current;
- } else if (current.interval[0].compareTo(last.interval[0]) >= 0
+ last = current;
+ } else if (current.interval[0].compareTo(last.interval[0]) >= 0
&& current.interval[1].compareTo(last.interval[1]) <= 0) {
- // Invalid files are completely contained in last interval.
+ // The current file is completely contained in the interval of the
+ // last file. Thus the last file must contain at least as much information
+ // as the current file, so delete the current file.
File invalidFile = new File(current.fullPath);
invalidFile.delete();
} else {
- // This scenario should not be possible.
- throw new HyracksDataException("Found LSM files with overlapping but not contained timetamp intervals.");
+ // This scenario should not be possible since timestamps are monotonically increasing.
+ throw new HyracksDataException("Found LSM files with overlapping timestamp intervals, "
+ + "but the intervals were not contained by another file.");
}
}
+
// Sort valid files in reverse lexicographical order, such that newer files come first.
Collections.sort(validComparableFiles, recencyCmp);
for (ComparableFileName cmpFileName : validComparableFiles) {
validFiles.add(cmpFileName.fullPath);
}
+
return validFiles;
}
-
+
protected class ComparableFileName implements Comparable<ComparableFileName> {
- public final String fullPath;
+ public final String fullPath;
public final String fileName;
+
// Timestamp interval.
public final String[] interval;
-
+
public ComparableFileName(String fullPath) {
- this.fullPath = fullPath;
+ this.fullPath = fullPath;
File f = new File(fullPath);
this.fileName = f.getName();
interval = fileName.split(SPLIT_STRING);
@@ -211,7 +224,7 @@
return b.interval[1].compareTo(interval[1]);
}
}
-
+
private class RecencyComparator implements Comparator<ComparableFileName> {
@Override
public int compare(ComparableFileName a, ComparableFileName b) {
@@ -222,7 +235,7 @@
return -a.interval[1].compareTo(b.interval[1]);
}
}
-
+
@Override
public IOManager getIOManager() {
return ioManager;
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
index 5df3c21..b132337 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
@@ -10,50 +25,46 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
public abstract class LSMTreeIndexAccessor implements ILSMIndexAccessor {
- protected LSMHarness lsmHarness;
- protected IIndexOpContext ctx;
+ protected LSMHarness lsmHarness;
+ protected IIndexOpContext ctx;
- public LSMTreeIndexAccessor(LSMHarness lsmHarness, IIndexOpContext ctx) {
- this.lsmHarness = lsmHarness;
- this.ctx = ctx;
- }
+ public LSMTreeIndexAccessor(LSMHarness lsmHarness, IIndexOpContext ctx) {
+ this.lsmHarness = lsmHarness;
+ this.ctx = ctx;
+ }
- @Override
- public void insert(ITupleReference tuple) throws HyracksDataException,
- IndexException {
- ctx.reset(IndexOp.INSERT);
- lsmHarness.insertUpdateOrDelete(tuple, ctx);
- }
+ @Override
+ public void insert(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.reset(IndexOp.INSERT);
+ lsmHarness.insertUpdateOrDelete(tuple, ctx);
+ }
- @Override
- public void update(ITupleReference tuple) throws HyracksDataException,
- IndexException {
- // Update is the same as insert.
- ctx.reset(IndexOp.INSERT);
- lsmHarness.insertUpdateOrDelete(tuple, ctx);
- }
+ @Override
+ public void update(ITupleReference tuple) throws HyracksDataException, IndexException {
+ // Update is the same as insert.
+ ctx.reset(IndexOp.INSERT);
+ lsmHarness.insertUpdateOrDelete(tuple, ctx);
+ }
- @Override
- public void delete(ITupleReference tuple) throws HyracksDataException,
- IndexException {
- ctx.reset(IndexOp.DELETE);
- lsmHarness.insertUpdateOrDelete(tuple, ctx);
- }
-
- @Override
- public void search(IIndexCursor cursor, ISearchPredicate searchPred)
- throws HyracksDataException, IndexException {
- ctx.reset(IndexOp.SEARCH);
- lsmHarness.search(cursor, searchPred, ctx, true);
- }
+ @Override
+ public void delete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.reset(IndexOp.DELETE);
+ lsmHarness.insertUpdateOrDelete(tuple, ctx);
+ }
- @Override
- public void flush() throws HyracksDataException, IndexException {
- lsmHarness.flush();
- }
+ @Override
+ public void search(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException, IndexException {
+ ctx.reset(IndexOp.SEARCH);
+ lsmHarness.search(cursor, searchPred, ctx, true);
+ }
- @Override
- public void merge() throws HyracksDataException, IndexException {
- lsmHarness.merge();
- }
+ @Override
+ public void flush() throws HyracksDataException, IndexException {
+ lsmHarness.flush();
+ }
+
+ @Override
+ public void merge() throws HyracksDataException, IndexException {
+ lsmHarness.merge();
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java
index a6aa7f9..d5b5650 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeFactory.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2009-2010 by The Regents of the University of California
+ * Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* you may obtain a copy of the License from
@@ -21,7 +21,7 @@
import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-public abstract class TreeFactory <T extends ITreeIndex> {
+public abstract class TreeFactory<T extends ITreeIndex> {
protected IBufferCache bufferCache;
protected int fieldCount;
@@ -30,8 +30,9 @@
protected ITreeIndexFrameFactory leafFrameFactory;
protected LinkedListFreePageManagerFactory freePageManagerFactory;
- public TreeFactory(IBufferCache bufferCache, LinkedListFreePageManagerFactory freePageManagerFactory, IBinaryComparatorFactory[] cmpFactories,
- int fieldCount, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
+ public TreeFactory(IBufferCache bufferCache, LinkedListFreePageManagerFactory freePageManagerFactory,
+ IBinaryComparatorFactory[] cmpFactories, int fieldCount, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory leafFrameFactory) {
this.bufferCache = bufferCache;
this.fieldCount = fieldCount;
this.cmpFactories = cmpFactories;
diff --git a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeIndexComponentFinalizer.java b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeIndexComponentFinalizer.java
index 0f5c2ff..34287cb 100644
--- a/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeIndexComponentFinalizer.java
+++ b/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/TreeIndexComponentFinalizer.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2009-2010 by The Regents of the University of California
+ * Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* you may obtain a copy of the License from
@@ -28,13 +28,13 @@
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class TreeIndexComponentFinalizer implements ILSMComponentFinalizer {
-
+
protected final IFileMapProvider fileMapProvider;
-
+
public TreeIndexComponentFinalizer(IFileMapProvider fileMapProvider) {
this.fileMapProvider = fileMapProvider;
}
-
+
@Override
public boolean isValid(File file, Object lsmComponent) throws HyracksDataException {
ITreeIndex treeIndex = (ITreeIndex) lsmComponent;
@@ -46,8 +46,10 @@
treeIndex.open(fileId);
try {
int metadataPage = treeIndex.getFreePageManager().getFirstMetadataPage();
- ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory().createFrame();
- ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(treeIndex.getFileId(), metadataPage), false);
+ ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory()
+ .createFrame();
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(treeIndex.getFileId(), metadataPage),
+ false);
page.acquireReadLatch();
try {
metadataFrame.setPage(page);
@@ -68,34 +70,42 @@
ITreeIndex treeIndex = (ITreeIndex) lsmComponent;
int fileId = treeIndex.getFileId();
IBufferCache bufferCache = treeIndex.getBufferCache();
+
// Flush all dirty pages of the tree.
- // By default, metadata and data are flushed async in the buffercache.
+ // By default, metadata and data are flushed asynchronously in the buffercache.
// This means that the flush issues writes to the OS, but the data may still lie in filesystem buffers.
- ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory().createFrame();
+ ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory().createFrame();
int startPage = 0;
int maxPage = treeIndex.getFreePageManager().getMaxPage(metadataFrame);
for (int i = startPage; i <= maxPage; i++) {
ICachedPage page = bufferCache.tryPin(BufferedFileHandle.getDiskPageId(fileId, i));
+
// If tryPin returns null, it means the page is not cached, and therefore cannot be dirty.
if (page == null) {
continue;
}
+
try {
bufferCache.flushDirtyPage(page);
} finally {
bufferCache.unpin(page);
}
}
+
// Forces all pages of given file to disk. This guarantees the data makes it to disk.
bufferCache.force(fileId, true);
+
+ // Mark the component as a valid component by flushing the metadata page to disk
int metadataPageId = treeIndex.getFreePageManager().getFirstMetadataPage();
ICachedPage metadataPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, metadataPageId), false);
metadataPage.acquireWriteLatch();
try {
metadataFrame.setPage(metadataPage);
metadataFrame.setValid(true);
+
// Flush the single modified page to disk.
bufferCache.flushDirtyPage(metadataPage);
+
// Force modified metadata page to disk.
bufferCache.force(fileId, true);
} finally {
diff --git a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
index fe28a50..2ee3788 100644
--- a/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
+++ b/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
@@ -75,7 +75,7 @@
// Gather files from all IODeviceHandles.
for (IODeviceHandle dev : ioManager.getIODevices()) {
- getValidFiles(dev, btreeFilter, component.getBTree(), componentFinalizer, allBTreeFiles);
+ cleanupAndGetValidFilesInternal(dev, btreeFilter, component.getBTree(), componentFinalizer, allBTreeFiles);
HashSet<String> btreeFilesSet = new HashSet<String>();
for (ComparableFileName cmpFileName : allBTreeFiles) {
int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
@@ -83,7 +83,7 @@
}
// List of valid RTree files that may or may not have a BTree buddy. Will check for buddies below.
ArrayList<ComparableFileName> tmpAllRTreeFiles = new ArrayList<ComparableFileName>();
- getValidFiles(dev, rtreeFilter, component.getRTree(), componentFinalizer, tmpAllRTreeFiles);
+ cleanupAndGetValidFilesInternal(dev, rtreeFilter, component.getRTree(), componentFinalizer, tmpAllRTreeFiles);
// Look for buddy BTrees for all valid RTrees.
// If no buddy is found, delete the file, otherwise add the RTree to allRTreeFiles.
for (ComparableFileName cmpFileName : tmpAllRTreeFiles) {