[NO ISSUE][STO] Ensure Index Component Sequence Uniqueness
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Currently, if two or more flushes are scheduled before any
of them create its files on disk, we will end up reusing
the same component sequence since the last used component
sequence is based on the files on disk. This change ensures
that the last use component sequence is only initialized
from the disk files and after that the last used component
sequence is incremented in memory.
Change-Id: I478ca3e2c9a98bab14a9145cc8b23eadbd0eab08
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2958
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Reviewed-by: Till Westmann <tillw@apache.org>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
index 1f481c9..904029b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
@@ -81,12 +81,13 @@
(dir, name) -> !name.startsWith(".") && name.endsWith(BLOOM_FILTER_SUFFIX);
protected static final Comparator<String> cmp = new FileNameComparator();
private static final FilenameFilter dummyFilter = (dir, name) -> true;
-
+ private static final long UNINITALIZED_COMPONENT_SEQ = -1;
protected final IIOManager ioManager;
// baseDir should reflect dataset name and partition name and be absolute
protected final FileReference baseDir;
protected final Comparator<IndexComponentFileReference> recencyCmp = new RecencyComparator();
protected final TreeIndexFactory<? extends ITreeIndex> treeFactory;
+ private long lastUsedComponentSeq = UNINITALIZED_COMPONENT_SEQ;
public AbstractLSMIndexFileManager(IIOManager ioManager, FileReference file,
TreeIndexFactory<? extends ITreeIndex> treeFactory) {
@@ -355,11 +356,18 @@
}
protected String getNextComponentSequence(FilenameFilter filenameFilter) throws HyracksDataException {
+ if (lastUsedComponentSeq == UNINITALIZED_COMPONENT_SEQ) {
+ lastUsedComponentSeq = getOnDiskLastUsedComponentSequence(filenameFilter);
+ }
+ return IndexComponentFileReference.getFlushSequence(++lastUsedComponentSeq);
+ }
+
+ private long getOnDiskLastUsedComponentSequence(FilenameFilter filenameFilter) throws HyracksDataException {
long maxComponentSeq = -1;
final String[] files = listDirFiles(baseDir, filenameFilter);
for (String fileName : files) {
maxComponentSeq = Math.max(maxComponentSeq, IndexComponentFileReference.of(fileName).getSequenceEnd());
}
- return IndexComponentFileReference.getFlushSequence(maxComponentSeq + 1);
+ return maxComponentSeq;
}
}