Exclude Temporary Indexes From Replication
- Exclude non-durable LSM indexes from replication.
- Stop heartbeat task after NC has stopped to avoid false failures detection.
Change-Id: Icce91a203e04cb068a7a5aa541720bbd0289eacb
Reviewed-on: https://asterix-gerrit.ics.uci.edu/777
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
index 7b5758c..598d6db 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
@@ -338,12 +338,17 @@
}
partitionManager.close();
datasetPartitionManager.close();
- heartbeatTask.cancel();
netManager.stop();
datasetNetworkManager.stop();
queue.stop();
- if (ncAppEntryPoint != null)
+ if (ncAppEntryPoint != null) {
ncAppEntryPoint.stop();
+ }
+ /**
+ * Stop heartbeat after NC has stopped to avoid false node failure detection
+ * on CC if an NC takes a long time to stop.
+ */
+ heartbeatTask.cancel();
LOGGER.log(Level.INFO, "Stopped NodeControllerService");
shuttedDown = true;
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
index c1cef2d..11b933d 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
@@ -39,6 +39,7 @@
public void deactivate(boolean flushOnExit) throws HyracksDataException;
+ @Override
public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) throws HyracksDataException;
@@ -51,4 +52,9 @@
public List<ILSMComponent> getImmutableComponents();
public boolean isPrimaryIndex();
+
+ /**
+ * @return true if the index is durable. Otherwise false.
+ */
+ public boolean isDurable();
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index 441dda1..440ad31 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -33,7 +33,6 @@
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.ComponentState;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
@@ -48,8 +47,6 @@
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
-import org.apache.hyracks.storage.common.file.BufferedFileHandle;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
public abstract class AbstractLSMIndex implements ILSMIndexInternal {
@@ -80,11 +77,10 @@
protected boolean memoryComponentsAllocated = false;
public AbstractLSMIndex(List<IVirtualBufferCache> virtualBufferCaches, IBufferCache diskBufferCache,
- ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider,
- double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
- ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
- int[] filterFields, boolean durable) {
+ ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFrameFactory filterFrameFactory,
+ LSMComponentFilterManager filterManager, int[] filterFields, boolean durable) {
this.virtualBufferCaches = virtualBufferCaches;
this.diskBufferCache = diskBufferCache;
this.diskFileMapProvider = diskFileMapProvider;
@@ -149,8 +145,8 @@
}
protected void markAsValidInternal(IBufferCache bufferCache, BloomFilter filter) throws HyracksDataException {
- if(durable){
- bufferCache.force(filter.getFileId(),true);
+ if (durable) {
+ bufferCache.force(filter.getFileId(), true);
}
}
@@ -208,7 +204,7 @@
return diskBufferCache;
}
- public boolean isEmptyIndex() throws HyracksDataException {
+ public boolean isEmptyIndex() {
boolean isModified = false;
for (ILSMComponent c : memoryComponents) {
AbstractMemoryLSMComponent mutableComponent = (AbstractMemoryLSMComponent) c;
@@ -288,9 +284,16 @@
}
}
+ @Override
public abstract void allocateMemoryComponents() throws HyracksDataException;
+ @Override
public boolean isMemoryComponentsAllocated() {
return memoryComponentsAllocated;
}
+
+ @Override
+ public boolean isDurable() {
+ return durable;
+ }
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 21b0d8a..0224c5c 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -63,7 +63,8 @@
this.opTracker = opTracker;
this.mergePolicy = mergePolicy;
fullMergeIsRequested = new AtomicBoolean();
- this.replicationEnabled = replicationEnabled;
+ //only durable indexes are replicated
+ this.replicationEnabled = replicationEnabled && lsmIndex.isDurable();
if (replicationEnabled) {
this.componentsToBeReplicated = new ArrayList<ILSMComponent>();
}