[ASTERIXDB-3594][OTH] Add metric for failed IO operations
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- failed flushes
- failed merges
Ext-ref: MB-65335
Change-Id: I5b0a80b61e73eb35d105827e9116a9a1a715b35f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19630
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
index 7f3642b..2803f21 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
@@ -23,6 +23,7 @@
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.asterix.common.api.IIOBlockingOperation;
import org.apache.asterix.common.transactions.ILogManager;
@@ -47,6 +48,8 @@
private final int datasetID;
private final ILogManager logManager;
private final LogRecord waitLog = new LogRecord();
+ private final AtomicInteger failedFlushes = new AtomicInteger();
+ private final AtomicInteger failedMerges = new AtomicInteger();
private int numActiveIOOps;
private int pendingFlushes;
private int pendingMerges;
@@ -317,4 +320,25 @@
public synchronized int getPendingReplications() {
return pendingReplications;
}
+
+ public int getFailedFlushes() {
+ return failedFlushes.get();
+ }
+
+ public int getFailedMerges() {
+ return failedMerges.get();
+ }
+
+ public void incrementFailedIoOp(ILSMIOOperation.LSMIOOperationType operation) {
+ switch (operation) {
+ case FLUSH:
+ failedFlushes.incrementAndGet();
+ break;
+ case MERGE:
+ failedMerges.incrementAndGet();
+ break;
+ default:
+ break;
+ }
+ }
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
index fc27bf6..0128297 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
@@ -955,6 +955,8 @@
stats.addPendingFlushes(dsr.getDatasetInfo().getPendingFlushes());
stats.addPendingMerges(dsr.getDatasetInfo().getPendingMerges());
stats.addPendingReplications(dsr.getDatasetInfo().getPendingReplications());
+ stats.addFailedFlushes(dsr.getDatasetInfo().getFailedFlushes());
+ stats.addFailedMerges(dsr.getDatasetInfo().getFailedMerges());
}
return stats;
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationCallback.java
index 52af329..6887782 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationCallback.java
@@ -143,6 +143,7 @@
@Override
public void afterFailure(ILSMIOOperation operation) {
+ dsInfo.incrementFailedIoOp(operation.getIOOperationType());
if (isMerge(operation)) {
try {
ioManager.delete(getOperationMaskFilePath(operation));
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/StorageIOStats.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/StorageIOStats.java
index b58f63d..2e2a4a7 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/StorageIOStats.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/StorageIOStats.java
@@ -23,6 +23,8 @@
private int pendingFlushes;
private int pendingMerges;
private int pendingReplications;
+ private int failedFlushes;
+ private int failedMerges;
public void addPendingFlushes(int pending) {
pendingFlushes += pending;
@@ -36,6 +38,22 @@
pendingReplications += pending;
}
+ public void addFailedFlushes(int failed) {
+ failedFlushes += failed;
+ }
+
+ public void addFailedMerges(int failed) {
+ failedMerges += failed;
+ }
+
+ public int getFailedFlushes() {
+ return failedFlushes;
+ }
+
+ public int getFailedMerges() {
+ return failedMerges;
+ }
+
public int getPendingFlushes() {
return pendingFlushes;
}