[ASTERIXDB-3594][OTH] Add metric for failed IO operations

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- failed flushes
- failed merges

Ext-ref: MB-65335

Change-Id: I5b0a80b61e73eb35d105827e9116a9a1a715b35f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19630
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
index 7f3642b..2803f21 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
@@ -23,6 +23,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.asterix.common.api.IIOBlockingOperation;
 import org.apache.asterix.common.transactions.ILogManager;
@@ -47,6 +48,8 @@
     private final int datasetID;
     private final ILogManager logManager;
     private final LogRecord waitLog = new LogRecord();
+    private final AtomicInteger failedFlushes = new AtomicInteger();
+    private final AtomicInteger failedMerges = new AtomicInteger();
     private int numActiveIOOps;
     private int pendingFlushes;
     private int pendingMerges;
@@ -317,4 +320,25 @@
     public synchronized int getPendingReplications() {
         return pendingReplications;
     }
+
+    public int getFailedFlushes() {
+        return failedFlushes.get();
+    }
+
+    public int getFailedMerges() {
+        return failedMerges.get();
+    }
+
+    public void incrementFailedIoOp(ILSMIOOperation.LSMIOOperationType operation) {
+        switch (operation) {
+            case FLUSH:
+                failedFlushes.incrementAndGet();
+                break;
+            case MERGE:
+                failedMerges.incrementAndGet();
+                break;
+            default:
+                break;
+        }
+    }
 }
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
index fc27bf6..0128297 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
@@ -955,6 +955,8 @@
             stats.addPendingFlushes(dsr.getDatasetInfo().getPendingFlushes());
             stats.addPendingMerges(dsr.getDatasetInfo().getPendingMerges());
             stats.addPendingReplications(dsr.getDatasetInfo().getPendingReplications());
+            stats.addFailedFlushes(dsr.getDatasetInfo().getFailedFlushes());
+            stats.addFailedMerges(dsr.getDatasetInfo().getFailedMerges());
         }
         return stats;
     }
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationCallback.java
index 52af329..6887782 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationCallback.java
@@ -143,6 +143,7 @@
 
     @Override
     public void afterFailure(ILSMIOOperation operation) {
+        dsInfo.incrementFailedIoOp(operation.getIOOperationType());
         if (isMerge(operation)) {
             try {
                 ioManager.delete(getOperationMaskFilePath(operation));
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/StorageIOStats.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/StorageIOStats.java
index b58f63d..2e2a4a7 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/StorageIOStats.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/StorageIOStats.java
@@ -23,6 +23,8 @@
     private int pendingFlushes;
     private int pendingMerges;
     private int pendingReplications;
+    private int failedFlushes;
+    private int failedMerges;
 
     public void addPendingFlushes(int pending) {
         pendingFlushes += pending;
@@ -36,6 +38,22 @@
         pendingReplications += pending;
     }
 
+    public void addFailedFlushes(int failed) {
+        failedFlushes += failed;
+    }
+
+    public void addFailedMerges(int failed) {
+        failedMerges += failed;
+    }
+
+    public int getFailedFlushes() {
+        return failedFlushes;
+    }
+
+    public int getFailedMerges() {
+        return failedMerges;
+    }
+
     public int getPendingFlushes() {
         return pendingFlushes;
     }