[NO ISSUE][CLUS] Avoid locking resources while waiting for IO

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
Waiting for all I/O operations should not acquire a lock on
when cleaning up a partition.

Change-Id: I767474703e5519bf576db63fc988aeb67aebffe4
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18400
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
index 9064db5..c8db4cd 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
@@ -266,6 +266,8 @@
         synchronized (this) {
             while (partitionPendingIO.getOrDefault(partition, 0) > 0) {
                 try {
+                    int numPendingIOOps = partitionPendingIO.getOrDefault(partition, 0);
+                    LOGGER.debug("Waiting for {} IO operations in {} partition {}", numPendingIOOps, this, partition);
                     wait();
                 } catch (InterruptedException e) {
                     Thread.currentThread().interrupt();
@@ -273,6 +275,8 @@
                 }
             }
 
+            LOGGER.debug("All IO operations for {} partition {} are finished", this, partition);
+
             Set<IndexInfo> indexes = partitionIndexes.get(partition);
             if (indexes != null) {
                 // Perform the required operation
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/CleanupBlockingIOOperation.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/CleanupBlockingIOOperation.java
index 0790a22..6875ebd 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/CleanupBlockingIOOperation.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/CleanupBlockingIOOperation.java
@@ -62,7 +62,7 @@
     }
 
     /**
-     * Clean all inactive indexes while the DatasetLifeCycleManager is synchronized
+     * Clean all inactive indexes
      */
     @Override
     public void afterOperation() throws HyracksDataException {
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index d0b4aa1..5c7d5ac 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -559,14 +559,10 @@
     }
 
     public void cleanup(int partition) throws HyracksDataException {
-        beforeReadAccess();
-        try {
-            CleanupBlockingIOOperation cleanupOp = new CleanupBlockingIOOperation(partition, this, ioManager);
-            datasetLifecycleManager.waitForIOAndPerform(AllDatasetsReplicationStrategy.INSTANCE, partition, cleanupOp);
-        } finally {
-            clearResourcesCache();
-            afterReadAccess();
-        }
+        LOGGER.info("Cleaning up indexes in partition {}", partition);
+        CleanupBlockingIOOperation cleanupOp = new CleanupBlockingIOOperation(partition, this, ioManager);
+        datasetLifecycleManager.waitForIOAndPerform(AllDatasetsReplicationStrategy.INSTANCE, partition, cleanupOp);
+        LOGGER.info("Finished cleaning up indexes in partition {} successfully", partition);
     }
 
     /**
@@ -575,6 +571,7 @@
      * @param index to clean
      */
     void cleanupIndex(FileReference index) throws HyracksDataException {
+        beforeReadAccess();
         try {
             deleteIndexMaskedFiles(index);
             if (isValidIndex(index)) {
@@ -582,6 +579,9 @@
             }
         } catch (IOException | ParseException e) {
             throw HyracksDataException.create(e);
+        } finally {
+            clearResourcesCache();
+            afterReadAccess();
         }
     }