commit abaf5f1f8613e7319abbcaaa8eb29d55f94de67d
author Savyasach Reddy <esavyareddy2@gmail.com> Wed Jan 22 23:33:22 2025 +0530
committer Murtadha Hubail <mhubail@apache.org> Thu Jan 23 16:55:44 2025 +0000
tree fa8c741dfee6d17c58b7fac19a15266cedb115a1
parent 822afae2293134c3af589daa368779892822aaf0

[NO ISSUE]: Account for file changes on directory traversal

- user model changes: no
- storage format changes: no
- interface changes: no

Details:

- Replace FileUtils.sizeOfDirectory with custom directory size function
  that accounts for file changes while traversing a directory.

Ext-ref: MB-64944
Change-Id: I91501a6ca01a872c28ae7773120750ac28770d05
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19355
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Tested-by: Murtadha Hubail <mhubail@apache.org>

hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java

diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java
index ae49cb6..9c29206 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java
@@ -24,13 +24,17 @@
 import java.io.FilenameFilter;
 import java.io.IOException;
 import java.nio.channels.FileChannel;
+import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
 import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.hyracks.api.exceptions.ErrorCode;
@@ -189,4 +193,31 @@
             }
         }
     }
+
+    public static long sizeOfDirectory(final Path path) {
+        final AtomicLong size = new AtomicLong(0);
+        try {
+            Files.walkFileTree(path, new SimpleFileVisitor<>() {
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+                    size.addAndGet(attrs.size());
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFileFailed(Path file, IOException exc) {
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        } catch (IOException e) {
+            // This should never happen
+            throw new IllegalStateException("Cannot get the size of directory " + path);
+        }
+        return size.get();
+    }
 }

hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java

diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index dee61a4..8d058e5 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -37,7 +37,6 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.io.FileUtils;
 import org.apache.hyracks.api.exceptions.ErrorCode;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.io.FileReference;
@@ -441,7 +440,7 @@
     public long getTotalDiskUsage() {
         long totalSize = 0;
         for (IODeviceHandle handle : ioDevices) {
-            totalSize += FileUtils.sizeOfDirectory(handle.getMount());
+            totalSize += IoUtil.sizeOfDirectory(handle.getMount().toPath());
         }
         return totalSize;
     }