commit 8b1003bd931136520f15f6eac103d0616026d229
author Savyasach Reddy <esavyareddy2@gmail.com> Wed Jun 05 23:59:42 2024 +0530
committer Murtadha Hubail <mhubail@apache.org> Fri Jun 07 04:31:27 2024 +0000
tree 5f8aa9bfab15371ca5a3436c1f4c9e5911ca8b90
parent 76a80b7976c1eb10aef8135923e002035a2e6a7d

[ASTERIXDB-3420][STO] Account for file changes on directory traversal

- user model changes: no
- storage format changes: no
- interface changes: no

Details:

- Replace FileUtils.sizeOfDirectory with custom directory size function
  that accounts for file changes while traversing a directory.

Change-Id: I660233e6b74429c75b9dd63103db1248846054a0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18343
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Tested-by: Murtadha Hubail <mhubail@apache.org>
Integration-Tests: Murtadha Hubail <mhubail@apache.org>

hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java

diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java
index fcf8c7a..ddb8ef6 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java
@@ -24,15 +24,19 @@
 import java.io.FilenameFilter;
 import java.io.IOException;
 import java.nio.channels.FileChannel;
+import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
 import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 
 import org.apache.commons.io.FileUtils;
@@ -210,4 +214,31 @@
         }
         return files;
     }
+
+    public static long sizeOfDirectory(final Path path) {
+        final AtomicLong size = new AtomicLong(0);
+        try {
+            Files.walkFileTree(path, new SimpleFileVisitor<>() {
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+                    size.addAndGet(attrs.size());
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFileFailed(Path file, IOException exc) {
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        } catch (IOException e) {
+            // This should never happen
+            throw new IllegalStateException("Cannot get the size of directory " + path);
+        }
+        return size.get();
+    }
 }

hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java

diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index 9909e97..b35111e 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -473,7 +473,7 @@
     public long getTotalDiskUsage() {
         long totalSize = 0;
         for (IODeviceHandle handle : ioDevices) {
-            totalSize += FileUtils.sizeOfDirectory(handle.getMount());
+            totalSize += IoUtil.sizeOfDirectory(handle.getMount().toPath());
         }
         return totalSize;
     }