[ASTERIXDB-3497][OTH]: Calculate physical disk size in cloud deployment
Ext-ref: MB-62733
Change-Id: Ic7b9605b704889836674f8d62a6c8dea86e7df52
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18786
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index c005253..da014a4 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -26,6 +26,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
+import java.nio.file.FileStore;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -51,6 +52,7 @@
import org.apache.hyracks.api.io.IIOBulkOperation;
import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.api.util.IoUtil;
+import org.apache.hyracks.cloud.filesystem.PhysicalDrive;
import org.apache.hyracks.cloud.io.ICloudIOManager;
import org.apache.hyracks.cloud.io.request.ICloudBeforeRetryRequest;
import org.apache.hyracks.cloud.io.request.ICloudRequest;
@@ -76,6 +78,7 @@
protected final List<FileReference> partitionPaths;
protected final IOManager localIoManager;
protected final INamespacePathResolver nsPathResolver;
+ private final List<FileStore> drivePaths;
public AbstractCloudIOManager(IOManager ioManager, CloudProperties cloudProperties,
INamespacePathResolver nsPathResolver, ICloudGuardian guardian) throws HyracksDataException {
@@ -90,6 +93,7 @@
partitions = new HashSet<>();
partitionPaths = new ArrayList<>();
this.localIoManager = ioManager;
+ drivePaths = PhysicalDrive.getDrivePaths(ioDevices);
}
/*
@@ -474,4 +478,9 @@
}
return size;
}
+
+ @Override
+ public long getTotalDiskUsage() {
+ return PhysicalDrive.getUsedSpace(drivePaths);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/cache/unit/AbstractIndexUnit.java b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/cache/unit/AbstractIndexUnit.java
index 1532340..54ea23c 100644
--- a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/cache/unit/AbstractIndexUnit.java
+++ b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/cache/unit/AbstractIndexUnit.java
@@ -36,7 +36,7 @@
public final void open() {
lastAccessTime.set(System.nanoTime());
- openCounter.get();
+ openCounter.incrementAndGet();
}
public final void close() {
diff --git a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/filesystem/PhysicalDrive.java b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/filesystem/PhysicalDrive.java
index 7010e88..28069a9 100644
--- a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/filesystem/PhysicalDrive.java
+++ b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/filesystem/PhysicalDrive.java
@@ -53,17 +53,18 @@
@Override
public boolean computeAndCheckIsPressured() {
- long usedSpace = getUsedSpace();
+ long usedSpace = getUsedSpace(drivePaths);
long pressureCapacity = diskSpace.getPressureCapacity();
boolean isPressured = usedSpace > pressureCapacity;
this.usedSpace.set(usedSpace);
- if (isPressured) {
- LOGGER.info("Used space: {}, pressureCapacity: {} (isPressured: {})",
+ if (usedSpace >= diskSpace.getAllocatedCapacity()) {
+ LOGGER.warn(
+ "Allocated disk space has been exceeded. Used space: {}, pressureCapacity: {} (isPressured: {})",
StorageUtil.toHumanReadableSize(usedSpace), StorageUtil.toHumanReadableSize(pressureCapacity),
true);
- } else if (usedSpace >= diskSpace.getAllocatedCapacity()) {
- LOGGER.warn("Allocated disk space has been exceeded. Used space: {}, pressureCapacity: {}",
+ } else if (isPressured) {
+ LOGGER.info("Used space: {}, pressureCapacity: {} (isPressured: {})",
StorageUtil.toHumanReadableSize(usedSpace), StorageUtil.toHumanReadableSize(pressureCapacity),
true);
} else {
@@ -85,10 +86,9 @@
return usedSpace.get() < diskSpace.getPressureCapacity();
}
- private long getUsedSpace() {
+ public static long getUsedSpace(List<FileStore> drivePaths) {
long totalUsedSpace = 0;
- for (int i = 0; i < drivePaths.size(); i++) {
- FileStore device = drivePaths.get(i);
+ for (FileStore device : drivePaths) {
try {
totalUsedSpace += getTotalSpace(device) - getUsableSpace(device);
} catch (HyracksDataException e) {
@@ -121,7 +121,7 @@
return new DiskSpace(allocatedCapacity, pressureCapacity);
}
- private static List<FileStore> getDrivePaths(List<IODeviceHandle> deviceHandles) throws HyracksDataException {
+ public static List<FileStore> getDrivePaths(List<IODeviceHandle> deviceHandles) throws HyracksDataException {
Set<String> distinctDrives = new HashSet<>();
List<FileStore> fileStores = new ArrayList<>();
for (IODeviceHandle handle : deviceHandles) {
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index b35111e..740940e 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -76,7 +76,7 @@
private final ExecutorService executor;
private final BlockingQueue<IoRequest> submittedRequests;
private final BlockingQueue<IoRequest> freeRequests;
- private final List<IODeviceHandle> ioDevices;
+ protected final List<IODeviceHandle> ioDevices;
private final List<IODeviceHandle> workspaces;
private final IFileDeviceResolver deviceComputer;
/*