[NO ISSUE][MISC] Add method to compute remote storage size used
- user model changes: no
- storage format changes: no
- interface changes: no
Ext-ref: MB-62718
Change-Id: I3496da75ecf2fcd3e6306b8fd2e2d722f0462516
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18491
Reviewed-by: Peeyush Gupta <peeyush.gupta@couchbase.com>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Peeyush Gupta <peeyush.gupta@couchbase.com>
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index 4913f83..c005253 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -375,43 +375,14 @@
*/
public final JsonNode listAsJson(ObjectMapper objectMapper) {
ArrayNode objectsInfo = objectMapper.createArrayNode();
- List<CloudFile> allFiles = new ArrayList<>();
try {
- // get cached files (read from disk)
- for (IODeviceHandle deviceHandle : getIODevices()) {
- FileReference storageRoot = deviceHandle.createFileRef(STORAGE_ROOT_DIR_NAME);
-
- Set<FileReference> deviceFiles;
- try {
- deviceFiles = localIoManager.list(storageRoot, IoUtil.NO_OP_FILTER);
- } catch (Throwable th) {
- LOGGER.warn("Failed to get local storage files for root {}", storageRoot.getRelativePath(), th);
- continue;
- }
-
- for (FileReference fileReference : deviceFiles) {
- try {
- allFiles.add(CloudFile.of(fileReference.getRelativePath(), fileReference.getFile().length()));
- } catch (Throwable th) {
- LOGGER.warn("Encountered issue for local storage file {}", fileReference.getRelativePath(), th);
- }
- }
- }
-
- // get uncached files from uncached files tracker
- for (UncachedFileReference uncachedFile : getUncachedFiles()) {
- allFiles.add(CloudFile.of(uncachedFile.getRelativePath(), uncachedFile.getSize()));
- }
-
- // combine all and sort
+ List<CloudFile> allFiles = list();
allFiles.sort((x, y) -> String.CASE_INSENSITIVE_ORDER.compare(x.getPath(), y.getPath()));
-
for (CloudFile file : allFiles) {
ObjectNode objectInfo = objectsInfo.addObject();
objectInfo.put("path", file.getPath());
objectInfo.put("size", file.getSize());
}
-
return objectsInfo;
} catch (Throwable th) {
LOGGER.warn("Failed to retrieve list of all cloud files", th);
@@ -422,6 +393,36 @@
}
}
+ private List<CloudFile> list() {
+ List<CloudFile> allFiles = new ArrayList<>();
+ // get cached files (read from disk)
+ for (IODeviceHandle deviceHandle : getIODevices()) {
+ FileReference storageRoot = deviceHandle.createFileRef(STORAGE_ROOT_DIR_NAME);
+
+ Set<FileReference> deviceFiles;
+ try {
+ deviceFiles = localIoManager.list(storageRoot, IoUtil.NO_OP_FILTER);
+ } catch (Throwable th) {
+ LOGGER.warn("Failed to get local storage files for root {}", storageRoot.getRelativePath(), th);
+ continue;
+ }
+
+ for (FileReference fileReference : deviceFiles) {
+ try {
+ allFiles.add(CloudFile.of(fileReference.getRelativePath(), fileReference.getFile().length()));
+ } catch (Throwable th) {
+ LOGGER.warn("Encountered issue for local storage file {}", fileReference.getRelativePath(), th);
+ }
+ }
+ }
+
+ // get uncached files from uncached files tracker
+ for (UncachedFileReference uncachedFile : getUncachedFiles()) {
+ allFiles.add(CloudFile.of(uncachedFile.getRelativePath(), uncachedFile.getSize()));
+ }
+ return allFiles;
+ }
+
/**
* Writes the bytes to the specified key in the bucket
*
@@ -465,4 +466,12 @@
+ ", cloudOffset: " + cloudOffset + " != requestedWriteOffset: " + requestedWriteOffset);
}
}
+
+ public long getTotalRemoteStorageSizeForNodeBytes() {
+ long size = 0;
+ for (CloudFile file : list()) {
+ size += file.getSize();
+ }
+ return size;
+ }
}