[MULTIPLE] Metrics and incomplete downloads fixes
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- ASTERIXDB-3449: 'metrics' object can miss commas
- ASTERIXDB-3448: Delete partially downloaded files
on bootstrap
Change-Id: I9796906ec2c68aed323c91ed673aba6833f257dd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18409
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
index a8a3135..1e65533 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
@@ -101,7 +101,7 @@
pw.print("\n");
pw.print("\t");
ResultUtil.printField(pw, Metrics.BUFFERCACHE_PAGEREAD_COUNT.str(), metrics.getBufferCachePageReadCount(),
- hasWarnings || hasErrors);
+ hasWarnings || hasErrors || madeCloudReadRequests);
pw.print("\n");
}
if (madeCloudReadRequests) {
@@ -115,7 +115,7 @@
pw.print("\n");
pw.print("\t");
ResultUtil.printField(pw, Metrics.REMOTE_PAGES_PERSISTED_COUNT.str(), metrics.getCloudPagesPersistedCount(),
- true);
+ hasWarnings || hasErrors);
pw.print("\n");
}
if (hasWarnings) {
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
index f8cf2c3..35b7255 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
@@ -113,7 +113,7 @@
}
}
- // Keep uncached files list (i.e., files exists in cloud only)
+ // Keep uncached files list (i.e., files exist in cloud only)
cloudFiles.removeAll(localFiles);
int remainingUncachedFiles = cloudFiles.size();
boolean canReplaceAccessor = replacer != InitialCloudAccessor.NO_OP_REPLACER;
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudFile.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudFile.java
index e1cf135..50edd7b 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudFile.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudFile.java
@@ -18,7 +18,12 @@
*/
package org.apache.asterix.cloud.clients;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
public final class CloudFile {
+ private static final long IGNORED_SIZE = -1;
private final String path;
private final long size;
@@ -47,7 +52,7 @@
}
CloudFile other = (CloudFile) obj;
- return path.equals(other.path);
+ return path.equals(other.path) && compareSize(other.size);
}
@Override
@@ -55,11 +60,25 @@
return path;
}
+ private boolean compareSize(long otherSize) {
+ // Compare sizes iff both sizes are not ignored
+ return size == otherSize || size == IGNORED_SIZE || otherSize == IGNORED_SIZE;
+ }
+
public static CloudFile of(String path, long size) {
return new CloudFile(path, size);
}
public static CloudFile of(String path) {
- return new CloudFile(path, 0);
+ return new CloudFile(path, IGNORED_SIZE);
+ }
+
+ public static Map<String, CloudFile> toMap(Set<CloudFile> cloudFiles) {
+ Map<String, CloudFile> map = new HashMap<>();
+ for (CloudFile cloudFile : cloudFiles) {
+ map.put(cloudFile.getPath(), cloudFile);
+ }
+
+ return map;
}
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
index dcc1b93..e4ad05e 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
@@ -58,9 +58,12 @@
continue;
}
- CloudFile path = CloudFile.of(file.getRelativePath());
+ CloudFile path = CloudFile.of(file.getRelativePath(), ioManager.getSize(file));
if (!cloudFiles.contains(path)) {
- // Delete local files that do not exist in cloud storage (the ground truth for valid files)
+ /*
+ * Delete local files that do not exist in cloud storage (the ground truth for valid files), or files
+ * that has not been downloaded completely.
+ */
logDeleteFile(file);
localFilesIter.remove();
ioManager.delete(file);
@@ -82,7 +85,8 @@
}
private static void logDeleteFile(FileReference fileReference) {
- LOGGER.info("Deleting {} from the local cache as {} doesn't exist in the cloud", fileReference,
- fileReference.getRelativePath());
+ LOGGER.info(
+ "Deleting {} from the local cache as {} either doesn't exist in the cloud or it wasn't downloaded completely",
+ fileReference, fileReference.getRelativePath());
}
}