[MULTIPLE] Metrics and incomplete downloads fixes

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- ASTERIXDB-3449: 'metrics' object can miss commas
- ASTERIXDB-3448: Delete partially downloaded files
  on bootstrap

Change-Id: I9796906ec2c68aed323c91ed673aba6833f257dd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18409
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
index a8a3135..1e65533 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
@@ -101,7 +101,7 @@
             pw.print("\n");
             pw.print("\t");
             ResultUtil.printField(pw, Metrics.BUFFERCACHE_PAGEREAD_COUNT.str(), metrics.getBufferCachePageReadCount(),
-                    hasWarnings || hasErrors);
+                    hasWarnings || hasErrors || madeCloudReadRequests);
             pw.print("\n");
         }
         if (madeCloudReadRequests) {
@@ -115,7 +115,7 @@
             pw.print("\n");
             pw.print("\t");
             ResultUtil.printField(pw, Metrics.REMOTE_PAGES_PERSISTED_COUNT.str(), metrics.getCloudPagesPersistedCount(),
-                    true);
+                    hasWarnings || hasErrors);
             pw.print("\n");
         }
         if (hasWarnings) {
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
index f8cf2c3..35b7255 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
@@ -113,7 +113,7 @@
             }
         }
 
-        // Keep uncached files list (i.e., files exists in cloud only)
+        // Keep uncached files list (i.e., files exist in cloud only)
         cloudFiles.removeAll(localFiles);
         int remainingUncachedFiles = cloudFiles.size();
         boolean canReplaceAccessor = replacer != InitialCloudAccessor.NO_OP_REPLACER;
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudFile.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudFile.java
index e1cf135..50edd7b 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudFile.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudFile.java
@@ -18,7 +18,12 @@
  */
 package org.apache.asterix.cloud.clients;
 
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
 public final class CloudFile {
+    private static final long IGNORED_SIZE = -1;
     private final String path;
     private final long size;
 
@@ -47,7 +52,7 @@
         }
 
         CloudFile other = (CloudFile) obj;
-        return path.equals(other.path);
+        return path.equals(other.path) && compareSize(other.size);
     }
 
     @Override
@@ -55,11 +60,25 @@
         return path;
     }
 
+    private boolean compareSize(long otherSize) {
+        // Compare sizes iff both sizes are not ignored
+        return size == otherSize || size == IGNORED_SIZE || otherSize == IGNORED_SIZE;
+    }
+
     public static CloudFile of(String path, long size) {
         return new CloudFile(path, size);
     }
 
     public static CloudFile of(String path) {
-        return new CloudFile(path, 0);
+        return new CloudFile(path, IGNORED_SIZE);
+    }
+
+    public static Map<String, CloudFile> toMap(Set<CloudFile> cloudFiles) {
+        Map<String, CloudFile> map = new HashMap<>();
+        for (CloudFile cloudFile : cloudFiles) {
+            map.put(cloudFile.getPath(), cloudFile);
+        }
+
+        return map;
     }
 }
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
index dcc1b93..e4ad05e 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
@@ -58,9 +58,12 @@
                 continue;
             }
 
-            CloudFile path = CloudFile.of(file.getRelativePath());
+            CloudFile path = CloudFile.of(file.getRelativePath(), ioManager.getSize(file));
             if (!cloudFiles.contains(path)) {
-                // Delete local files that do not exist in cloud storage (the ground truth for valid files)
+                /*
+                 * Delete local files that do not exist in cloud storage (the ground truth for valid files), or files
+                 * that has not been downloaded completely.
+                 */
                 logDeleteFile(file);
                 localFilesIter.remove();
                 ioManager.delete(file);
@@ -82,7 +85,8 @@
     }
 
     private static void logDeleteFile(FileReference fileReference) {
-        LOGGER.info("Deleting {} from the local cache as {} doesn't exist in the cloud", fileReference,
-                fileReference.getRelativePath());
+        LOGGER.info(
+                "Deleting {} from the local cache as {} either doesn't exist in the cloud or it wasn't downloaded completely",
+                fileReference, fileReference.getRelativePath());
     }
 }