[ASTERIXDB-3260][STO] Expose stored cloud objects for diagnostics
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
Expose the list of objects (as a JSON array) stored in the
cloud object store (S3)
Change-Id: I37900dc4d5401530a25ef66b916e7b6827ca93dd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17773
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index 6973b7b..d9c248e 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -49,6 +49,9 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
public abstract class AbstractCloudIOManager extends IOManager implements IPartitionBootstrapper {
private static final Logger LOGGER = LogManager.getLogger();
private static final String DATAVERSE_PATH =
@@ -260,4 +263,14 @@
super.close();
localIoManager.close();
}
+
+ /**
+ * Returns a list of all stored objects (sorted ASC by path) in the cloud and their sizes
+ *
+ * @param objectMapper to create the result {@link JsonNode}
+ * @return {@link JsonNode} with stored objects' information
+ */
+ public final JsonNode listAsJson(ObjectMapper objectMapper) {
+ return cloudClient.listAsJson(objectMapper, bucket);
+ }
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
index ff26915..f2b7ff4 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
@@ -28,6 +28,9 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
/**
* Interface containing methods to perform IO operation on the Cloud Storage
*/
@@ -137,6 +140,15 @@
void syncFiles(String bucket, Map<String, String> cloudToLocalStoragePaths) throws HyracksDataException;
/**
+ * Produces a {@link JsonNode} that contains information about the stored objects in the cloud
+ *
+ * @param objectMapper to create the result {@link JsonNode}
+ * @param bucket bucket name
+ * @return {@link JsonNode} with stored objects' information
+ */
+ JsonNode listAsJson(ObjectMapper objectMapper, String bucket);
+
+ /**
* Performs any necessary closing and cleaning up
*/
void close();
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index 5619fc8..5faf663 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -53,6 +53,11 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
@@ -100,22 +105,6 @@
}
- private S3Client buildClient() {
- S3ClientBuilder builder = S3Client.builder();
- builder.credentialsProvider(config.createCredentialsProvider());
- builder.region(Region.of(config.getRegion()));
- if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
- URI uri;
- try {
- uri = new URI(config.getEndpoint());
- } catch (URISyntaxException ex) {
- throw new IllegalArgumentException(ex);
- }
- builder.endpointOverride(uri);
- }
- return builder.build();
- }
-
@Override
public ICloudBufferedWriter createBufferedWriter(String bucket, String path) {
return new S3BufferedWriter(s3Client, profiler, bucket, path);
@@ -253,17 +242,6 @@
}
}
- private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter filter) {
- Set<String> files = new HashSet<>();
- for (S3Object s3Object : contents) {
- String path = config.isEncodeKeys() ? S3Utils.decodeURI(s3Object.key()) : s3Object.key();
- if (filter.accept(null, IoUtil.getFileNameFromPath(path))) {
- files.add(path);
- }
- }
- return files;
- }
-
@Override
public void syncFiles(String bucket, Map<String, String> cloudToLocalStoragePaths) throws HyracksDataException {
LOGGER.info("Syncing cloud storage to local storage started");
@@ -311,6 +289,58 @@
LOGGER.info("Syncing cloud storage to local storage successful");
}
+ @Override
+ public JsonNode listAsJson(ObjectMapper objectMapper, String bucket) {
+ List<S3Object> objects = listS3Objects(s3Client, bucket, "/");
+ ArrayNode objectsInfo = objectMapper.createArrayNode();
+
+ objects.sort((x, y) -> String.CASE_INSENSITIVE_ORDER.compare(x.key(), y.key()));
+ for (S3Object object : objects) {
+ ObjectNode objectInfo = objectsInfo.addObject();
+ objectInfo.put("path", object.key());
+ objectInfo.put("size", object.size());
+ }
+ return objectsInfo;
+ }
+
+ @Override
+ public void close() {
+ if (s3Client != null) {
+ s3Client.close();
+ }
+
+ if (s3TransferManager != null) {
+ s3TransferManager.close();
+ }
+ }
+
+ private S3Client buildClient() {
+ S3ClientBuilder builder = S3Client.builder();
+ builder.credentialsProvider(config.createCredentialsProvider());
+ builder.region(Region.of(config.getRegion()));
+ if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
+ URI uri;
+ try {
+ uri = new URI(config.getEndpoint());
+ } catch (URISyntaxException ex) {
+ throw new IllegalArgumentException(ex);
+ }
+ builder.endpointOverride(uri);
+ }
+ return builder.build();
+ }
+
+ private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter filter) {
+ Set<String> files = new HashSet<>();
+ for (S3Object s3Object : contents) {
+ String path = config.isEncodeKeys() ? S3Utils.decodeURI(s3Object.key()) : s3Object.key();
+ if (filter.accept(null, IoUtil.getFileNameFromPath(path))) {
+ files.add(path);
+ }
+ }
+ return files;
+ }
+
private void downloadFiles(String bucket, Map<String, String> cloudToLocalStoragePaths)
throws HyracksDataException {
byte[] buffer = new byte[8 * 1024];
@@ -364,15 +394,4 @@
s3TransferManager = S3TransferManager.builder().s3Client(client).build();
return s3TransferManager;
}
-
- @Override
- public void close() {
- if (s3Client != null) {
- s3Client.close();
- }
-
- if (s3TransferManager != null) {
- s3TransferManager.close();
- }
- }
}