[ASTERIXDB-3190][*DB] Use HeadObject when only metadata info is needed
Change-Id: I9c201cbce0bd92ca2c7a2ae87aebf399c88b55af
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17544
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
index 97d8d80..f87adba 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
@@ -217,7 +217,7 @@
}
@Override
- public long getSize(IFileHandle fileHandle) {
+ public long getSize(IFileHandle fileHandle) throws HyracksDataException {
if (!fileHandle.getFileReference().getFile().exists()) {
return cloudClient.getObjectSize(bucket, fileHandle.getFileReference().getRelativePath());
}
@@ -225,7 +225,7 @@
}
@Override
- public long getSize(FileReference fileReference) {
+ public long getSize(FileReference fileReference) throws HyracksDataException {
if (!fileReference.getFile().exists()) {
return cloudClient.getObjectSize(bucket, fileReference.getRelativePath());
}
@@ -265,7 +265,7 @@
}
@Override
- public boolean exists(FileReference fileRef) {
+ public boolean exists(FileReference fileRef) throws HyracksDataException {
// Check if the file exists locally first as newly created files (i.e., they are empty) are not stored in cloud storage
return fileRef.getFile().exists() || cloudClient.exists(bucket, fileRef.getRelativePath());
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
index fe2a2d6..4f02df2 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
@@ -115,7 +115,7 @@
* @param path path
* @return size
*/
- long getObjectSize(String bucket, String path);
+ long getObjectSize(String bucket, String path) throws HyracksDataException;
/**
* Checks if an object exists at the specified path
@@ -124,7 +124,7 @@
* @param path path
* @return {@code true} if the object exists, {@code false} otherwise
*/
- boolean exists(String bucket, String path);
+ boolean exists(String bucket, String path) throws HyracksDataException;
/**
* Syncs files by downloading them from cloud storage to local storage
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index b800b96..6db784d 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -62,6 +62,7 @@
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
@@ -201,23 +202,27 @@
s3Client.deleteObjects(deleteReq);
}
- // TODO(htowaileb): Use the following cheaper request to check if an object exists
- // https://stackoverflow.com/questions/3910071/check-file-size-on-s3-without-downloading
@Override
- public long getObjectSize(String bucket, String path) {
- List<S3Object> objects = listS3Objects(s3Client, bucket, path);
- if (objects.isEmpty()) {
+ public long getObjectSize(String bucket, String path) throws HyracksDataException {
+ try {
+ return s3Client.headObject(HeadObjectRequest.builder().bucket(bucket).key(path).build()).contentLength();
+ } catch (NoSuchKeyException ex) {
return 0;
+ } catch (Exception ex) {
+ throw HyracksDataException.create(ex);
}
- return objects.get(0).size();
}
- // TODO(htowaileb): Use the following cheaper request to check if an object exists
- // https://docs.aws.amazon.com/AmazonS3/latest/userguide/example_s3_HeadObject_section.html
@Override
- public boolean exists(String bucket, String path) {
- List<S3Object> objects = listS3Objects(s3Client, bucket, path);
- return !objects.isEmpty();
+ public boolean exists(String bucket, String path) throws HyracksDataException {
+ try {
+ s3Client.headObject(HeadObjectRequest.builder().bucket(bucket).key(path).build());
+ return true;
+ } catch (NoSuchKeyException ex) {
+ return false;
+ } catch (Exception ex) {
+ throw HyracksDataException.create(ex);
+ }
}
private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter filter) {
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
index 07e1c78..63226ac 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
@@ -65,9 +65,9 @@
void truncate(IFileHandle fileHandle, long size) throws HyracksDataException;
- long getSize(IFileHandle fileHandle);
+ long getSize(IFileHandle fileHandle) throws HyracksDataException;
- long getSize(FileReference fileReference);
+ long getSize(FileReference fileReference) throws HyracksDataException;
WritableByteChannel newWritableChannel(IFileHandle fileHandle);
@@ -139,7 +139,7 @@
// TODO: Remove and use list
Collection<FileReference> getMatchingFiles(FileReference root, FilenameFilter filter) throws HyracksDataException;
- boolean exists(FileReference fileRef);
+ boolean exists(FileReference fileRef) throws HyracksDataException;
void create(FileReference fileRef) throws HyracksDataException;
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index 24d1061..19fbff3 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -399,12 +399,12 @@
}
@Override
- public long getSize(IFileHandle fileHandle) {
+ public long getSize(IFileHandle fileHandle) throws HyracksDataException {
return getSize(fileHandle.getFileReference());
}
@Override
- public long getSize(FileReference fileReference) {
+ public long getSize(FileReference fileReference) throws HyracksDataException {
return fileReference.getFile().length();
}
@@ -596,7 +596,7 @@
}
@Override
- public boolean exists(FileReference fileRef) {
+ public boolean exists(FileReference fileRef) throws HyracksDataException {
return fileRef.getFile().exists();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java
index 44a2031..6a4f10d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java
@@ -284,7 +284,7 @@
return ioManager.doSyncWrite(fileHandle, offset, buf);
}
- protected final long getFileSize() {
+ protected final long getFileSize() throws HyracksDataException {
return ioManager.getSize(fileHandle);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java
index 11862dc..3d4d80b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java
@@ -121,7 +121,7 @@
}
@Override
- public int getNumberOfPages() {
+ public int getNumberOfPages() throws HyracksDataException {
if (DEBUG) {
assert getFileSize() % bufferCache.getPageSizeWithHeader() == 0;
}