[ASTERIXDB-3390][STO]: Update storage scheme for GCS
Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Savyasach Reddy <savyasach.enukonda@couchbase.com>
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
index d0ebd24..ccb7f9d 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
@@ -64,7 +64,7 @@
cloud.deployment=true
storage.buffercache.pagesize=32KB
storage.partitioning=static
-cloud.storage.scheme=gcs
+cloud.storage.scheme=gs
cloud.storage.bucket=cloud-storage-container
cloud.storage.region=us-west2
cloud.storage.endpoint=http://127.0.0.1:4443
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index 8cf5ac5..9df26f2 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -239,14 +239,17 @@
public final long doSyncWrite(IFileHandle fHandle, long offset, ByteBuffer[] dataArray)
throws HyracksDataException {
long writtenBytes = localIoManager.doSyncWrite(fHandle, offset, dataArray);
+ dataArray[0].flip();
+ dataArray[1].flip();
cloudWrite(fHandle, dataArray);
return writtenBytes;
}
@Override
- public final int doSyncWrite(IFileHandle fHandle, long offset, ByteBuffer dataArray) throws HyracksDataException {
- int writtenBytes = localIoManager.doSyncWrite(fHandle, offset, dataArray);
- cloudWrite(fHandle, dataArray);
+ public final int doSyncWrite(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException {
+ int writtenBytes = localIoManager.doSyncWrite(fHandle, offset, data);
+ data.flip();
+ cloudWrite(fHandle, data);
return writtenBytes;
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
index 35ab467..d5f508d 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
@@ -27,7 +27,7 @@
public class CloudClientProvider {
private static final String S3 = "s3";
- private static final String GCS = "gcs";
+ private static final String GCS = "gs";
private CloudClientProvider() {
throw new AssertionError("do not instantiate");
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
index 9eb3fe0..9bcd386 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
@@ -204,7 +204,7 @@
public boolean exists(String bucket, String path) {
profiler.objectGet();
Blob blob = gcsClient.get(bucket, path, Storage.BlobGetOption.fields(Storage.BlobField.values()));
- return blob != null;
+ return blob != null && blob.exists();
}
@Override
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
index cccd9ec..a6dade5 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
@@ -40,7 +40,6 @@
private final String path;
private final IRequestProfiler profiler;
private final Storage gcsClient;
- private boolean uploadStarted = false;
private WriteChannel writer = null;
public GCSWriter(String bucket, String path, Storage gcsClient, IRequestProfiler profiler) {
@@ -88,7 +87,6 @@
try {
writer.close();
writer = null;
- uploadStarted = false;
} catch (IOException e) {
throw HyracksDataException.create(e);
}
@@ -104,8 +102,7 @@
}
private void setUploadId() {
- if (!uploadStarted) {
- uploadStarted = true;
+ if (writer == null) {
writer = gcsClient.writer(BlobInfo.newBuilder(BlobId.of(bucket, path)).build());
writer.setChunkSize(WRITE_BUFFER_SIZE);
log("STARTED");