[ASTERIXDB-3182][*DB]: Add support to write to cloud as backend storage
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Add interface and APIs to write to cloud as backend storage.
- Add capability to read cloud client config from file or default to mock.
- Add capability to read cloud storage config from file or default to mock.
Change-Id: I116e6dd3bfcfca389108d6233326cb509854228c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17490
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml
index 6e8224d..a0ff91a 100644
--- a/asterixdb/asterix-app/pom.xml
+++ b/asterixdb/asterix-app/pom.xml
@@ -780,6 +780,11 @@
<artifactId>hyracks-storage-am-lsm-invertedindex</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-cloud</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.testcontainers</groupId>
<artifactId>postgresql</artifactId>
<scope>test</scope>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
index e4247f0..3fad2d1 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.cc;
+import static org.apache.hyracks.control.common.controllers.ControllerConfig.Option.CLOUD_DEPLOYMENT;
+
import java.io.IOException;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Supplier;
@@ -366,4 +368,9 @@
public IDataPartitioningProvider getDataPartitioningProvider() {
return dataPartitioningProvider;
}
+
+ @Override
+ public boolean isCloudDeployment() {
+ return ccServiceCtx.getAppConfig().getBoolean(CLOUD_DEPLOYMENT);
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index a46522e..f9646b0 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.nc;
+import static org.apache.hyracks.control.common.controllers.ControllerConfig.Option.CLOUD_DEPLOYMENT;
+
import java.io.IOException;
import java.rmi.RemoteException;
import java.rmi.server.UnicastRemoteObject;
@@ -28,6 +30,7 @@
import org.apache.asterix.active.ActiveManager;
import org.apache.asterix.app.result.ResultReader;
+import org.apache.asterix.cloud.CloudIOManager;
import org.apache.asterix.common.api.IConfigValidator;
import org.apache.asterix.common.api.IConfigValidatorFactory;
import org.apache.asterix.common.api.ICoordinationService;
@@ -91,6 +94,7 @@
import org.apache.hyracks.client.result.ResultSet;
import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.control.nc.NodeControllerService;
+import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.ipc.impl.HyracksConnection;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
@@ -143,6 +147,7 @@
private ILSMIOOperationScheduler lsmIOScheduler;
private PersistentLocalResourceRepository localResourceRepository;
private IIOManager ioManager;
+ private IIOManager cloudIoManager;
private boolean isShuttingdown;
private ActiveManager activeManager;
private IReplicationChannel replicationChannel;
@@ -185,6 +190,9 @@
IConfigValidatorFactory configValidatorFactory, IReplicationStrategyFactory replicationStrategyFactory,
boolean initialRun) throws IOException {
ioManager = getServiceContext().getIoManager();
+ if (isCloudDeployment()) {
+ cloudIoManager = new CloudIOManager((IOManager) ioManager);
+ }
int ioQueueLen = getServiceContext().getAppConfig().getInt(NCConfig.Option.IO_QUEUE_SIZE);
threadExecutor =
MaintainedThreadNameExecutorService.newCachedThreadPool(getServiceContext().getThreadFactory());
@@ -238,6 +246,9 @@
receptionist = receptionistFactory.create();
if (replicationProperties.isReplicationEnabled()) {
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info("Replication is enabled");
+ }
replicationManager = new ReplicationManager(this, replicationStrategyFactory, replicationProperties);
//pass replication manager to replication required object
@@ -250,11 +261,11 @@
//initialize replication channel
replicationChannel = new ReplicationChannel(this);
- bufferCache = new BufferCache(ioManager, prs, pcp, new FileMapManager(),
+ bufferCache = new BufferCache(getLocalOrCloudIoManager(), prs, pcp, new FileMapManager(),
storageProperties.getBufferCacheMaxOpenFiles(), ioQueueLen, getServiceContext().getThreadFactory(),
replicationManager);
} else {
- bufferCache = new BufferCache(ioManager, prs, pcp, new FileMapManager(),
+ bufferCache = new BufferCache(getLocalOrCloudIoManager(), prs, pcp, new FileMapManager(),
storageProperties.getBufferCacheMaxOpenFiles(), ioQueueLen, getServiceContext().getThreadFactory());
}
@@ -359,6 +370,15 @@
}
@Override
+ public IIOManager getCloudIoManager() {
+ return cloudIoManager;
+ }
+
+ private IIOManager getLocalOrCloudIoManager() {
+ return isCloudDeployment() ? cloudIoManager : ioManager;
+ }
+
+ @Override
public StorageProperties getStorageProperties() {
return storageProperties;
}
@@ -638,4 +658,9 @@
public IDiskWriteRateLimiterProvider getDiskWriteRateLimiterProvider() {
return diskWriteRateLimiterProvider;
}
+
+ @Override
+ public boolean isCloudDeployment() {
+ return ncServiceContext.getAppConfig().getBoolean(CLOUD_DEPLOYMENT);
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/CloudToLocalStorageCachingTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/CloudToLocalStorageCachingTask.java
new file mode 100644
index 0000000..55906b6
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/CloudToLocalStorageCachingTask.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.nc.task;
+
+import java.util.Arrays;
+import java.util.Set;
+
+import org.apache.asterix.cloud.CloudIOManager;
+import org.apache.asterix.common.api.INCLifecycleTask;
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.hyracks.api.control.CcId;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.service.IControllerService;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class CloudToLocalStorageCachingTask implements INCLifecycleTask {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private static final long serialVersionUID = 1L;
+ private final Set<Integer> partitions;
+
+ public CloudToLocalStorageCachingTask(Set<Integer> partitions) {
+ this.partitions = partitions;
+ }
+
+ @Override
+ public void perform(CcId ccId, IControllerService cs) throws HyracksDataException {
+ INcApplicationContext applicationContext = (INcApplicationContext) cs.getApplicationContext();
+
+ String nodeId = applicationContext.getServiceContext().getNodeId();
+ LOGGER.info("Syncing cloud to local storage for node {}. for partitions: {}", nodeId, partitions);
+
+ CloudIOManager cloudIOManager = (CloudIOManager) applicationContext.getCloudIoManager();
+
+ // TODO(htowaileb): eager caching is disabled for now as it depends on static partitioning work
+ cloudIOManager.syncFiles(partitions);
+ }
+
+ @Override
+ public String toString() {
+ return "{ \"class\" : \"" + getClass().getSimpleName() + "\", \"partitions\" : "
+ + Arrays.toString(partitions.toArray()) + " }";
+ }
+}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
index 06005a9..4f9613d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
@@ -35,6 +35,7 @@
import org.apache.asterix.app.nc.task.BindMetadataNodeTask;
import org.apache.asterix.app.nc.task.CheckpointTask;
+import org.apache.asterix.app.nc.task.CloudToLocalStorageCachingTask;
import org.apache.asterix.app.nc.task.ExportMetadataNodeTask;
import org.apache.asterix.app.nc.task.LocalRecoveryTask;
import org.apache.asterix.app.nc.task.LocalStorageCleanupTask;
@@ -51,6 +52,7 @@
import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
import org.apache.asterix.common.api.INCLifecycleTask;
import org.apache.asterix.common.cluster.IClusterStateManager;
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.common.messaging.api.ICCMessageBroker;
@@ -83,8 +85,10 @@
private final boolean replicationEnabled;
private final IGatekeeper gatekeeper;
Map<String, Map<String, Object>> nodeSecretsMap;
+ private ICCServiceContext serviceContext;
public NcLifecycleCoordinator(ICCServiceContext serviceCtx, boolean replicationEnabled) {
+ this.serviceContext = serviceCtx;
this.messageBroker = (ICCMessageBroker) serviceCtx.getMessageBroker();
this.replicationEnabled = replicationEnabled;
this.gatekeeper =
@@ -218,6 +222,11 @@
tasks.add(new UpdateNodeStatusTask(NodeStatus.BOOTING, nodeActivePartitions));
int metadataPartitionId = clusterManager.getMetadataPartition().getPartitionId();
tasks.add(new LocalStorageCleanupTask(metadataPartitionId));
+
+ if (((ICcApplicationContext) (serviceContext.getControllerService()).getApplicationContext())
+ .isCloudDeployment()) {
+ tasks.add(new CloudToLocalStorageCachingTask(activePartitions));
+ }
if (state == SystemState.CORRUPTED) {
// need to perform local recovery for node active partitions
LocalRecoveryTask rt = new LocalRecoveryTask(nodeActivePartitions);
diff --git a/asterixdb/asterix-app/src/main/resources/cc-cloud-storage.conf b/asterixdb/asterix-app/src/main/resources/cc-cloud-storage.conf
new file mode 100644
index 0000000..2b03c9a
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/resources/cc-cloud-storage.conf
@@ -0,0 +1,68 @@
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements. See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership. The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License. You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing,
+; software distributed under the License is distributed on an
+; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+; KIND, either express or implied. See the License for the
+; specific language governing permissions and limitations
+; under the License.
+
+[nc/asterix_nc1]
+txn.log.dir=target/tmp/asterix_nc1/txnlog
+core.dump.dir=target/tmp/asterix_nc1/coredump
+iodevices=asterix_nc1/iodevice1
+iodevices=asterix_nc1/iodevice2
+nc.api.port=19004
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006
+
+[nc/asterix_nc2]
+ncservice.port=9091
+txn.log.dir=target/tmp/asterix_nc2/txnlog
+core.dump.dir=target/tmp/asterix_nc2/coredump
+iodevices=asterix_nc2/iodevice1
+iodevices=asterix_nc2/iodevice2
+nc.api.port=19005
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007
+
+[nc]
+address=127.0.0.1
+command=asterixnc
+app.class=org.apache.asterix.hyracks.bootstrap.NCApplication
+jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
+storage.buffercache.pagesize=32KB
+storage.buffercache.size=128MB
+storage.memorycomponent.globalbudget=512MB
+storage.io.scheduler=greedy
+storage.filtered.memorycomponent.max.size=16MB
+
+[cc]
+address = 127.0.0.1
+app.class=org.apache.asterix.hyracks.bootstrap.CCApplication
+heartbeat.period=2000
+heartbeat.max.misses=25
+
+[common]
+log.dir = logs/
+log.level = INFO
+compiler.cbo=false
+compiler.cbotest=true
+compiler.queryplanshape=zigzag
+compiler.framesize=32KB
+compiler.sortmemory=320KB
+compiler.groupmemory=160KB
+compiler.joinmemory=256KB
+compiler.textsearchmemory=160KB
+compiler.windowmemory=192KB
+compiler.sort.parallel=false
+compiler.internal.sanitycheck=true
+messaging.frame.size=4096
+messaging.frame.count=512
+cloud.deployment=true
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
index e8c2c1d..ccedd08 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
@@ -398,7 +398,7 @@
/**
* @return the asterix-app absolute path if found, otherwise the default user path.
*/
- private static Path getProjectPath() {
+ static Path getProjectPath() {
final String targetDir = "asterix-app";
final BiPredicate<Path, BasicFileAttributes> matcher =
(path, attributes) -> path.getFileName().toString().equals(targetDir) && path.toFile().isDirectory()
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudStorageIntegrationUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudStorageIntegrationUtil.java
new file mode 100644
index 0000000..cc69bb1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudStorageIntegrationUtil.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.api.common;
+
+import static org.apache.asterix.api.common.AsterixHyracksIntegrationUtil.LoggerHolder.LOGGER;
+import static org.apache.hyracks.util.file.FileUtil.joinPath;
+
+public class CloudStorageIntegrationUtil extends AsterixHyracksIntegrationUtil {
+
+ public static final String RESOURCES_PATH = joinPath(getProjectPath().toString(), "src", "test", "resources");
+ public static final String CONFIG_FILE = joinPath(RESOURCES_PATH, "cc-cloud-storage.conf");
+
+ public static void main(String[] args) throws Exception {
+ // CloudUtils.startS3CloudEnvironment();
+ final AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil();
+ try {
+ integrationUtil.run(Boolean.getBoolean("cleanup.start"), Boolean.getBoolean("cleanup.shutdown"),
+ System.getProperty("external.lib", ""), CONFIG_FILE);
+ } catch (Exception e) {
+ LOGGER.fatal("Unexpected exception", e);
+ System.exit(1);
+ }
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudUtils.java
new file mode 100644
index 0000000..5f13e0b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudUtils.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.api.common;
+
+import java.net.URI;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import io.findify.s3mock.S3Mock;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+
+public class CloudUtils {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private static final int MOCK_SERVER_PORT = 8001;
+ private static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:" + MOCK_SERVER_PORT;
+ private static final String CLOUD_STORAGE_BUCKET = "cloud-storage-bucket";
+ private static final String MOCK_SERVER_REGION = "us-west-2";
+ private static S3Mock s3MockServer;
+
+ private CloudUtils() {
+ throw new AssertionError("Do not instantiate");
+ }
+
+ public static void startS3CloudEnvironment() {
+ // Starting S3 mock server to be used instead of real S3 server
+ LOGGER.info("Starting S3 mock server");
+ s3MockServer = new S3Mock.Builder().withPort(MOCK_SERVER_PORT).withInMemoryBackend().build();
+ shutdownSilently();
+ try {
+ s3MockServer.start();
+ } catch (Exception ex) {
+ // it might already be started, do nothing
+ }
+ LOGGER.info("S3 mock server started successfully");
+
+ S3ClientBuilder builder = S3Client.builder();
+ URI endpoint = URI.create(MOCK_SERVER_HOSTNAME); // endpoint pointing to S3 mock server
+ builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create())
+ .endpointOverride(endpoint);
+ S3Client client = builder.build();
+ client.createBucket(CreateBucketRequest.builder().bucket(CLOUD_STORAGE_BUCKET).build());
+ LOGGER.info("Created bucket {} for cloud storage", CLOUD_STORAGE_BUCKET);
+ client.close();
+ }
+
+ private static void shutdownSilently() {
+ if (s3MockServer != null) {
+ try {
+ s3MockServer.shutdown();
+ } catch (Exception ex) {
+ // do nothing
+ }
+ }
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java
new file mode 100644
index 0000000..aaa2f38
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.cloud_storage;
+
+import java.util.Collection;
+
+import org.apache.asterix.api.common.CloudUtils;
+import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.test.runtime.LangExecutionUtil;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.MethodSorters;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Run tests in cloud deployment environment
+ */
+@RunWith(Parameterized.class)
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class CloudStorageTest {
+
+ protected TestCaseContext tcCtx;
+ private static final TestExecutor testExecutor = new TestExecutor();
+ private static final String SUITE_TESTS = "testsuite_cloud_storage.xml";
+ private static final String ONLY_TESTS = "testsuite_cloud_storage_only.xml";
+ private static final String CONFIG_FILE_NAME = "src/main/resources/cc-cloud-storage.conf";
+
+ public CloudStorageTest(TestCaseContext tcCtx) {
+ this.tcCtx = tcCtx;
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ CloudUtils.startS3CloudEnvironment();
+ LangExecutionUtil.setUp(CONFIG_FILE_NAME, testExecutor);
+ System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, CONFIG_FILE_NAME);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ LangExecutionUtil.tearDown();
+ }
+
+ @Parameters(name = "CloudStorageTest {index}: {0}")
+ public static Collection<Object[]> tests() throws Exception {
+ return LangExecutionUtil.tests(ONLY_TESTS, SUITE_TESTS);
+ }
+
+ @Test
+ public void test() throws Exception {
+ LangExecutionUtil.test(tcCtx);
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
index f721aab..a2fbb37 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
@@ -27,7 +27,7 @@
public static final String S3_REGION_PLACEHOLDER = "%region%";
public static final String S3_REGION_DEFAULT = "us-west-2";
public static final String S3_SERVICE_ENDPOINT_PLACEHOLDER = "%serviceEndpoint%";
- public static final String S3_SERVICE_ENDPOINT_DEFAULT = "http://localhost:8001";
+ public static final String S3_SERVICE_ENDPOINT_DEFAULT = "http://127.0.0.1:8001";
public static final String S3_TEMPLATE = "(\"accessKeyId\"=\"" + S3_ACCESS_KEY_ID_DEFAULT + "\"),\n"
+ "(\"secretAccessKey\"=\"" + S3_SECRET_ACCESS_KEY_DEFAULT + "\"),\n" + "(\"region\"=\""
+ S3_REGION_PLACEHOLDER + "\"),\n" + "(\"serviceEndpoint\"=\"" + S3_SERVICE_ENDPOINT_PLACEHOLDER + "\")";
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index ed167e6..0121b58 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -124,7 +124,7 @@
// Service endpoint
private static final int MOCK_SERVER_PORT = 8001;
- private static final String MOCK_SERVER_HOSTNAME = "http://localhost:" + MOCK_SERVER_PORT;
+ private static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:" + MOCK_SERVER_PORT;
// Region, bucket and definitions
private static final String MOCK_SERVER_REGION = "us-west-2";
@@ -376,7 +376,12 @@
// Starting S3 mock server to be used instead of real S3 server
LOGGER.info("Starting S3 mock server");
s3MockServer = new S3Mock.Builder().withPort(MOCK_SERVER_PORT).withInMemoryBackend().build();
- s3MockServer.start();
+ try {
+ s3MockServer.start();
+ } catch (Exception ex) {
+ // it might already be started, do nothing
+ }
+
LOGGER.info("S3 mock server started successfully");
// Create a client and add some files to the S3 mock server
diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf
new file mode 100644
index 0000000..b3206a6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf
@@ -0,0 +1,64 @@
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements. See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership. The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License. You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing,
+; software distributed under the License is distributed on an
+; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+; KIND, either express or implied. See the License for the
+; specific language governing permissions and limitations
+; under the License.
+
+[nc/asterix_nc1]
+txn.log.dir=target/tmp/asterix_nc1/txnlog
+core.dump.dir=target/tmp/asterix_nc1/coredump
+iodevices=target/tmp/asterix_nc1/iodevice1,
+iodevices=../asterix-server/target/tmp/asterix_nc1/iodevice2
+nc.api.port=19004
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006
+
+[nc/asterix_nc2]
+ncservice.port=9091
+txn.log.dir=target/tmp/asterix_nc2/txnlog
+core.dump.dir=target/tmp/asterix_nc2/coredump
+iodevices=target/tmp/asterix_nc2/iodevice1,../asterix-server/target/tmp/asterix_nc2/iodevice2
+nc.api.port=19005
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007
+
+[nc]
+credential.file=src/test/resources/security/passwd
+python.cmd.autolocate=true
+python.env=FOO=BAR=BAZ,BAR=BAZ
+address=127.0.0.1
+command=asterixnc
+app.class=org.apache.asterix.hyracks.bootstrap.NCApplication
+jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
+storage.buffercache.pagesize=32KB
+storage.buffercache.size=128MB
+storage.memorycomponent.globalbudget=512MB
+
+[cc]
+address = 127.0.0.1
+app.class=org.apache.asterix.hyracks.bootstrap.CCApplication
+heartbeat.period=2000
+heartbeat.max.misses=25
+credential.file=src/test/resources/security/passwd
+
+[common]
+log.dir = logs/
+log.level = INFO
+compiler.framesize=32KB
+compiler.sortmemory=320KB
+compiler.groupmemory=160KB
+compiler.joinmemory=256KB
+compiler.textsearchmemory=160KB
+compiler.windowmemory=192KB
+messaging.frame.size=4096
+messaging.frame.count=512
+cloud.deployment=true
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.000.ddl.sqlpp
new file mode 100644
index 0000000..45f5132
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.000.ddl.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open { id: uuid };
+
+drop dataset test if exists;
+create dataset test(test) primary key id autogenerated;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.001.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.001.update.sqlpp
new file mode 100644
index 0000000..ab0b2c3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.001.update.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+upsert into test([
+{"name": "foo", "age": 1},
+{"name": "bar", "age": 2}
+]);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.002.query.sqlpp
new file mode 100644
index 0000000..43668a9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.002.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+select name from test order by age;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.003.get.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.003.get.http
new file mode 100644
index 0000000..87cb432
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.003.get.http
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/connector?dataverseName=test&datasetName=test
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.999.ddl.sqlpp
new file mode 100644
index 0000000..dc10acd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.999.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-2/copy-2.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-2/copy-2.2.update.sqlpp
index a974a62..baae560 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-2/copy-2.2.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-2/copy-2.2.update.sqlpp
@@ -30,7 +30,7 @@
copy test1 USING S3 (
("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="playground"),
("definition"="data_dir"),
("format"="json")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
index 22a30ca..df7f788 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
@@ -30,7 +30,7 @@
("accessKeyId"="dummyAccessKey"),
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="playground"),
("definition"="json-data/reviews/single-line/json"),
("format"="json"));
@@ -40,7 +40,7 @@
("accessKeyId"="dummyAccessKey"),
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="playground"),
("definition"="json-data/reviews/multi-lines/json"),
("format"="json"));
@@ -50,7 +50,7 @@
("accessKeyId"="dummyAccessKey"),
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="playground"),
("definition"="json-data/reviews/multi-lines-with-arrays/json"),
("format"="json"));
@@ -60,7 +60,7 @@
("accessKeyId"="dummyAccessKey"),
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="playground"),
("definition"="json-data/reviews/multi-lines-with-nested-objects/json"),
("format"="json"));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp
index 13cfe8a..5164086 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp
@@ -29,7 +29,7 @@
CREATE EXTERNAL DATASET test(test) USING S3 (
("accessKeyId"="dummyAccessKey"),
("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="playground"),
("definition"="json-data/reviews/single-line/json"),
("format"="json")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp
index b8d0945..a46a43a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp
@@ -29,7 +29,7 @@
CREATE EXTERNAL DATASET test(test) USING S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="playground"),
("definition"="json-data/reviews/single-line/json"),
("format"="json")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp
index 9eda057..23306e0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp
@@ -28,7 +28,7 @@
drop dataset test if exists;
CREATE EXTERNAL DATASET test(test) USING S3 (
("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="playground"),
("definition"="json-data"),
("format"="json")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/create-with-session-token/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/create-with-session-token/test.000.ddl.sqlpp
index 3d62e15..0a42978 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/create-with-session-token/test.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/create-with-session-token/test.000.ddl.sqlpp
@@ -31,7 +31,7 @@
("secretAccessKey"="dummySecretKey"),
("sessionToken"="dummySessionToken"),
("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="playground"),
("definition"="json-data/reviews/single-line/json"),
("format"="json")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-empty/iceberg-empty.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-empty/iceberg-empty.00.ddl.sqlpp
index 127a2a9..3db9286 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-empty/iceberg-empty.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-empty/iceberg-empty.00.ddl.sqlpp
@@ -28,7 +28,7 @@
("accessKeyId"="dummyAccessKey"),
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
- ("serviceEndpoint"="http://localhost:8001"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="iceberg-container"),
("definition"="my-table-empty"),
("table-format"="apache-iceberg"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-metadata-invalid-location/iceberg-metadata-invalid-location.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-metadata-invalid-location/iceberg-metadata-invalid-location.00.ddl.sqlpp
index 5123c78..d337c1a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-metadata-invalid-location/iceberg-metadata-invalid-location.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-metadata-invalid-location/iceberg-metadata-invalid-location.00.ddl.sqlpp
@@ -28,7 +28,7 @@
("accessKeyId"="dummyAccessKey"),
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
- ("serviceEndpoint"="http://localhost:8001"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="iceberg-container"),
("definition"="my-table-invalid-path"),
("table-format"="apache-iceberg"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-metadata-specific-location/iceberg-load-selective-metadata.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-metadata-specific-location/iceberg-load-selective-metadata.00.ddl.sqlpp
index ee301bb..4505423 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-metadata-specific-location/iceberg-load-selective-metadata.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-metadata-specific-location/iceberg-load-selective-metadata.00.ddl.sqlpp
@@ -28,7 +28,7 @@
("accessKeyId"="dummyAccessKey"),
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
- ("serviceEndpoint"="http://localhost:8001"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="iceberg-container"),
("definition"="my-table#DATA_FILES"),
("table-format"="apache-iceberg"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-mixed-data-format/iceberg-mixed-data-format.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-mixed-data-format/iceberg-mixed-data-format.00.ddl.sqlpp
index f06c7ed..4f4ebd4 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-mixed-data-format/iceberg-mixed-data-format.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-mixed-data-format/iceberg-mixed-data-format.00.ddl.sqlpp
@@ -28,7 +28,7 @@
("accessKeyId"="dummyAccessKey"),
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
- ("serviceEndpoint"="http://localhost:8001"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="iceberg-container"),
("definition"="my-table-mixed-data-format"),
("table-format"="apache-iceberg"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-modified-data/iceberg-modified-data.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-modified-data/iceberg-modified-data.00.ddl.sqlpp
index 09f2849..afdcefd 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-modified-data/iceberg-modified-data.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-modified-data/iceberg-modified-data.00.ddl.sqlpp
@@ -28,7 +28,7 @@
("accessKeyId"="dummyAccessKey"),
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
- ("serviceEndpoint"="http://localhost:8001"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="iceberg-container"),
("definition"="my-table-modified-data"),
("table-format"="apache-iceberg"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-multiple-data-files/iceberg-multiple-data-files.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-multiple-data-files/iceberg-multiple-data-files.00.ddl.sqlpp
index 4e1b811..be140ea 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-multiple-data-files/iceberg-multiple-data-files.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-multiple-data-files/iceberg-multiple-data-files.00.ddl.sqlpp
@@ -28,7 +28,7 @@
("accessKeyId"="dummyAccessKey"),
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
- ("serviceEndpoint"="http://localhost:8001"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="iceberg-container"),
("definition"="my-table-multiple-data-files"),
("table-format"="apache-iceberg"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-unsupported-version/iceberg-unsupported-version.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-unsupported-version/iceberg-unsupported-version.00.ddl.sqlpp
index d37316f..71f5eba 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-unsupported-version/iceberg-unsupported-version.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-unsupported-version/iceberg-unsupported-version.00.ddl.sqlpp
@@ -28,7 +28,7 @@
("accessKeyId"="dummyAccessKey"),
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
- ("serviceEndpoint"="http://localhost:8001"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="iceberg-container"),
("definition"="my-table-format-version-2"),
("table-format"="apache-iceberg"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg/iceberg-read-from-latest-snapshot.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg/iceberg-read-from-latest-snapshot.00.ddl.sqlpp
index 526e75e..6e61b9e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg/iceberg-read-from-latest-snapshot.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg/iceberg-read-from-latest-snapshot.00.ddl.sqlpp
@@ -28,7 +28,7 @@
("accessKeyId"="dummyAccessKey"),
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
- ("serviceEndpoint"="http://localhost:8001"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="iceberg-container"),
("definition"="my-table"),
("table-format"="apache-iceberg"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/non-s3-region/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/non-s3-region/external_dataset.000.ddl.sqlpp
index 4d0941d..1b91ae8 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/non-s3-region/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/non-s3-region/external_dataset.000.ddl.sqlpp
@@ -34,7 +34,7 @@
("accessKeyId"="dummyAccessKey"),
("secretAccessKey"="dummySecretKey"),
("region"="some-new-region"),
- ("serviceEndpoint"="http://localhost:8001"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="playground"),
("definition"="json-data/reviews/single-line/json"),
("format"="json")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/parquet-anonymous-access/parquet-anonymous-access.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/parquet-anonymous-access/parquet-anonymous-access.00.ddl.sqlpp
index 2337ac4..47167c2 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/parquet-anonymous-access/parquet-anonymous-access.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/parquet-anonymous-access/parquet-anonymous-access.00.ddl.sqlpp
@@ -26,7 +26,7 @@
CREATE EXTERNAL DATASET ParquetDataset(ParquetType) USING S3 (
("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="playground"),
("definition"="parquet-data/reviews"),
("format"="parquet"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/parquet-temporary-access/parquet-temporary-access.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/parquet-temporary-access/parquet-temporary-access.00.ddl.sqlpp
index 15ec56a..ddad7f0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/parquet-temporary-access/parquet-temporary-access.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/parquet-temporary-access/parquet-temporary-access.00.ddl.sqlpp
@@ -29,7 +29,7 @@
("secretAccessKey"="dummySecretKey"),
("sessionToken"="dummySessionToken"),
("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
("container"="playground"),
("definition"="parquet-data/reviews"),
("format"="parquet"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/cloud_storage/query/result.002.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/cloud_storage/query/result.002.adm
new file mode 100644
index 0000000..9f5f27f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/cloud_storage/query/result.002.adm
@@ -0,0 +1,2 @@
+{ "name": "foo" }
+{ "name": "bar" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/cloud_storage/query/result.003.ignore b/asterixdb/asterix-app/src/test/resources/runtimets/results/cloud_storage/query/result.003.ignore
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/cloud_storage/query/result.003.ignore
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage.xml
new file mode 100644
index 0000000..5360f43
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
+ <test-group name="aws-s3-external-dataset">
+ <test-case FilePath="cloud_storage">
+ <compilation-unit name="query">
+ <output-dir compare="Text">query</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+</test-suite>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage_only.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage_only.xml
new file mode 100644
index 0000000..a28be6c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage_only.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
+ <test-group name="cloud_storage">
+ </test-group>
+</test-suite>
diff --git a/asterixdb/asterix-cloud/pom.xml b/asterixdb/asterix-cloud/pom.xml
new file mode 100644
index 0000000..542e6ae
--- /dev/null
+++ b/asterixdb/asterix-cloud/pom.xml
@@ -0,0 +1,116 @@
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>apache-asterixdb</artifactId>
+ <groupId>org.apache.asterix</groupId>
+ <version>0.9.9-SNAPSHOT</version>
+ </parent>
+ <artifactId>asterix-cloud</artifactId>
+
+ <licenses>
+ <license>
+ <name>Apache License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ <comments>A business-friendly OSS license</comments>
+ </license>
+ </licenses>
+
+ <properties>
+ <root.dir>${basedir}/..</root.dir>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ <configuration>
+ <licenses>
+ <license implementation="org.apache.rat.analysis.license.ApacheSoftwareLicense20"/>
+ </licenses>
+ <excludes combine.children="append">
+ <exclude>src/test/resources/result/**</exclude>
+ </excludes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <!-- aws s3 start -->
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>sdk-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>regions</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>auth</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3-transfer-manager</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk.crt</groupId>
+ <artifactId>aws-crt</artifactId>
+ <version>0.21.10</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.findify</groupId>
+ <artifactId>s3mock_2.12</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.typesafe.akka</groupId>
+ <artifactId>akka-http-core_2.12</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <!-- aws s3 end -->
+ </dependencies>
+</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudFileHandle.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudFileHandle.java
similarity index 89%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudFileHandle.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudFileHandle.java
index c7ed9c0..24cb0bb 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudFileHandle.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudFileHandle.java
@@ -16,15 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.control.nc.io.cloud;
+package org.apache.asterix.cloud;
import java.io.IOException;
+import org.apache.asterix.cloud.clients.ICloudBufferedWriter;
+import org.apache.asterix.cloud.clients.ICloudClient;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.control.nc.io.FileHandle;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudBufferedWriter;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudClient;
public class CloudFileHandle extends FileHandle {
private final CloudResettableInputStream inputStream;
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
similarity index 60%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudIOManager.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
index f753f4f..6f4ce69 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudIOManager.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
@@ -16,30 +16,40 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.control.nc.io.cloud;
+package org.apache.asterix.cloud;
+
+import static org.apache.asterix.common.utils.StorageConstants.*;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
+import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.channels.ClosedByInterruptException;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import org.apache.asterix.cloud.clients.CloudClientProvider;
+import org.apache.asterix.cloud.clients.CloudClientProvider.ClientType;
+import org.apache.asterix.cloud.clients.ICloudClient;
+import org.apache.asterix.cloud.clients.ICloudClientCredentialsProvider.CredentialsType;
+import org.apache.asterix.cloud.storage.CloudStorageConfigurationProvider;
+import org.apache.asterix.cloud.storage.ICloudStorageConfiguration;
+import org.apache.asterix.cloud.storage.ICloudStorageConfiguration.ConfigurationType;
import org.apache.commons.io.FileUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IFileDeviceResolver;
import org.apache.hyracks.api.io.IFileHandle;
-import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.api.util.IoUtil;
+import org.apache.hyracks.control.nc.io.FileHandle;
import org.apache.hyracks.control.nc.io.IOManager;
-import org.apache.hyracks.control.nc.io.cloud.clients.CloudClientProvider;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudClient;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -48,26 +58,25 @@
private final ICloudClient cloudClient;
private final WriteBufferProvider writeBufferProvider;
private final String bucket;
+ private IOManager localIoManager;
- // TODO(htowaileb): temporary, will need to be read from somewhere
- public static final String STORAGE_ROOT_DIR_NAME = "storage";
-
- public CloudIOManager(IIOManager ioManager, int queueSize, int ioParallelism) throws HyracksDataException {
- super((IOManager) ioManager, queueSize, ioParallelism);
-
- // TODO(htowaileb): temporary, this needs to be provided somehow
- try {
- List<String> lines = FileUtils.readLines(new File("/etc/s3"), "UTF-8");
- bucket = lines.get(0);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- cloudClient = CloudClientProvider.getClient(CloudClientProvider.ClientType.S3);
- int numOfThreads = ioManager.getIODevices().size() * ioParallelism;
+ private CloudIOManager(List<IODeviceHandle> devices, IFileDeviceResolver deviceComputer, int ioParallelism,
+ int queueSize) throws HyracksDataException {
+ super(devices, deviceComputer, ioParallelism, queueSize);
+ ICloudStorageConfiguration cloudStorageConfiguration =
+ CloudStorageConfigurationProvider.INSTANCE.getConfiguration(ConfigurationType.FILE);
+ this.bucket = cloudStorageConfiguration.getContainer();
+ cloudClient = CloudClientProvider.getClient(ClientType.S3, CredentialsType.FILE);
+ int numOfThreads = getIODevices().size() * getIoParallelism();
writeBufferProvider = new WriteBufferProvider(numOfThreads);
}
+ public CloudIOManager(IOManager ioManager) throws HyracksDataException {
+ this(ioManager.getIoDevices(), ioManager.getDeviceComputer(), ioManager.getIoParallelism(),
+ ioManager.getQueueSize());
+ this.localIoManager = ioManager;
+ }
+
public String getBucket() {
return bucket;
}
@@ -82,36 +91,34 @@
inputStream.abort();
throw e;
}
-
return writtenBytes;
}
@Override
public int doSyncWrite(IFileHandle fHandle, long offset, ByteBuffer dataArray) throws HyracksDataException {
int writtenBytes = super.doSyncWrite(fHandle, offset, dataArray);
- CloudResettableInputStream cloudInputStream = ((CloudFileHandle) fHandle).getInputStream();
+ CloudResettableInputStream inputStream = ((CloudFileHandle) fHandle).getInputStream();
try {
- cloudInputStream.write(dataArray);
+ inputStream.write(dataArray);
} catch (HyracksDataException e) {
- cloudInputStream.abort();
+ inputStream.abort();
throw e;
}
-
return writtenBytes;
}
@Override
public IFileHandle open(FileReference fileRef, FileReadWriteMode rwMode, FileSyncMode syncMode)
throws HyracksDataException {
+
CloudFileHandle fHandle = new CloudFileHandle(cloudClient, bucket, fileRef, writeBufferProvider);
if (!super.exists(fileRef) && cloudClient.exists(bucket, fileRef.getRelativePath())) {
ByteBuffer writeBuffer = writeBufferProvider.getBuffer();
try {
- // TODO: We need a proper caching mechanism
- LOGGER.info("Downloading {} from cloud storage..", fileRef.getRelativePath());
- LocalCacheUtil.download(cloudClient, this, fHandle, rwMode, syncMode, writeBuffer);
+ LOGGER.info("Downloading {} from S3..", fileRef.getRelativePath());
+ downloadFile(fHandle, rwMode, syncMode, writeBuffer);
super.close(fHandle);
- LOGGER.info("Finished downloading {} from cloud storage..", fileRef.getRelativePath());
+ LOGGER.info("Finished downloading {} from S3..", fileRef.getRelativePath());
} finally {
writeBufferProvider.recycle(writeBuffer);
}
@@ -148,6 +155,7 @@
public Set<FileReference> list(FileReference dir, FilenameFilter filter) throws HyracksDataException {
Set<String> cloudFiles = cloudClient.listObjects(bucket, dir.getRelativePath(), filter);
if (cloudFiles.isEmpty()) {
+ // TODO(htowaileb): Can we end up in a state where local has files but cloud does not?
return Collections.emptySet();
}
@@ -208,11 +216,10 @@
@Override
public long getSize(IFileHandle fileHandle) {
- if (fileHandle.getFileReference().getFile().exists()) {
- // This should always provide the correct size despite what is buffered in local disk
- return super.getSize(fileHandle);
+ if (!fileHandle.getFileReference().getFile().exists()) {
+ return cloudClient.getObjectSize(bucket, fileHandle.getFileReference().getRelativePath());
}
- return cloudClient.getObjectSize(bucket, fileHandle.getFileReference().getRelativePath());
+ return super.getSize(fileHandle);
}
@Override
@@ -222,26 +229,24 @@
cloudClient.write(bucket, fileRef.getRelativePath(), bytes);
}
- // TODO utilize locally stored files for reading
@Override
public int doSyncRead(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException {
- if (fHandle.getFileReference().getFile().exists()) {
- return super.doSyncRead(fHandle, offset, data);
- }
- return cloudClient.read(bucket, fHandle.getFileReference().getRelativePath(), offset, data);
+ return super.doSyncRead(fHandle, offset, data);
}
// TODO: We need to download this too
@Override
public byte[] readAllBytes(FileReference fileRef) throws HyracksDataException {
- if (fileRef.getFile().exists()) {
- return super.readAllBytes(fileRef);
+ if (!fileRef.getFile().exists()) {
+ // TODO(htowaileb): if it does not exist, download (lazy)
+ // TODO(htowaileb): make sure downloading the file is synchronous since many can request it at the same time
}
- return cloudClient.readAllBytes(bucket, fileRef.getRelativePath());
+ return super.readAllBytes(fileRef);
}
@Override
public void deleteDirectory(FileReference fileRef) throws HyracksDataException {
+ // TODO(htowaileb): Should we delete the cloud first?
super.deleteDirectory(fileRef);
if (!STORAGE_ROOT_DIR_NAME.equals(IoUtil.getFileNameFromPath(fileRef.getAbsolutePath()))) {
// Never delete the storage dir in cloud storage
@@ -250,16 +255,6 @@
}
@Override
- public Collection<FileReference> getMatchingFiles(FileReference root, FilenameFilter filter) {
- Set<String> paths = cloudClient.listObjects(bucket, root.getRelativePath(), filter);
- List<FileReference> fileReferences = new ArrayList<>();
- for (String path : paths) {
- fileReferences.add(new FileReference(root.getDeviceHandle(), path));
- }
- return fileReferences;
- }
-
- @Override
public boolean exists(FileReference fileRef) {
// Check if the file exists locally first as newly created files (i.e., they are empty) are not stored in cloud storage
return fileRef.getFile().exists() || cloudClient.exists(bucket, fileRef.getRelativePath());
@@ -286,4 +281,78 @@
protected void syncLocally(IFileHandle fileHandle) throws HyracksDataException {
super.sync(fileHandle, true);
}
+
+ @Override
+ public void syncFiles(Set<Integer> activePartitions) throws HyracksDataException {
+ Map<String, String> cloudToLocalStoragePaths = new HashMap<>();
+ for (Integer partition : activePartitions) {
+ String partitionToFind = PARTITION_DIR_PREFIX + partition + "/";
+ IODeviceHandle deviceHandle = getDeviceComputer().resolve(partitionToFind, getIODevices());
+
+ String cloudStoragePath = STORAGE_ROOT_DIR_NAME + "/" + partitionToFind;
+ String localStoragePath = deviceHandle.getMount().getAbsolutePath() + "/" + cloudStoragePath;
+ cloudToLocalStoragePaths.put(cloudStoragePath, localStoragePath);
+ }
+ LOGGER.info("Resolved paths to io devices: {}", cloudToLocalStoragePaths);
+ cloudClient.syncFiles(bucket, cloudToLocalStoragePaths);
+ }
+
+ // TODO(htowaileb): the localIoManager is closed by the node controller service as well, check if we need this
+ @Override
+ public void close() throws IOException {
+ cloudClient.close();
+ super.close();
+ localIoManager.close();
+ }
+
+ private void downloadFile(FileHandle fileHandle, FileReadWriteMode rwMode, FileSyncMode syncMode,
+ ByteBuffer writeBuffer) throws HyracksDataException {
+ FileReference fileRef = fileHandle.getFileReference();
+ File file = fileRef.getFile();
+
+ try (InputStream inputStream = cloudClient.getObjectStream(bucket, fileRef.getRelativePath())) {
+ FileUtils.createParentDirectories(file);
+ if (!file.createNewFile()) {
+ throw new IllegalStateException("Couldn't create local file");
+ }
+
+ fileHandle.open(rwMode, syncMode);
+ writeToFile(fileHandle, inputStream, writeBuffer);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ private void writeToFile(IFileHandle fileHandle, InputStream inStream, ByteBuffer writeBuffer)
+ throws HyracksDataException {
+ writeBuffer.clear();
+ try {
+ int position = 0;
+ long offset = 0;
+ int read;
+ while ((read = inStream.read(writeBuffer.array(), position, writeBuffer.remaining())) >= 0) {
+ position += read;
+ writeBuffer.position(position);
+ if (writeBuffer.remaining() == 0) {
+ offset += writeBufferToFile(fileHandle, writeBuffer, offset);
+ position = 0;
+ }
+ }
+
+ if (writeBuffer.position() > 0) {
+ writeBufferToFile(fileHandle, writeBuffer, offset);
+ syncLocally(fileHandle);
+ }
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ private long writeBufferToFile(IFileHandle fileHandle, ByteBuffer writeBuffer, long offset)
+ throws HyracksDataException {
+ writeBuffer.flip();
+ long written = writeLocally(fileHandle, offset, writeBuffer);
+ writeBuffer.clear();
+ return written;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudResettableInputStream.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java
similarity index 96%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudResettableInputStream.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java
index 84af84f..3a7504a 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudResettableInputStream.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java
@@ -16,14 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.control.nc.io.cloud;
+package org.apache.asterix.cloud;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
+import org.apache.asterix.cloud.clients.ICloudBufferedWriter;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudBufferedWriter;
public class CloudResettableInputStream extends InputStream {
// TODO: make configurable
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/WriteBufferProvider.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/WriteBufferProvider.java
similarity index 90%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/WriteBufferProvider.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/WriteBufferProvider.java
index 44adf45..5e49be3 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/WriteBufferProvider.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/WriteBufferProvider.java
@@ -16,9 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.control.nc.io.cloud;
+package org.apache.asterix.cloud;
-import static org.apache.hyracks.control.nc.io.cloud.CloudResettableInputStream.MIN_BUFFER_SIZE;
+import static org.apache.asterix.cloud.CloudResettableInputStream.MIN_BUFFER_SIZE;
import java.nio.ByteBuffer;
import java.util.concurrent.ArrayBlockingQueue;
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/CloudClientProvider.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
similarity index 62%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/CloudClientProvider.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
index 72aa566..d808ea5 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/CloudClientProvider.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
@@ -16,14 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.control.nc.io.cloud.clients;
+package org.apache.asterix.cloud.clients;
-import java.util.HashMap;
-
+import org.apache.asterix.cloud.clients.ICloudClientCredentialsProvider.CredentialsType;
+import org.apache.asterix.cloud.clients.aws.s3.S3CloudClient;
+import org.apache.asterix.cloud.clients.aws.s3.credentials.IS3Credentials;
+import org.apache.asterix.cloud.clients.aws.s3.credentials.S3CredentialsProvider;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.control.nc.io.cloud.clients.aws.s3.S3CloudClient;
-import org.apache.hyracks.control.nc.io.cloud.clients.azure.blob.AzureBlobCloudClient;
-import org.apache.hyracks.control.nc.io.cloud.clients.gcp.gcs.GCSCloudClient;
public class CloudClientProvider {
@@ -38,17 +37,12 @@
throw new AssertionError("do not instantiate");
}
- public static ICloudClient getClient(ClientType clientType) throws HyracksDataException {
+ public static ICloudClient getClient(ClientType clientType, CredentialsType credentialsType)
+ throws HyracksDataException {
switch (clientType) {
- case NO_OP:
- return NoOpCloudClient.INSTANCE;
case S3:
- // TODO: map should have the config already
- return new S3CloudClient(new HashMap<>());
- case AZURE_BLOB:
- return new AzureBlobCloudClient();
- case GOOGLE_CLOUD_STORAGE:
- return new GCSCloudClient();
+ IS3Credentials credentials = S3CredentialsProvider.INSTANCE.getCredentials(credentialsType);
+ return new S3CloudClient(credentials);
default:
throw HyracksDataException.create(new IllegalArgumentException("Unknown cloud client type"));
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/ICloudBufferedWriter.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudBufferedWriter.java
similarity index 95%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/ICloudBufferedWriter.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudBufferedWriter.java
index 8bb2d68..4f08111 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/ICloudBufferedWriter.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudBufferedWriter.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.control.nc.io.cloud.clients;
+package org.apache.asterix.cloud.clients;
import java.io.InputStream;
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/ICloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
similarity index 87%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/ICloudClient.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
index bb12769..fe2a2d6 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/ICloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
@@ -16,11 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.control.nc.io.cloud.clients;
+package org.apache.asterix.cloud.clients;
import java.io.FilenameFilter;
import java.io.InputStream;
import java.nio.ByteBuffer;
+import java.util.Map;
import java.util.Set;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -58,7 +59,6 @@
* @param path path
* @param offset offset
* @param buffer buffer
- * TODO(htowaileb) should this be returning the buffer position or the total amount read?
* @return returns the buffer position
*/
int read(String bucket, String path, long offset, ByteBuffer buffer) throws HyracksDataException;
@@ -125,4 +125,18 @@
* @return {@code true} if the object exists, {@code false} otherwise
*/
boolean exists(String bucket, String path);
+
+ /**
+ * Syncs files by downloading them from cloud storage to local storage
+ *
+ * @param bucket bucket to sync from
+ * @param cloudToLocalStoragePaths map of cloud storage partition to local storage path
+ * @throws HyracksDataException HyracksDataException
+ */
+ void syncFiles(String bucket, Map<String, String> cloudToLocalStoragePaths) throws HyracksDataException;
+
+ /**
+ * Performs any necessary closing and cleaning up
+ */
+ void close();
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClientCredentialsProvider.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClientCredentialsProvider.java
new file mode 100644
index 0000000..1bb94b2
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClientCredentialsProvider.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface ICloudClientCredentialsProvider {
+ enum CredentialsType {
+ FILE,
+ MOCK
+ }
+
+ ICredentials getCredentials(CredentialsType type) throws HyracksDataException;
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICredentials.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICredentials.java
new file mode 100644
index 0000000..abf5284
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICredentials.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients;
+
+public interface ICredentials {
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3BufferedWriter.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
similarity index 96%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3BufferedWriter.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
index a793f40..9104d9a 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3BufferedWriter.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
@@ -16,15 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.control.nc.io.cloud.clients.aws.s3;
+package org.apache.asterix.cloud.clients.aws.s3;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import org.apache.asterix.cloud.clients.ICloudBufferedWriter;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudBufferedWriter;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
new file mode 100644
index 0000000..b800b96
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -0,0 +1,344 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients.aws.s3;
+
+import static org.apache.asterix.cloud.clients.aws.s3.S3Utils.listS3Objects;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.asterix.cloud.clients.ICloudBufferedWriter;
+import org.apache.asterix.cloud.clients.ICloudClient;
+import org.apache.asterix.cloud.clients.aws.s3.credentials.IS3Credentials;
+import org.apache.commons.io.FileUtils;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.util.IoUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.ResponseInputStream;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
+import software.amazon.awssdk.services.s3.S3CrtAsyncClientBuilder;
+import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
+import software.amazon.awssdk.services.s3.model.Delete;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.transfer.s3.S3TransferManager;
+import software.amazon.awssdk.transfer.s3.model.CompletedDirectoryDownload;
+import software.amazon.awssdk.transfer.s3.model.DirectoryDownload;
+import software.amazon.awssdk.transfer.s3.model.DownloadDirectoryRequest;
+
+public class S3CloudClient implements ICloudClient {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private final IS3Credentials credentials;
+ private final S3Client s3Client;
+ private S3TransferManager s3TransferManager;
+
+ // TODO(htowaileb): Temporary variables, can we get this from the used instance?
+ private static final double MAX_HOST_BANDWIDTH = 10.0; // in Gbps
+
+ public S3CloudClient(IS3Credentials credentials) throws HyracksDataException {
+ this.credentials = credentials;
+ s3Client = buildClient();
+ }
+
+ private S3Client buildClient() throws HyracksDataException {
+ AwsCredentialsProvider credentialsProvider = StaticCredentialsProvider
+ .create(AwsBasicCredentials.create(credentials.getAccessKeyId(), credentials.getSecretAccessKey()));
+ S3ClientBuilder builder = S3Client.builder();
+ builder.credentialsProvider(credentialsProvider);
+ builder.region(Region.of(credentials.getRegion()));
+
+ if (credentials.getEndpoint() != null && !credentials.getEndpoint().isEmpty()) {
+ try {
+ URI uri = new URI(credentials.getEndpoint());
+ builder.endpointOverride(uri);
+ } catch (Exception ex) {
+ throw HyracksDataException.create(ex);
+ }
+ }
+ return builder.build();
+ }
+
+ @Override
+ public ICloudBufferedWriter createBufferedWriter(String bucket, String path) {
+ return new S3BufferedWriter(s3Client, bucket, path);
+ }
+
+ @Override
+ public Set<String> listObjects(String bucket, String path, FilenameFilter filter) {
+ return filterAndGet(listS3Objects(s3Client, bucket, path), filter);
+ }
+
+ @Override
+ public int read(String bucket, String path, long offset, ByteBuffer buffer) throws HyracksDataException {
+ long readTo = offset + buffer.remaining();
+ GetObjectRequest rangeGetObjectRequest =
+ GetObjectRequest.builder().range("bytes=" + offset + "-" + readTo).bucket(bucket).key(path).build();
+
+ int totalRead = 0;
+ int read = 0;
+
+ // TODO(htowaileb): add retry logic here
+ try (ResponseInputStream<GetObjectResponse> response = s3Client.getObject(rangeGetObjectRequest)) {
+ while (buffer.remaining() > 0) {
+ read = response.read(buffer.array(), buffer.position(), buffer.remaining());
+ buffer.position(buffer.position() + read);
+ totalRead += read;
+ }
+ } catch (IOException ex) {
+ throw HyracksDataException.create(ex);
+ }
+
+ if (buffer.remaining() != 0) {
+ throw new IllegalStateException("Expected buffer remaining = 0, found: " + buffer.remaining());
+ }
+ return totalRead;
+ }
+
+ @Override
+ public byte[] readAllBytes(String bucket, String path) throws HyracksDataException {
+ GetObjectRequest getReq = GetObjectRequest.builder().bucket(bucket).key(path).build();
+ try {
+ ResponseInputStream<GetObjectResponse> stream = s3Client.getObject(getReq);
+ return stream.readAllBytes();
+ } catch (NoSuchKeyException e) {
+ return null;
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ public InputStream getObjectStream(String bucket, String path) {
+ GetObjectRequest getReq = GetObjectRequest.builder().bucket(bucket).key(path).build();
+ try {
+ return s3Client.getObject(getReq);
+ } catch (NoSuchKeyException e) {
+ // This should not happen at least from the only caller of this method
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public void write(String bucket, String path, byte[] data) {
+ PutObjectRequest putReq = PutObjectRequest.builder().bucket(bucket).key(path).build();
+
+ // TODO(htowaileb): add retry logic here
+ s3Client.putObject(putReq, RequestBody.fromBytes(data));
+ }
+
+ @Override
+ public void copy(String bucket, String srcPath, FileReference destPath) {
+ List<S3Object> objects = listS3Objects(s3Client, bucket, srcPath);
+ for (S3Object object : objects) {
+ String srcKey = object.key();
+ String destKey = destPath.getChildPath(IoUtil.getFileNameFromPath(srcKey));
+ CopyObjectRequest copyReq = CopyObjectRequest.builder().sourceBucket(bucket).sourceKey(srcKey)
+ .destinationBucket(bucket).destinationKey(destKey).build();
+ s3Client.copyObject(copyReq);
+ }
+ }
+
+ @Override
+ public void deleteObject(String bucket, String path) {
+ Set<String> fileList = listObjects(bucket, path, IoUtil.NO_OP_FILTER);
+ if (fileList.isEmpty()) {
+ return;
+ }
+
+ List<ObjectIdentifier> objectIdentifiers = new ArrayList<>();
+ for (String file : fileList) {
+ objectIdentifiers.add(ObjectIdentifier.builder().key(file).build());
+ }
+ Delete delete = Delete.builder().objects(objectIdentifiers).build();
+ DeleteObjectsRequest deleteReq = DeleteObjectsRequest.builder().bucket(bucket).delete(delete).build();
+ s3Client.deleteObjects(deleteReq);
+ }
+
+ // TODO(htowaileb): Use the following cheaper request to check if an object exists
+ // https://stackoverflow.com/questions/3910071/check-file-size-on-s3-without-downloading
+ @Override
+ public long getObjectSize(String bucket, String path) {
+ List<S3Object> objects = listS3Objects(s3Client, bucket, path);
+ if (objects.isEmpty()) {
+ return 0;
+ }
+ return objects.get(0).size();
+ }
+
+ // TODO(htowaileb): Use the following cheaper request to check if an object exists
+ // https://docs.aws.amazon.com/AmazonS3/latest/userguide/example_s3_HeadObject_section.html
+ @Override
+ public boolean exists(String bucket, String path) {
+ List<S3Object> objects = listS3Objects(s3Client, bucket, path);
+ return !objects.isEmpty();
+ }
+
+ private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter filter) {
+ Set<String> files = new HashSet<>();
+ for (S3Object s3Object : contents) {
+ String path = s3Object.key();
+ if (filter.accept(null, IoUtil.getFileNameFromPath(path))) {
+ files.add(path);
+ }
+ }
+ return files;
+ }
+
+ @Override
+ public void syncFiles(String bucket, Map<String, String> cloudToLocalStoragePaths) throws HyracksDataException {
+ LOGGER.info("Syncing cloud storage to local storage started");
+
+ S3TransferManager s3TransferManager = getS3TransferManager();
+
+ List<CompletableFuture<CompletedDirectoryDownload>> downloads = new ArrayList<>();
+ cloudToLocalStoragePaths.forEach((cloudStoragePath, localStoragePath) -> {
+ DownloadDirectoryRequest.Builder builder = DownloadDirectoryRequest.builder();
+ builder.bucket(bucket);
+ builder.destination(Paths.get(localStoragePath));
+ builder.listObjectsV2RequestTransformer(l -> l.prefix(cloudStoragePath));
+
+ LOGGER.info("TransferManager started downloading from cloud \"{}\" to local storage \"{}\"",
+ cloudStoragePath, localStoragePath);
+ DirectoryDownload directoryDownload = s3TransferManager.downloadDirectory(builder.build());
+ downloads.add(directoryDownload.completionFuture());
+ });
+
+ try {
+ for (CompletableFuture<CompletedDirectoryDownload> download : downloads) {
+ download.join();
+ CompletedDirectoryDownload completedDirectoryDownload = download.get();
+
+ // if we have failed downloads with transfer manager, try to download them with GetObject
+ if (!completedDirectoryDownload.failedTransfers().isEmpty()) {
+ LOGGER.warn("TransferManager failed to download file(s), will retry to download each separately");
+ completedDirectoryDownload.failedTransfers().forEach(LOGGER::warn);
+
+ Map<String, String> failedFiles = new HashMap<>();
+ completedDirectoryDownload.failedTransfers().forEach(failed -> {
+ String cloudStoragePath = failed.request().getObjectRequest().key();
+ String localStoragePath = failed.request().destination().toAbsolutePath().toString();
+ failedFiles.put(cloudStoragePath, localStoragePath);
+ });
+ downloadFiles(bucket, failedFiles);
+ }
+ LOGGER.info("TransferManager finished downloading {} to local storage", completedDirectoryDownload);
+ }
+ } catch (ExecutionException | InterruptedException e) {
+ throw HyracksDataException.create(e);
+ }
+ LOGGER.info("Syncing cloud storage to local storage successful");
+ }
+
+ private void downloadFiles(String bucket, Map<String, String> cloudToLocalStoragePaths)
+ throws HyracksDataException {
+ byte[] buffer = new byte[8 * 1024];
+ for (Map.Entry<String, String> entry : cloudToLocalStoragePaths.entrySet()) {
+ String cloudStoragePath = entry.getKey();
+ String localStoragePath = entry.getValue();
+
+ LOGGER.info("GetObject started downloading from cloud \"{}\" to local storage \"{}\"", cloudStoragePath,
+ localStoragePath);
+
+ // TODO(htowaileb): add retry logic here
+ try {
+ File localFile = new File(localStoragePath);
+ FileUtils.createParentDirectories(localFile);
+ if (!localFile.createNewFile()) {
+ // do nothing for now, a restart has the files when trying to flush, for testing
+ //throw new IllegalStateException("Couldn't create local file");
+ }
+
+ try (InputStream inputStream = getObjectStream(bucket, cloudStoragePath);
+ FileOutputStream outputStream = new FileOutputStream(localFile)) {
+ int bytesRead;
+ while ((bytesRead = inputStream.read(buffer)) != -1) {
+ outputStream.write(buffer, 0, bytesRead);
+ }
+ }
+ } catch (IOException ex) {
+ throw HyracksDataException.create(ex);
+ }
+ LOGGER.info("GetObject successful downloading from cloud \"{}\" to local storage \"{}\"", cloudStoragePath,
+ localStoragePath);
+ }
+ }
+
+ private S3TransferManager getS3TransferManager() {
+ if (s3TransferManager != null) {
+ return s3TransferManager;
+ }
+
+ S3CrtAsyncClientBuilder builder = S3AsyncClient.crtBuilder();
+ builder.credentialsProvider(StaticCredentialsProvider
+ .create(AwsBasicCredentials.create(credentials.getAccessKeyId(), credentials.getSecretAccessKey())));
+ builder.region(Region.of(credentials.getRegion()));
+ builder.targetThroughputInGbps(MAX_HOST_BANDWIDTH);
+ builder.minimumPartSizeInBytes((long) 8 * 1024 * 1024);
+
+ if (credentials.getEndpoint() != null && !credentials.getEndpoint().isEmpty()) {
+ builder.endpointOverride(URI.create(credentials.getEndpoint()));
+ }
+
+ S3AsyncClient client = builder.build();
+ s3TransferManager = S3TransferManager.builder().s3Client(client).build();
+ return s3TransferManager;
+ }
+
+ @Override
+ public void close() {
+ if (s3Client != null) {
+ s3Client.close();
+ }
+
+ if (s3TransferManager != null) {
+ s3TransferManager.close();
+ }
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3Utils.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
similarity index 63%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3Utils.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
index e29b611..996b22d 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3Utils.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.control.nc.io.cloud.clients.aws.s3;
+package org.apache.asterix.cloud.clients.aws.s3;
import java.util.List;
@@ -55,21 +55,4 @@
}
return listObjectsResponse.contents();
}
-
- // TODO(htowaileb): Test few runs with default client and see if any failures are encountered
- // private static SdkHttpClient buildHttpClient() {
- // ApacheHttpClient.Builder apacheClientBuilder = ApacheHttpClient.builder();
- //
- // AttributeMap.Builder overriddenConfigBuilder = AttributeMap.builder();
- // overriddenConfigBuilder.put(SdkHttpConfigurationOption.MAX_CONNECTIONS, 128);
- // overriddenConfigBuilder.put(SdkHttpConfigurationOption.CONNECTION_TIMEOUT, Duration.ofMinutes(60));
- // overriddenConfigBuilder.put(SdkHttpConfigurationOption.CONNECTION_MAX_IDLE_TIMEOUT, Duration.ofMinutes(60));
- // overriddenConfigBuilder.put(SdkHttpConfigurationOption.CONNECTION_ACQUIRE_TIMEOUT, Duration.ofMinutes(60));
- // overriddenConfigBuilder.put(SdkHttpConfigurationOption.READ_TIMEOUT, Duration.ofMinutes(60));
- // overriddenConfigBuilder.put(SdkHttpConfigurationOption.WRITE_TIMEOUT, Duration.ofMinutes(60));
- // overriddenConfigBuilder.put(SdkHttpConfigurationOption.TCP_KEEPALIVE, Boolean.TRUE);
- // AttributeMap configuration = overriddenConfigBuilder.build();
- //
- // return apacheClientBuilder.buildWithDefaults(configuration);
- // }
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/FileCredentials.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/FileCredentials.java
new file mode 100644
index 0000000..342e1a1
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/FileCredentials.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients.aws.s3.credentials;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class FileCredentials implements IS3Credentials {
+
+ private final String accessKeyId;
+ private final String secretAccessKey;
+ private final String region;
+ private String endpoint;
+
+ // TODO(htowaileb): change the credential file to be json object instead of reading per line
+ public FileCredentials(File file) throws HyracksDataException {
+ if (!file.exists()) {
+ throw new IllegalStateException("No cloud configuration file found");
+ }
+
+ try {
+ List<String> lines = FileUtils.readLines(file, "UTF-8");
+ this.accessKeyId = lines.get(1);
+ this.secretAccessKey = lines.get(2);
+ this.region = lines.get(3);
+
+ if (lines.size() > 4) {
+ this.endpoint = lines.get(4);
+ }
+ } catch (IOException ex) {
+ throw HyracksDataException.create(ex);
+ }
+ }
+
+ @Override
+ public String getAccessKeyId() {
+ return accessKeyId;
+ }
+
+ @Override
+ public String getSecretAccessKey() {
+ return secretAccessKey;
+ }
+
+ @Override
+ public String getRegion() {
+ return region;
+ }
+
+ @Override
+ public String getEndpoint() {
+ return endpoint;
+ }
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/IS3Credentials.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/IS3Credentials.java
new file mode 100644
index 0000000..32da3da
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/IS3Credentials.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients.aws.s3.credentials;
+
+import org.apache.asterix.cloud.clients.ICredentials;
+
+public interface IS3Credentials extends ICredentials {
+
+ /**
+ * @return access key id
+ */
+ String getAccessKeyId();
+
+ /**
+ * @return secret access key
+ */
+ String getSecretAccessKey();
+
+ /**
+ * @return region
+ */
+ String getRegion();
+
+ /**
+ * @return endpoint
+ */
+ String getEndpoint();
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3CredentialsProvider.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3CredentialsProvider.java
new file mode 100644
index 0000000..e605ea1
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3CredentialsProvider.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients.aws.s3.credentials;
+
+import java.io.File;
+
+import org.apache.asterix.cloud.clients.ICloudClientCredentialsProvider;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class S3CredentialsProvider implements ICloudClientCredentialsProvider {
+
+ public static S3CredentialsProvider INSTANCE = new S3CredentialsProvider();
+
+ private S3CredentialsProvider() {
+ }
+
+ @Override
+ public IS3Credentials getCredentials(CredentialsType type) throws HyracksDataException {
+ switch (type) {
+ case FILE:
+ File file = new File("/etc/s3");
+ if (file.exists()) {
+ return new FileCredentials(file);
+ }
+ default:
+ return S3MockCredentials.INSTANCE;
+ }
+ }
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3MockCredentials.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3MockCredentials.java
new file mode 100644
index 0000000..56bb08a
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3MockCredentials.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients.aws.s3.credentials;
+
+public class S3MockCredentials implements IS3Credentials {
+
+ private static final String ACCESS_KEY_ID = "dummyAccessKeyId";
+ private static final String SECRET_ACCESS_KEY = "dummySecretAccessKey";
+ private static final String REGION = "us-west-2";
+ private static final String ENDPOINT = "http://127.0.0.1:8001";
+
+ public static final S3MockCredentials INSTANCE = new S3MockCredentials();
+
+ private S3MockCredentials() {
+ }
+
+ @Override
+ public String getAccessKeyId() {
+ return ACCESS_KEY_ID;
+ }
+
+ @Override
+ public String getSecretAccessKey() {
+ return SECRET_ACCESS_KEY;
+ }
+
+ @Override
+ public String getRegion() {
+ return REGION;
+ }
+
+ @Override
+ public String getEndpoint() {
+ return ENDPOINT;
+ }
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/CloudStorageConfigurationProvider.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/CloudStorageConfigurationProvider.java
new file mode 100644
index 0000000..7a94f98
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/CloudStorageConfigurationProvider.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.storage;
+
+import java.io.File;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class CloudStorageConfigurationProvider {
+
+ public static CloudStorageConfigurationProvider INSTANCE = new CloudStorageConfigurationProvider();
+
+ private CloudStorageConfigurationProvider() {
+ }
+
+ public ICloudStorageConfiguration getConfiguration(ICloudStorageConfiguration.ConfigurationType type)
+ throws HyracksDataException {
+ switch (type) {
+ case FILE:
+ File file = new File("/etc/storage");
+ if (file.exists()) {
+ return new FileCloudStorageConfiguration(file);
+ }
+ default:
+ return MockCloudStorageConfiguration.INSTANCE;
+ }
+ }
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/FileCloudStorageConfiguration.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/FileCloudStorageConfiguration.java
new file mode 100644
index 0000000..7cee20a
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/FileCloudStorageConfiguration.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.storage;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class FileCloudStorageConfiguration implements ICloudStorageConfiguration {
+
+ private final String containerName;
+ private final int storagePartitionsCount;
+
+ public FileCloudStorageConfiguration(File file) throws HyracksDataException {
+ if (!file.exists()) {
+ throw new IllegalStateException("No cloud configuration file found");
+ }
+
+ try {
+ List<String> lines = FileUtils.readLines(file, "UTF-8");
+ this.containerName = lines.get(0);
+ this.storagePartitionsCount = Integer.parseInt(lines.get(1));
+ } catch (IOException ex) {
+ throw HyracksDataException.create(ex);
+ }
+ }
+
+ @Override
+ public String getContainer() {
+ return containerName;
+ }
+
+ @Override
+ public int getPartitionsCount() {
+ return storagePartitionsCount;
+ }
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/ICloudStorageConfiguration.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/ICloudStorageConfiguration.java
new file mode 100644
index 0000000..e140c7c
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/ICloudStorageConfiguration.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.storage;
+
+public interface ICloudStorageConfiguration {
+
+ enum ConfigurationType {
+ FILE,
+ MOCK
+ }
+
+ /**
+ * @return returns the container name used for the storage
+ */
+ String getContainer();
+
+ /**
+ * @return returns the number of storage partitions
+ */
+ int getPartitionsCount();
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/MockCloudStorageConfiguration.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/MockCloudStorageConfiguration.java
new file mode 100644
index 0000000..cf4f46f
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/MockCloudStorageConfiguration.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.storage;
+
+public class MockCloudStorageConfiguration implements ICloudStorageConfiguration {
+
+ private static final String CONTAINER_NAME = "cloud-storage-container";
+ private static final int NUMBER_OF_STORAGE_PARTITIONS = 8;
+
+ public static MockCloudStorageConfiguration INSTANCE = new MockCloudStorageConfiguration();
+
+ private MockCloudStorageConfiguration() {
+
+ }
+
+ @Override
+ public String getContainer() {
+ return CONTAINER_NAME;
+ }
+
+ @Override
+ public int getPartitionsCount() {
+ return NUMBER_OF_STORAGE_PARTITIONS;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/test/java/org/apache/hyracks/control/nc/lsm/LSMTest.java b/asterixdb/asterix-cloud/src/test/java/org/apach/asterix/cloud/LSMTest.java
similarity index 91%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/test/java/org/apache/hyracks/control/nc/lsm/LSMTest.java
rename to asterixdb/asterix-cloud/src/test/java/org/apach/asterix/cloud/LSMTest.java
index d62242f..bd0ab28 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/test/java/org/apache/hyracks/control/nc/lsm/LSMTest.java
+++ b/asterixdb/asterix-cloud/src/test/java/org/apach/asterix/cloud/LSMTest.java
@@ -16,16 +16,16 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.control.nc.lsm;
+package org.apach.asterix.cloud;
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.hyracks.control.nc.io.cloud.CloudResettableInputStream;
-import org.apache.hyracks.control.nc.io.cloud.WriteBufferProvider;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudBufferedWriter;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudClient;
+import org.apache.asterix.cloud.CloudResettableInputStream;
+import org.apache.asterix.cloud.WriteBufferProvider;
+import org.apache.asterix.cloud.clients.ICloudBufferedWriter;
+import org.apache.asterix.cloud.clients.ICloudClient;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.FixMethodOrder;
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/test/java/org/apache/hyracks/control/nc/lsm/aws/s3/LSMS3Test.java b/asterixdb/asterix-cloud/src/test/java/org/apach/asterix/cloud/s3/LSMS3Test.java
similarity index 80%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/test/java/org/apache/hyracks/control/nc/lsm/aws/s3/LSMS3Test.java
rename to asterixdb/asterix-cloud/src/test/java/org/apach/asterix/cloud/s3/LSMS3Test.java
index e9d0689..2424ced 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/test/java/org/apache/hyracks/control/nc/lsm/aws/s3/LSMS3Test.java
+++ b/asterixdb/asterix-cloud/src/test/java/org/apach/asterix/cloud/s3/LSMS3Test.java
@@ -16,14 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.control.nc.lsm.aws.s3;
+package org.apach.asterix.cloud.s3;
import java.net.URI;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.hyracks.control.nc.io.cloud.clients.aws.s3.S3CloudClient;
-import org.apache.hyracks.control.nc.lsm.LSMTest;
+import org.apach.asterix.cloud.LSMTest;
+import org.apache.asterix.cloud.clients.aws.s3.S3CloudClient;
+import org.apache.asterix.cloud.clients.aws.s3.credentials.S3MockCredentials;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -40,7 +39,7 @@
private static S3Client client;
private static S3Mock s3MockServer;
private static final int MOCK_SERVER_PORT = 8001;
- private static final String MOCK_SERVER_HOSTNAME = "http://localhost:" + MOCK_SERVER_PORT;
+ private static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:" + MOCK_SERVER_PORT;
private static final String MOCK_SERVER_REGION = "us-west-2"; // does not matter the value
@BeforeClass
@@ -48,7 +47,11 @@
LOGGER.info("LSMS3Test setup");
LOGGER.info("Starting S3 mock server");
s3MockServer = new S3Mock.Builder().withPort(MOCK_SERVER_PORT).withInMemoryBackend().build();
- s3MockServer.start();
+ try {
+ s3MockServer.start();
+ } catch (Exception ex) {
+ // it might already be started, do nothing
+ }
LOGGER.info("S3 mock server started successfully");
// Create a client and add some files to the S3 mock server
@@ -62,12 +65,7 @@
client.createBucket(CreateBucketRequest.builder().bucket(PLAYGROUND_CONTAINER).build());
LOGGER.info("Client created successfully");
- Map<String, String> clientConfiguration = new HashMap<>();
- clientConfiguration.put("accessKeyId", "randomValue");
- clientConfiguration.put("secretAccessKey", "randomValue");
- clientConfiguration.put("region", "randomValue");
- clientConfiguration.put("endpoint", MOCK_SERVER_HOSTNAME);
- CLOUD_CLIENT = new S3CloudClient(clientConfiguration);
+ CLOUD_CLIENT = new S3CloudClient(S3MockCredentials.INSTANCE);
}
private static void cleanup() {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
index f4e241c..def3634 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
@@ -90,4 +90,9 @@
* @return the extension manager instance
*/
Object getExtensionManager();
+
+ /**
+ * @return true if running in cloud deployment, false otherwise.
+ */
+ boolean isCloudDeployment();
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java
index 5475b97..434d61c 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java
@@ -49,6 +49,8 @@
IIOManager getIoManager();
+ IIOManager getCloudIoManager();
+
Executor getThreadExecutor();
ITransactionSubsystem getTransactionSubsystem();
diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml
index e223d54..e068a2a 100644
--- a/asterixdb/asterix-server/pom.xml
+++ b/asterixdb/asterix-server/pom.xml
@@ -217,6 +217,10 @@
</gavs>
<noticeUrl>https://raw.githubusercontent.com/aws/aws-sdk-java-v2/2.10.83/NOTICE.txt</noticeUrl>
</override>
+ <override>
+ <gav>software.amazon.awssdk.crt:aws-crt:0.21.10</gav>
+ <noticeUrl>https://raw.githubusercontent.com/awslabs/aws-crt-java/v0.21.10/NOTICE</noticeUrl>
+ </override>
<!-- Hadoop AWS SDK -->
<override>
<gavs>
@@ -679,6 +683,7 @@
<aliasUrl>https://raw.githubusercontent.com/apache/orc/v1.8.0/LICENSE</aliasUrl>
<aliasUrl>https://raw.githubusercontent.com/RoaringBitmap/RoaringBitmap/0.9.39/LICENSE</aliasUrl>
<aliasUrl>https://raw.githubusercontent.com/JetBrains/java-annotations/master/LICENSE.txt</aliasUrl>
+ <aliasUrl>https://raw.githubusercontent.com/awslabs/aws-crt-java/v0.21.10/LICENSE</aliasUrl>
</aliasUrls>
<metric>1</metric>
</license>
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index 0d6c70b..47544bb 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -87,7 +87,7 @@
<hadoop.version>3.3.4</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
<log4j.version>2.19.0</log4j.version>
- <awsjavasdk.version>2.17.218</awsjavasdk.version>
+ <awsjavasdk.version>2.20.37</awsjavasdk.version>
<parquet.version>1.12.3</parquet.version>
<hadoop-awsjavasdk.version>1.12.402</hadoop-awsjavasdk.version>
<azureblobjavasdk.version>12.14.2</azureblobjavasdk.version>
@@ -943,6 +943,7 @@
<module>asterix-geo</module>
<module>asterix-spidersilk</module>
<module>asterix-column</module>
+ <module>asterix-cloud</module>
</modules>
<dependencyManagement>
@@ -1601,6 +1602,11 @@
<artifactId>http-client-spi</artifactId>
<version>${awsjavasdk.version}</version>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3-transfer-manager</artifactId>
+ <version>${awsjavasdk.version}</version>
+ </dependency>
<!-- Mock for AWS S3 -->
<dependency>
<groupId>io.findify</groupId>
diff --git a/asterixdb/src/main/appended-resources/supplemental-models.xml b/asterixdb/src/main/appended-resources/supplemental-models.xml
index c0555ec..3727fee 100644
--- a/asterixdb/src/main/appended-resources/supplemental-models.xml
+++ b/asterixdb/src/main/appended-resources/supplemental-models.xml
@@ -575,6 +575,21 @@
</properties>
</project>
</supplement>
+
+ <!-- software.amazon.eventstream is ALv2, and does not contain any embedded LICENSE or NOTICE file -->
+ <!-- license override not needed, ALv2 is specified in its pom.xml -->
+ <!-- see https://github.com/aws/aws-sdk-java -->
+ <supplement>
+ <project>
+ <groupId>software.amazon.awssdk.crt</groupId>
+ <artifactId>aws-crt</artifactId>
+ <properties>
+ <license.ignoreMissingEmbeddedLicense>0.21.10</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>0.21.10</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreNoticeOverride>0.21.10</license.ignoreNoticeOverride>
+ </properties>
+ </project>
+ </supplement>
<!-- AWS SDK end -->
<!-- AWS Hadoop SDK start -->
diff --git a/asterixdb/src/main/licenses/content/raw.githubusercontent.com_awslabs_aws-crt-java_v0.21.10_NOTICE.txt b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_awslabs_aws-crt-java_v0.21.10_NOTICE.txt
new file mode 100644
index 0000000..9a649a2
--- /dev/null
+++ b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_awslabs_aws-crt-java_v0.21.10_NOTICE.txt
@@ -0,0 +1,3 @@
+AWS Crt Java
+Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+SPDX-License-Identifier: Apache-2.0.
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
index 20e9ff6..cec1598 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
@@ -100,7 +100,7 @@
public void register() {
if (registrationTime != 0) {
throw new IllegalStateException(
- "File " + toString() + " was already registered at " + new Date(registrationTime));
+ "File " + this + " was already registered at " + new Date(registrationTime));
}
registrationTime = System.currentTimeMillis();
}
@@ -111,7 +111,7 @@
public void unregister() {
if (registrationTime == 0) {
- throw new IllegalStateException("File " + toString() + " wasn't registered before");
+ throw new IllegalStateException("File " + this + " wasn't registered before");
}
registrationTime = 0;
}
@@ -119,4 +119,13 @@
public boolean isCompressed() {
return false;
}
+
+ public FileReference getParent() {
+ int parentIndex = path.lastIndexOf(File.separatorChar);
+ if (parentIndex < 0) {
+ return new FileReference(dev, "");
+ }
+ String parentPath = path.substring(parentIndex);
+ return new FileReference(dev, parentPath);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
index c3ff70a..5f93c21 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
@@ -134,10 +134,11 @@
void deleteDirectory(FileReference root) throws HyracksDataException;
- // TODO: Remove and use list
Collection<FileReference> getMatchingFiles(FileReference root, FilenameFilter filter) throws HyracksDataException;
boolean exists(FileReference fileRef);
void create(FileReference fileRef) throws HyracksDataException;
+
+ void syncFiles(Set<Integer> activePartitions) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
index 223c71c..d68c291 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
@@ -95,36 +95,5 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
- <dependency>
- <groupId>software.amazon.awssdk</groupId>
- <artifactId>sdk-core</artifactId>
- </dependency>
- <dependency>
- <groupId>software.amazon.awssdk</groupId>
- <artifactId>s3</artifactId>
- </dependency>
- <dependency>
- <groupId>software.amazon.awssdk</groupId>
- <artifactId>regions</artifactId>
- </dependency>
- <dependency>
- <groupId>software.amazon.awssdk</groupId>
- <artifactId>auth</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>io.findify</groupId>
- <artifactId>s3mock_2.12</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.typesafe.akka</groupId>
- <artifactId>akka-http-core_2.12</artifactId>
- <scope>test</scope>
- </dependency>
</dependencies>
</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index 984ab08..4eebf6b 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -67,6 +67,8 @@
/*
* Finals
*/
+ private final int queueSize;
+ private final int ioParallelism;
private final ExecutorService executor;
private final BlockingQueue<IoRequest> submittedRequests;
private final BlockingQueue<IoRequest> freeRequests;
@@ -81,6 +83,8 @@
public IOManager(List<IODeviceHandle> devices, IFileDeviceResolver deviceComputer, int ioParallelism, int queueSize)
throws HyracksDataException {
this.ioDevices = Collections.unmodifiableList(devices);
+ this.queueSize = queueSize;
+ this.ioParallelism = ioParallelism;
checkDeviceValidity(devices);
workspaces = new ArrayList<>();
for (IODeviceHandle d : ioDevices) {
@@ -98,27 +102,29 @@
}
workspaceIndex = 0;
this.deviceComputer = deviceComputer;
- submittedRequests = new ArrayBlockingQueue<>(queueSize);
- freeRequests = new ArrayBlockingQueue<>(queueSize);
- int numIoThreads = ioDevices.size() * ioParallelism;
+ submittedRequests = new ArrayBlockingQueue<>(this.queueSize);
+ freeRequests = new ArrayBlockingQueue<>(this.queueSize);
+ int numIoThreads = ioDevices.size() * this.ioParallelism;
executor = Executors.newFixedThreadPool(numIoThreads);
for (int i = 0; i < numIoThreads; i++) {
executor.execute(new IoRequestHandler(i, submittedRequests));
}
}
- protected IOManager(IOManager ioManager, int queueSize, int ioParallelism) {
- this.ioDevices = ioManager.ioDevices;
- workspaces = ioManager.workspaces;
- workspaceIndex = 0;
- this.deviceComputer = ioManager.deviceComputer;
- submittedRequests = new ArrayBlockingQueue<>(queueSize);
- freeRequests = new ArrayBlockingQueue<>(queueSize);
- int numIoThreads = ioDevices.size() * ioParallelism;
- executor = Executors.newFixedThreadPool(numIoThreads);
- for (int i = 0; i < numIoThreads; i++) {
- executor.execute(new IoRequestHandler(i, submittedRequests));
- }
+ public int getQueueSize() {
+ return queueSize;
+ }
+
+ public int getIoParallelism() {
+ return ioParallelism;
+ }
+
+ public List<IODeviceHandle> getIoDevices() {
+ return ioDevices;
+ }
+
+ public IFileDeviceResolver getDeviceComputer() {
+ return deviceComputer;
}
public IoRequest getOrAllocRequest() {
@@ -570,7 +576,12 @@
@Override
public Collection<FileReference> getMatchingFiles(FileReference root, FilenameFilter filter)
throws HyracksDataException {
- Collection<File> files = IoUtil.getMatchingFiles(root.getFile().toPath(), filter);
+ File rootFile = root.getFile();
+ if (!rootFile.exists()) {
+ return Collections.emptyList();
+ }
+
+ Collection<File> files = IoUtil.getMatchingFiles(rootFile.toPath(), filter);
Set<FileReference> fileReferences = new HashSet<>();
for (File file : files) {
fileReferences.add(resolveAbsolutePath(file.getAbsolutePath()));
@@ -597,4 +608,9 @@
throw HyracksDataException.create(e);
}
}
+
+ @Override
+ public void syncFiles(Set<Integer> activePartitions) throws HyracksDataException {
+ // do nothing
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/LocalCacheUtil.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/LocalCacheUtil.java
deleted file mode 100644
index ada32b6..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/LocalCacheUtil.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.io.cloud;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IFileHandle;
-import org.apache.hyracks.api.io.IIOManager;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudClient;
-import org.apache.hyracks.util.file.FileUtil;
-
-// TODO replace with a proper caching mechanism
-public class LocalCacheUtil {
- private LocalCacheUtil() {
-
- }
-
- public static void writeToFile(FileReference fileRef, byte[] bytes) throws HyracksDataException {
- try {
- File file = fileRef.getFile();
- FileUtils.createParentDirectories(file);
- FileUtil.writeAndForce(file.toPath(), bytes);
- } catch (IOException e) {
- throw HyracksDataException.create(e);
- }
- }
-
- // TODO: replace with proper caching policy
- public static void download(ICloudClient cloudClient, CloudIOManager ioManager, CloudFileHandle fileHandle,
- IIOManager.FileReadWriteMode rwMode, IIOManager.FileSyncMode syncMode, ByteBuffer writeBuffer)
- throws HyracksDataException {
- FileReference fileRef = fileHandle.getFileReference();
- // write the file locally (the call to open is synchronized hence only one thread can perform this call)
- try (InputStream inStream = cloudClient.getObjectStream(ioManager.getBucket(), fileRef.getRelativePath())) {
- File file = fileRef.getFile();
- FileUtils.createParentDirectories(fileRef.getFile());
- if (!file.createNewFile()) {
- throw new IllegalStateException("Couldn't create local file");
- }
- fileHandle.open(rwMode, syncMode);
- LocalCacheUtil.writeToFile(ioManager, fileHandle, inStream, writeBuffer);
- } catch (IOException e) {
- throw HyracksDataException.create(e);
- }
- }
-
- private static void writeToFile(CloudIOManager ioManager, IFileHandle fileHandle, InputStream inStream,
- ByteBuffer writeBuffer) throws HyracksDataException {
- writeBuffer.clear();
- try {
- int position = 0;
- long offset = 0;
- int read;
- while ((read = inStream.read(writeBuffer.array(), position, writeBuffer.remaining())) >= 0) {
- position += read;
- writeBuffer.position(position);
- if (writeBuffer.remaining() == 0) {
- offset += writeBufferToFile(ioManager, fileHandle, writeBuffer, offset);
- position = 0;
- }
- }
-
- if (writeBuffer.position() > 0) {
- writeBufferToFile(ioManager, fileHandle, writeBuffer, offset);
- ioManager.syncLocally(fileHandle);
- }
- } catch (IOException e) {
- throw HyracksDataException.create(e);
- }
-
- }
-
- private static long writeBufferToFile(CloudIOManager ioManager, IFileHandle fileHandle, ByteBuffer writeBuffer,
- long offset) throws HyracksDataException {
- writeBuffer.flip();
- long written = ioManager.writeLocally(fileHandle, offset, writeBuffer);
- writeBuffer.clear();
- return written;
- }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/NoOpCloudClient.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/NoOpCloudClient.java
deleted file mode 100644
index 2751181..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/NoOpCloudClient.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.io.cloud.clients;
-
-import java.io.FilenameFilter;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.util.Set;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-
-public class NoOpCloudClient implements ICloudClient {
-
- public static final NoOpCloudClient INSTANCE = new NoOpCloudClient();
-
- private NoOpCloudClient() {
- // do not instantiate
- }
-
- @Override
- public ICloudBufferedWriter createBufferedWriter(String bucket, String path) {
- return null;
- }
-
- @Override
- public Set<String> listObjects(String bucket, String path, FilenameFilter filter) {
- return null;
- }
-
- @Override
- public int read(String bucket, String path, long offset, ByteBuffer buffer) throws HyracksDataException {
- return 0;
- }
-
- @Override
- public byte[] readAllBytes(String bucket, String path) throws HyracksDataException {
- return new byte[0];
- }
-
- @Override
- public InputStream getObjectStream(String bucket, String path) {
- return null;
- }
-
- @Override
- public void write(String bucket, String path, byte[] data) {
-
- }
-
- @Override
- public void copy(String bucket, String srcPath, FileReference destPath) {
-
- }
-
- @Override
- public void deleteObject(String bucket, String path) {
-
- }
-
- @Override
- public long getObjectSize(String bucket, String path) {
- return 0;
- }
-
- @Override
- public boolean exists(String bucket, String path) {
- return false;
- }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3CloudClient.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3CloudClient.java
deleted file mode 100644
index af1441e..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3CloudClient.java
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.io.cloud.clients.aws.s3;
-
-import static org.apache.hyracks.control.nc.io.cloud.clients.aws.s3.S3Utils.listS3Objects;
-
-import java.io.File;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.util.IoUtil;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudBufferedWriter;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudClient;
-
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import software.amazon.awssdk.core.ResponseInputStream;
-import software.amazon.awssdk.core.sync.RequestBody;
-import software.amazon.awssdk.regions.Region;
-import software.amazon.awssdk.services.s3.S3Client;
-import software.amazon.awssdk.services.s3.S3ClientBuilder;
-import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
-import software.amazon.awssdk.services.s3.model.Delete;
-import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
-import software.amazon.awssdk.services.s3.model.GetObjectRequest;
-import software.amazon.awssdk.services.s3.model.GetObjectResponse;
-import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
-import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
-import software.amazon.awssdk.services.s3.model.PutObjectRequest;
-import software.amazon.awssdk.services.s3.model.S3Object;
-
-public class S3CloudClient implements ICloudClient {
-
- private static final String ACCESS_KEY_ID_FIELD = "accessKeyId";
- private static final String SECRET_ACCESS_KEY_FIELD = "secretAccessKey";
- private static final String REGION_FIELD = "region";
- private final static String ENDPOINT_FIELD = "endpoint";
-
- private final S3Client s3Client;
-
- // TODO fix the throws exception
- public S3CloudClient(Map<String, String> clientConfiguration) throws HyracksDataException {
- setClientConfig(clientConfiguration); // TODO: remove later, this is temporary
- s3Client = buildClient(clientConfiguration);
- }
-
- private S3Client buildClient(Map<String, String> clientConfiguration) throws HyracksDataException {
- String accessKeyId = clientConfiguration.get(ACCESS_KEY_ID_FIELD);
- String secretAccessKey = clientConfiguration.get(SECRET_ACCESS_KEY_FIELD);
- String region = clientConfiguration.get(REGION_FIELD);
- String endpoint = clientConfiguration.get(ENDPOINT_FIELD);
-
- AwsCredentialsProvider credentialsProvider =
- StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey));
- S3ClientBuilder builder = S3Client.builder();
- builder.credentialsProvider(credentialsProvider);
- builder.region(Region.of(region));
-
- if (endpoint != null) {
- try {
- URI uri = new URI(endpoint);
- builder.endpointOverride(uri);
- } catch (Exception ex) {
- throw HyracksDataException.create(ex);
- }
- }
- return builder.build();
- }
-
- // TODO: temporarily setting the client config, this should be provided
- private void setClientConfig(Map<String, String> clientConfiguration) throws HyracksDataException {
- if (!clientConfiguration.isEmpty()) {
- return;
- }
-
- try {
- List<String> lines = FileUtils.readLines(new File("/etc/s3"), "UTF-8");
- String accessKeyId = lines.get(1);
- String secretAccessKey = lines.get(2);
- String region = lines.get(3);
-
- clientConfiguration.put(ACCESS_KEY_ID_FIELD, accessKeyId);
- clientConfiguration.put(SECRET_ACCESS_KEY_FIELD, secretAccessKey);
- clientConfiguration.put(REGION_FIELD, region);
-
- if (lines.size() > 4) {
- String serviceEndpoint = lines.get(4);
- clientConfiguration.put(ENDPOINT_FIELD, serviceEndpoint);
- }
- } catch (IOException ex) {
- throw HyracksDataException.create(ex);
- }
- }
-
- @Override
- public ICloudBufferedWriter createBufferedWriter(String bucket, String path) {
- return new S3BufferedWriter(s3Client, bucket, path);
- }
-
- @Override
- public Set<String> listObjects(String bucket, String path, FilenameFilter filter) {
- return filterAndGet(listS3Objects(s3Client, bucket, path), filter);
- }
-
- @Override
- public int read(String bucket, String path, long offset, ByteBuffer buffer) throws HyracksDataException {
- long readTo = offset + buffer.remaining();
- GetObjectRequest rangeGetObjectRequest =
- GetObjectRequest.builder().range("bytes=" + offset + "-" + readTo).bucket(bucket).key(path).build();
-
- int totalRead = 0;
- int read = 0;
-
- // TODO(htowaileb): add retry logic here
- try (ResponseInputStream<GetObjectResponse> response = s3Client.getObject(rangeGetObjectRequest)) {
- while (buffer.remaining() > 0) {
- read = response.read(buffer.array(), buffer.position(), buffer.remaining());
- buffer.position(buffer.position() + read);
- totalRead += read;
- }
- } catch (IOException ex) {
- throw HyracksDataException.create(ex);
- }
-
- if (buffer.remaining() != 0) {
- throw new IllegalStateException("Expected buffer remaining = 0, found: " + buffer.remaining());
- }
- return totalRead;
- }
-
- @Override
- public byte[] readAllBytes(String bucket, String path) throws HyracksDataException {
- GetObjectRequest getReq = GetObjectRequest.builder().bucket(bucket).key(path).build();
- try {
- ResponseInputStream<GetObjectResponse> stream = s3Client.getObject(getReq);
- return stream.readAllBytes();
- } catch (NoSuchKeyException e) {
- return null;
- } catch (IOException e) {
- throw HyracksDataException.create(e);
- }
- }
-
- @Override
- public InputStream getObjectStream(String bucket, String path) {
- GetObjectRequest getReq = GetObjectRequest.builder().bucket(bucket).key(path).build();
- try {
- return s3Client.getObject(getReq);
- } catch (NoSuchKeyException e) {
- // This should not happen at least from the only caller of this method
- throw new IllegalStateException(e);
- }
- }
-
- @Override
- public void write(String bucket, String path, byte[] data) {
- PutObjectRequest putReq = PutObjectRequest.builder().bucket(bucket).key(path).build();
-
- // TODO(htowaileb): add retry logic here
- s3Client.putObject(putReq, RequestBody.fromBytes(data));
- }
-
- @Override
- public void copy(String bucket, String srcPath, FileReference destPath) {
- List<S3Object> objects = listS3Objects(s3Client, bucket, srcPath);
- for (S3Object object : objects) {
- String srcKey = object.key();
- String destKey = destPath.getChildPath(IoUtil.getFileNameFromPath(srcKey));
- CopyObjectRequest copyReq = CopyObjectRequest.builder().sourceBucket(bucket).sourceKey(srcKey)
- .destinationBucket(bucket).destinationKey(destKey).build();
- s3Client.copyObject(copyReq);
- }
- }
-
- @Override
- public void deleteObject(String bucket, String path) {
- Set<String> fileList = listObjects(bucket, path, IoUtil.NO_OP_FILTER);
- if (fileList.isEmpty()) {
- return;
- }
-
- List<ObjectIdentifier> objectIdentifiers = new ArrayList<>();
- for (String file : fileList) {
- objectIdentifiers.add(ObjectIdentifier.builder().key(file).build());
- }
- Delete delete = Delete.builder().objects(objectIdentifiers).build();
- DeleteObjectsRequest deleteReq = DeleteObjectsRequest.builder().bucket(bucket).delete(delete).build();
- s3Client.deleteObjects(deleteReq);
- }
-
- @Override
- public long getObjectSize(String bucket, String path) {
- List<S3Object> objects = listS3Objects(s3Client, bucket, path);
- if (objects.isEmpty()) {
- return 0;
- }
- return objects.get(0).size();
- }
-
- @Override
- public boolean exists(String bucket, String path) {
- List<S3Object> objects = listS3Objects(s3Client, bucket, path);
- return !objects.isEmpty();
- }
-
- private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter filter) {
- Set<String> files = new HashSet<>();
- for (S3Object s3Object : contents) {
- String path = s3Object.key();
- if (filter.accept(null, IoUtil.getFileNameFromPath(path))) {
- files.add(path);
- }
- }
- return files;
- }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/azure/blob/AzureBlobCloudClient.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/azure/blob/AzureBlobCloudClient.java
deleted file mode 100644
index 6dfe05a..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/azure/blob/AzureBlobCloudClient.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.io.cloud.clients.azure.blob;
-
-import java.io.FilenameFilter;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.util.Set;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudBufferedWriter;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudClient;
-
-public class AzureBlobCloudClient implements ICloudClient {
-
- public AzureBlobCloudClient() {
- throw new IllegalStateException("NYI");
- }
-
- @Override
- public ICloudBufferedWriter createBufferedWriter(String bucket, String path) {
- return null;
- }
-
- @Override
- public Set<String> listObjects(String bucket, String path, FilenameFilter filter) {
- return null;
- }
-
- @Override
- public int read(String bucket, String path, long offset, ByteBuffer buffer) throws HyracksDataException {
- return 0;
- }
-
- @Override
- public byte[] readAllBytes(String bucket, String path) throws HyracksDataException {
- return new byte[0];
- }
-
- @Override
- public InputStream getObjectStream(String bucket, String path) {
- return null;
- }
-
- @Override
- public void write(String bucket, String path, byte[] data) {
-
- }
-
- @Override
- public void copy(String bucket, String srcPath, FileReference destPath) {
-
- }
-
- @Override
- public void deleteObject(String bucket, String path) {
-
- }
-
- @Override
- public long getObjectSize(String bucket, String path) {
- return 0;
- }
-
- @Override
- public boolean exists(String bucket, String path) {
- return false;
- }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/gcp/gcs/GCSCloudClient.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/gcp/gcs/GCSCloudClient.java
deleted file mode 100644
index a9a6f06..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/gcp/gcs/GCSCloudClient.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.io.cloud.clients.gcp.gcs;
-
-import java.io.FilenameFilter;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.util.Set;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudBufferedWriter;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudClient;
-
-public class GCSCloudClient implements ICloudClient {
-
- public GCSCloudClient() {
- throw new IllegalStateException("NYI");
- }
-
- @Override
- public ICloudBufferedWriter createBufferedWriter(String bucket, String path) {
- return null;
- }
-
- @Override
- public Set<String> listObjects(String bucket, String path, FilenameFilter filter) {
- return null;
- }
-
- @Override
- public int read(String bucket, String path, long offset, ByteBuffer buffer) throws HyracksDataException {
- return 0;
- }
-
- @Override
- public byte[] readAllBytes(String bucket, String path) throws HyracksDataException {
- return new byte[0];
- }
-
- @Override
- public InputStream getObjectStream(String bucket, String path) {
- return null;
- }
-
- @Override
- public void write(String bucket, String path, byte[] data) {
-
- }
-
- @Override
- public void copy(String bucket, String srcPath, FileReference destPath) {
-
- }
-
- @Override
- public void deleteObject(String bucket, String path) {
-
- }
-
- @Override
- public long getObjectSize(String bucket, String path) {
- return 0;
- }
-
- @Override
- public boolean exists(String bucket, String path) {
- return false;
- }
-}
diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml
index c1b8bfc..348b5bf 100644
--- a/hyracks-fullstack/pom.xml
+++ b/hyracks-fullstack/pom.xml
@@ -77,7 +77,6 @@
<jackson.version>2.14.1</jackson.version>
<jackson-databind.version>${jackson.version}</jackson-databind.version>
<netty.version>4.1.87.Final</netty.version>
- <awsjavasdk.version>2.17.218</awsjavasdk.version>
<implementation.title>Apache Hyracks and Algebricks - ${project.name}</implementation.title>
<implementation.url>https://asterixdb.apache.org/</implementation.url>
@@ -481,38 +480,6 @@
<artifactId>jetty-util-ajax</artifactId>
<version>9.4.48.v20220622</version>
</dependency>
- <dependency>
- <groupId>software.amazon.awssdk</groupId>
- <artifactId>sdk-core</artifactId>
- <version>${awsjavasdk.version}</version>
- </dependency>
- <dependency>
- <groupId>software.amazon.awssdk</groupId>
- <artifactId>s3</artifactId>
- <version>${awsjavasdk.version}</version>
- </dependency>
- <dependency>
- <groupId>software.amazon.awssdk</groupId>
- <artifactId>regions</artifactId>
- <version>${awsjavasdk.version}</version>
- </dependency>
- <dependency>
- <groupId>software.amazon.awssdk</groupId>
- <artifactId>auth</artifactId>
- <version>${awsjavasdk.version}</version>
- </dependency>
- <dependency>
- <groupId>io.findify</groupId>
- <artifactId>s3mock_2.12</artifactId>
- <version>0.2.5</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.typesafe.akka</groupId>
- <artifactId>akka-http-core_2.12</artifactId>
- <version>10.1.0</version>
- <scope>test</scope>
- </dependency>
</dependencies>
</dependencyManagement>
<build>