diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml
index 6e8224d..a0ff91a 100644
--- a/asterixdb/asterix-app/pom.xml
+++ b/asterixdb/asterix-app/pom.xml
@@ -780,6 +780,11 @@
       <artifactId>hyracks-storage-am-lsm-invertedindex</artifactId>
     </dependency>
     <dependency>
+      <groupId>org.apache.asterix</groupId>
+      <artifactId>asterix-cloud</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>postgresql</artifactId>
       <scope>test</scope>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
index e4247f0..3fad2d1 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
@@ -18,6 +18,8 @@
  */
 package org.apache.asterix.app.cc;
 
+import static org.apache.hyracks.control.common.controllers.ControllerConfig.Option.CLOUD_DEPLOYMENT;
+
 import java.io.IOException;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.Supplier;
@@ -366,4 +368,9 @@
     public IDataPartitioningProvider getDataPartitioningProvider() {
         return dataPartitioningProvider;
     }
+
+    @Override
+    public boolean isCloudDeployment() {
+        return ccServiceCtx.getAppConfig().getBoolean(CLOUD_DEPLOYMENT);
+    }
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index a46522e..f9646b0 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -18,6 +18,8 @@
  */
 package org.apache.asterix.app.nc;
 
+import static org.apache.hyracks.control.common.controllers.ControllerConfig.Option.CLOUD_DEPLOYMENT;
+
 import java.io.IOException;
 import java.rmi.RemoteException;
 import java.rmi.server.UnicastRemoteObject;
@@ -28,6 +30,7 @@
 
 import org.apache.asterix.active.ActiveManager;
 import org.apache.asterix.app.result.ResultReader;
+import org.apache.asterix.cloud.CloudIOManager;
 import org.apache.asterix.common.api.IConfigValidator;
 import org.apache.asterix.common.api.IConfigValidatorFactory;
 import org.apache.asterix.common.api.ICoordinationService;
@@ -91,6 +94,7 @@
 import org.apache.hyracks.client.result.ResultSet;
 import org.apache.hyracks.control.common.controllers.NCConfig;
 import org.apache.hyracks.control.nc.NodeControllerService;
+import org.apache.hyracks.control.nc.io.IOManager;
 import org.apache.hyracks.ipc.impl.HyracksConnection;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
@@ -143,6 +147,7 @@
     private ILSMIOOperationScheduler lsmIOScheduler;
     private PersistentLocalResourceRepository localResourceRepository;
     private IIOManager ioManager;
+    private IIOManager cloudIoManager;
     private boolean isShuttingdown;
     private ActiveManager activeManager;
     private IReplicationChannel replicationChannel;
@@ -185,6 +190,9 @@
             IConfigValidatorFactory configValidatorFactory, IReplicationStrategyFactory replicationStrategyFactory,
             boolean initialRun) throws IOException {
         ioManager = getServiceContext().getIoManager();
+        if (isCloudDeployment()) {
+            cloudIoManager = new CloudIOManager((IOManager) ioManager);
+        }
         int ioQueueLen = getServiceContext().getAppConfig().getInt(NCConfig.Option.IO_QUEUE_SIZE);
         threadExecutor =
                 MaintainedThreadNameExecutorService.newCachedThreadPool(getServiceContext().getThreadFactory());
@@ -238,6 +246,9 @@
         receptionist = receptionistFactory.create();
 
         if (replicationProperties.isReplicationEnabled()) {
+            if (LOGGER.isInfoEnabled()) {
+                LOGGER.info("Replication is enabled");
+            }
             replicationManager = new ReplicationManager(this, replicationStrategyFactory, replicationProperties);
 
             //pass replication manager to replication required object
@@ -250,11 +261,11 @@
             //initialize replication channel
             replicationChannel = new ReplicationChannel(this);
 
-            bufferCache = new BufferCache(ioManager, prs, pcp, new FileMapManager(),
+            bufferCache = new BufferCache(getLocalOrCloudIoManager(), prs, pcp, new FileMapManager(),
                     storageProperties.getBufferCacheMaxOpenFiles(), ioQueueLen, getServiceContext().getThreadFactory(),
                     replicationManager);
         } else {
-            bufferCache = new BufferCache(ioManager, prs, pcp, new FileMapManager(),
+            bufferCache = new BufferCache(getLocalOrCloudIoManager(), prs, pcp, new FileMapManager(),
                     storageProperties.getBufferCacheMaxOpenFiles(), ioQueueLen, getServiceContext().getThreadFactory());
         }
 
@@ -359,6 +370,15 @@
     }
 
     @Override
+    public IIOManager getCloudIoManager() {
+        return cloudIoManager;
+    }
+
+    private IIOManager getLocalOrCloudIoManager() {
+        return isCloudDeployment() ? cloudIoManager : ioManager;
+    }
+
+    @Override
     public StorageProperties getStorageProperties() {
         return storageProperties;
     }
@@ -638,4 +658,9 @@
     public IDiskWriteRateLimiterProvider getDiskWriteRateLimiterProvider() {
         return diskWriteRateLimiterProvider;
     }
+
+    @Override
+    public boolean isCloudDeployment() {
+        return ncServiceContext.getAppConfig().getBoolean(CLOUD_DEPLOYMENT);
+    }
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/CloudToLocalStorageCachingTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/CloudToLocalStorageCachingTask.java
new file mode 100644
index 0000000..55906b6
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/CloudToLocalStorageCachingTask.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.nc.task;
+
+import java.util.Arrays;
+import java.util.Set;
+
+import org.apache.asterix.cloud.CloudIOManager;
+import org.apache.asterix.common.api.INCLifecycleTask;
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.hyracks.api.control.CcId;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.service.IControllerService;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class CloudToLocalStorageCachingTask implements INCLifecycleTask {
+
+    private static final Logger LOGGER = LogManager.getLogger();
+
+    private static final long serialVersionUID = 1L;
+    private final Set<Integer> partitions;
+
+    public CloudToLocalStorageCachingTask(Set<Integer> partitions) {
+        this.partitions = partitions;
+    }
+
+    @Override
+    public void perform(CcId ccId, IControllerService cs) throws HyracksDataException {
+        INcApplicationContext applicationContext = (INcApplicationContext) cs.getApplicationContext();
+
+        String nodeId = applicationContext.getServiceContext().getNodeId();
+        LOGGER.info("Syncing cloud to local storage for node {}. for partitions: {}", nodeId, partitions);
+
+        CloudIOManager cloudIOManager = (CloudIOManager) applicationContext.getCloudIoManager();
+
+        // TODO(htowaileb): eager caching is disabled for now as it depends on static partitioning work
+        cloudIOManager.syncFiles(partitions);
+    }
+
+    @Override
+    public String toString() {
+        return "{ \"class\" : \"" + getClass().getSimpleName() + "\", \"partitions\" : "
+                + Arrays.toString(partitions.toArray()) + " }";
+    }
+}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
index 06005a9..4f9613d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
@@ -35,6 +35,7 @@
 
 import org.apache.asterix.app.nc.task.BindMetadataNodeTask;
 import org.apache.asterix.app.nc.task.CheckpointTask;
+import org.apache.asterix.app.nc.task.CloudToLocalStorageCachingTask;
 import org.apache.asterix.app.nc.task.ExportMetadataNodeTask;
 import org.apache.asterix.app.nc.task.LocalRecoveryTask;
 import org.apache.asterix.app.nc.task.LocalStorageCleanupTask;
@@ -51,6 +52,7 @@
 import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
 import org.apache.asterix.common.api.INCLifecycleTask;
 import org.apache.asterix.common.cluster.IClusterStateManager;
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
 import org.apache.asterix.common.exceptions.ErrorCode;
 import org.apache.asterix.common.exceptions.RuntimeDataException;
 import org.apache.asterix.common.messaging.api.ICCMessageBroker;
@@ -83,8 +85,10 @@
     private final boolean replicationEnabled;
     private final IGatekeeper gatekeeper;
     Map<String, Map<String, Object>> nodeSecretsMap;
+    private ICCServiceContext serviceContext;
 
     public NcLifecycleCoordinator(ICCServiceContext serviceCtx, boolean replicationEnabled) {
+        this.serviceContext = serviceCtx;
         this.messageBroker = (ICCMessageBroker) serviceCtx.getMessageBroker();
         this.replicationEnabled = replicationEnabled;
         this.gatekeeper =
@@ -218,6 +222,11 @@
         tasks.add(new UpdateNodeStatusTask(NodeStatus.BOOTING, nodeActivePartitions));
         int metadataPartitionId = clusterManager.getMetadataPartition().getPartitionId();
         tasks.add(new LocalStorageCleanupTask(metadataPartitionId));
+
+        if (((ICcApplicationContext) (serviceContext.getControllerService()).getApplicationContext())
+                .isCloudDeployment()) {
+            tasks.add(new CloudToLocalStorageCachingTask(activePartitions));
+        }
         if (state == SystemState.CORRUPTED) {
             // need to perform local recovery for node active partitions
             LocalRecoveryTask rt = new LocalRecoveryTask(nodeActivePartitions);
diff --git a/asterixdb/asterix-app/src/main/resources/cc-cloud-storage.conf b/asterixdb/asterix-app/src/main/resources/cc-cloud-storage.conf
new file mode 100644
index 0000000..2b03c9a
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/resources/cc-cloud-storage.conf
@@ -0,0 +1,68 @@
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements.  See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership.  The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License.  You may obtain a copy of the License at
+;
+;   http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing,
+; software distributed under the License is distributed on an
+; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+; KIND, either express or implied.  See the License for the
+; specific language governing permissions and limitations
+; under the License.
+
+[nc/asterix_nc1]
+txn.log.dir=target/tmp/asterix_nc1/txnlog
+core.dump.dir=target/tmp/asterix_nc1/coredump
+iodevices=asterix_nc1/iodevice1
+iodevices=asterix_nc1/iodevice2
+nc.api.port=19004
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006
+
+[nc/asterix_nc2]
+ncservice.port=9091
+txn.log.dir=target/tmp/asterix_nc2/txnlog
+core.dump.dir=target/tmp/asterix_nc2/coredump
+iodevices=asterix_nc2/iodevice1
+iodevices=asterix_nc2/iodevice2
+nc.api.port=19005
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007
+
+[nc]
+address=127.0.0.1
+command=asterixnc
+app.class=org.apache.asterix.hyracks.bootstrap.NCApplication
+jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
+storage.buffercache.pagesize=32KB
+storage.buffercache.size=128MB
+storage.memorycomponent.globalbudget=512MB
+storage.io.scheduler=greedy
+storage.filtered.memorycomponent.max.size=16MB
+
+[cc]
+address = 127.0.0.1
+app.class=org.apache.asterix.hyracks.bootstrap.CCApplication
+heartbeat.period=2000
+heartbeat.max.misses=25
+
+[common]
+log.dir = logs/
+log.level = INFO
+compiler.cbo=false
+compiler.cbotest=true
+compiler.queryplanshape=zigzag
+compiler.framesize=32KB
+compiler.sortmemory=320KB
+compiler.groupmemory=160KB
+compiler.joinmemory=256KB
+compiler.textsearchmemory=160KB
+compiler.windowmemory=192KB
+compiler.sort.parallel=false
+compiler.internal.sanitycheck=true
+messaging.frame.size=4096
+messaging.frame.count=512
+cloud.deployment=true
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
index e8c2c1d..ccedd08 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
@@ -398,7 +398,7 @@
     /**
      * @return the asterix-app absolute path if found, otherwise the default user path.
      */
-    private static Path getProjectPath() {
+    static Path getProjectPath() {
         final String targetDir = "asterix-app";
         final BiPredicate<Path, BasicFileAttributes> matcher =
                 (path, attributes) -> path.getFileName().toString().equals(targetDir) && path.toFile().isDirectory()
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudStorageIntegrationUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudStorageIntegrationUtil.java
new file mode 100644
index 0000000..cc69bb1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudStorageIntegrationUtil.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.api.common;
+
+import static org.apache.asterix.api.common.AsterixHyracksIntegrationUtil.LoggerHolder.LOGGER;
+import static org.apache.hyracks.util.file.FileUtil.joinPath;
+
+public class CloudStorageIntegrationUtil extends AsterixHyracksIntegrationUtil {
+
+    public static final String RESOURCES_PATH = joinPath(getProjectPath().toString(), "src", "test", "resources");
+    public static final String CONFIG_FILE = joinPath(RESOURCES_PATH, "cc-cloud-storage.conf");
+
+    public static void main(String[] args) throws Exception {
+        //        CloudUtils.startS3CloudEnvironment();
+        final AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil();
+        try {
+            integrationUtil.run(Boolean.getBoolean("cleanup.start"), Boolean.getBoolean("cleanup.shutdown"),
+                    System.getProperty("external.lib", ""), CONFIG_FILE);
+        } catch (Exception e) {
+            LOGGER.fatal("Unexpected exception", e);
+            System.exit(1);
+        }
+    }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudUtils.java
new file mode 100644
index 0000000..5f13e0b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudUtils.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.api.common;
+
+import java.net.URI;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import io.findify.s3mock.S3Mock;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+
+public class CloudUtils {
+
+    private static final Logger LOGGER = LogManager.getLogger();
+
+    private static final int MOCK_SERVER_PORT = 8001;
+    private static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:" + MOCK_SERVER_PORT;
+    private static final String CLOUD_STORAGE_BUCKET = "cloud-storage-bucket";
+    private static final String MOCK_SERVER_REGION = "us-west-2";
+    private static S3Mock s3MockServer;
+
+    private CloudUtils() {
+        throw new AssertionError("Do not instantiate");
+    }
+
+    public static void startS3CloudEnvironment() {
+        // Starting S3 mock server to be used instead of real S3 server
+        LOGGER.info("Starting S3 mock server");
+        s3MockServer = new S3Mock.Builder().withPort(MOCK_SERVER_PORT).withInMemoryBackend().build();
+        shutdownSilently();
+        try {
+            s3MockServer.start();
+        } catch (Exception ex) {
+            // it might already be started, do nothing
+        }
+        LOGGER.info("S3 mock server started successfully");
+
+        S3ClientBuilder builder = S3Client.builder();
+        URI endpoint = URI.create(MOCK_SERVER_HOSTNAME); // endpoint pointing to S3 mock server
+        builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create())
+                .endpointOverride(endpoint);
+        S3Client client = builder.build();
+        client.createBucket(CreateBucketRequest.builder().bucket(CLOUD_STORAGE_BUCKET).build());
+        LOGGER.info("Created bucket {} for cloud storage", CLOUD_STORAGE_BUCKET);
+        client.close();
+    }
+
+    private static void shutdownSilently() {
+        if (s3MockServer != null) {
+            try {
+                s3MockServer.shutdown();
+            } catch (Exception ex) {
+                // do nothing
+            }
+        }
+    }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java
new file mode 100644
index 0000000..aaa2f38
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.cloud_storage;
+
+import java.util.Collection;
+
+import org.apache.asterix.api.common.CloudUtils;
+import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.test.runtime.LangExecutionUtil;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.MethodSorters;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Run tests in cloud deployment environment
+ */
+@RunWith(Parameterized.class)
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class CloudStorageTest {
+
+    protected TestCaseContext tcCtx;
+    private static final TestExecutor testExecutor = new TestExecutor();
+    private static final String SUITE_TESTS = "testsuite_cloud_storage.xml";
+    private static final String ONLY_TESTS = "testsuite_cloud_storage_only.xml";
+    private static final String CONFIG_FILE_NAME = "src/main/resources/cc-cloud-storage.conf";
+
+    public CloudStorageTest(TestCaseContext tcCtx) {
+        this.tcCtx = tcCtx;
+    }
+
+    @Before
+    public void setUp() throws Exception {
+        CloudUtils.startS3CloudEnvironment();
+        LangExecutionUtil.setUp(CONFIG_FILE_NAME, testExecutor);
+        System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, CONFIG_FILE_NAME);
+    }
+
+    @AfterClass
+    public static void tearDown() throws Exception {
+        LangExecutionUtil.tearDown();
+    }
+
+    @Parameters(name = "CloudStorageTest {index}: {0}")
+    public static Collection<Object[]> tests() throws Exception {
+        return LangExecutionUtil.tests(ONLY_TESTS, SUITE_TESTS);
+    }
+
+    @Test
+    public void test() throws Exception {
+        LangExecutionUtil.test(tcCtx);
+    }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
index f721aab..a2fbb37 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
@@ -27,7 +27,7 @@
     public static final String S3_REGION_PLACEHOLDER = "%region%";
     public static final String S3_REGION_DEFAULT = "us-west-2";
     public static final String S3_SERVICE_ENDPOINT_PLACEHOLDER = "%serviceEndpoint%";
-    public static final String S3_SERVICE_ENDPOINT_DEFAULT = "http://localhost:8001";
+    public static final String S3_SERVICE_ENDPOINT_DEFAULT = "http://127.0.0.1:8001";
     public static final String S3_TEMPLATE = "(\"accessKeyId\"=\"" + S3_ACCESS_KEY_ID_DEFAULT + "\"),\n"
             + "(\"secretAccessKey\"=\"" + S3_SECRET_ACCESS_KEY_DEFAULT + "\"),\n" + "(\"region\"=\""
             + S3_REGION_PLACEHOLDER + "\"),\n" + "(\"serviceEndpoint\"=\"" + S3_SERVICE_ENDPOINT_PLACEHOLDER + "\")";
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index ed167e6..0121b58 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -124,7 +124,7 @@
 
     // Service endpoint
     private static final int MOCK_SERVER_PORT = 8001;
-    private static final String MOCK_SERVER_HOSTNAME = "http://localhost:" + MOCK_SERVER_PORT;
+    private static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:" + MOCK_SERVER_PORT;
 
     // Region, bucket and definitions
     private static final String MOCK_SERVER_REGION = "us-west-2";
@@ -376,7 +376,12 @@
         // Starting S3 mock server to be used instead of real S3 server
         LOGGER.info("Starting S3 mock server");
         s3MockServer = new S3Mock.Builder().withPort(MOCK_SERVER_PORT).withInMemoryBackend().build();
-        s3MockServer.start();
+        try {
+            s3MockServer.start();
+        } catch (Exception ex) {
+            // it might already be started, do nothing
+        }
+
         LOGGER.info("S3 mock server started successfully");
 
         // Create a client and add some files to the S3 mock server
diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf
new file mode 100644
index 0000000..b3206a6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf
@@ -0,0 +1,64 @@
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements.  See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership.  The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License.  You may obtain a copy of the License at
+;
+;   http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing,
+; software distributed under the License is distributed on an
+; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+; KIND, either express or implied.  See the License for the
+; specific language governing permissions and limitations
+; under the License.
+
+[nc/asterix_nc1]
+txn.log.dir=target/tmp/asterix_nc1/txnlog
+core.dump.dir=target/tmp/asterix_nc1/coredump
+iodevices=target/tmp/asterix_nc1/iodevice1,
+iodevices=../asterix-server/target/tmp/asterix_nc1/iodevice2
+nc.api.port=19004
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006
+
+[nc/asterix_nc2]
+ncservice.port=9091
+txn.log.dir=target/tmp/asterix_nc2/txnlog
+core.dump.dir=target/tmp/asterix_nc2/coredump
+iodevices=target/tmp/asterix_nc2/iodevice1,../asterix-server/target/tmp/asterix_nc2/iodevice2
+nc.api.port=19005
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007
+
+[nc]
+credential.file=src/test/resources/security/passwd
+python.cmd.autolocate=true
+python.env=FOO=BAR=BAZ,BAR=BAZ
+address=127.0.0.1
+command=asterixnc
+app.class=org.apache.asterix.hyracks.bootstrap.NCApplication
+jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
+storage.buffercache.pagesize=32KB
+storage.buffercache.size=128MB
+storage.memorycomponent.globalbudget=512MB
+
+[cc]
+address = 127.0.0.1
+app.class=org.apache.asterix.hyracks.bootstrap.CCApplication
+heartbeat.period=2000
+heartbeat.max.misses=25
+credential.file=src/test/resources/security/passwd
+
+[common]
+log.dir = logs/
+log.level = INFO
+compiler.framesize=32KB
+compiler.sortmemory=320KB
+compiler.groupmemory=160KB
+compiler.joinmemory=256KB
+compiler.textsearchmemory=160KB
+compiler.windowmemory=192KB
+messaging.frame.size=4096
+messaging.frame.count=512
+cloud.deployment=true
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.000.ddl.sqlpp
new file mode 100644
index 0000000..45f5132
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.000.ddl.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open { id: uuid };
+
+drop dataset test if exists;
+create dataset test(test) primary key id autogenerated;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.001.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.001.update.sqlpp
new file mode 100644
index 0000000..ab0b2c3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.001.update.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+upsert into test([
+{"name": "foo", "age": 1},
+{"name": "bar", "age": 2}
+]);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.002.query.sqlpp
new file mode 100644
index 0000000..43668a9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.002.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+select name from test order by age;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.003.get.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.003.get.http
new file mode 100644
index 0000000..87cb432
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.003.get.http
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/connector?dataverseName=test&datasetName=test
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.999.ddl.sqlpp
new file mode 100644
index 0000000..dc10acd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/query/test.999.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-2/copy-2.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-2/copy-2.2.update.sqlpp
index a974a62..baae560 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-2/copy-2.2.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-2/copy-2.2.update.sqlpp
@@ -30,7 +30,7 @@
 
 copy test1 USING S3 (
 ("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
 ("container"="playground"),
 ("definition"="data_dir"),
 ("format"="json")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
index 22a30ca..df7f788 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
@@ -30,7 +30,7 @@
 ("accessKeyId"="dummyAccessKey"),
 ("secretAccessKey"="dummySecretKey"),
 ("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
 ("container"="playground"),
 ("definition"="json-data/reviews/single-line/json"),
 ("format"="json"));
@@ -40,7 +40,7 @@
 ("accessKeyId"="dummyAccessKey"),
 ("secretAccessKey"="dummySecretKey"),
 ("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
 ("container"="playground"),
 ("definition"="json-data/reviews/multi-lines/json"),
 ("format"="json"));
@@ -50,7 +50,7 @@
 ("accessKeyId"="dummyAccessKey"),
 ("secretAccessKey"="dummySecretKey"),
 ("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
 ("container"="playground"),
 ("definition"="json-data/reviews/multi-lines-with-arrays/json"),
 ("format"="json"));
@@ -60,7 +60,7 @@
 ("accessKeyId"="dummyAccessKey"),
 ("secretAccessKey"="dummySecretKey"),
 ("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
 ("container"="playground"),
 ("definition"="json-data/reviews/multi-lines-with-nested-objects/json"),
 ("format"="json"));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp
index 13cfe8a..5164086 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp
@@ -29,7 +29,7 @@
 CREATE EXTERNAL DATASET test(test) USING S3 (
 ("accessKeyId"="dummyAccessKey"),
 ("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
 ("container"="playground"),
 ("definition"="json-data/reviews/single-line/json"),
 ("format"="json")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp
index b8d0945..a46a43a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp
@@ -29,7 +29,7 @@
 CREATE EXTERNAL DATASET test(test) USING S3 (
 ("secretAccessKey"="dummySecretKey"),
 ("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
 ("container"="playground"),
 ("definition"="json-data/reviews/single-line/json"),
 ("format"="json")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp
index 9eda057..23306e0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp
@@ -28,7 +28,7 @@
 drop dataset test if exists;
 CREATE EXTERNAL DATASET test(test) USING S3 (
 ("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
 ("container"="playground"),
 ("definition"="json-data"),
 ("format"="json")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/create-with-session-token/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/create-with-session-token/test.000.ddl.sqlpp
index 3d62e15..0a42978 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/create-with-session-token/test.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/create-with-session-token/test.000.ddl.sqlpp
@@ -31,7 +31,7 @@
 ("secretAccessKey"="dummySecretKey"),
 ("sessionToken"="dummySessionToken"),
 ("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
 ("container"="playground"),
 ("definition"="json-data/reviews/single-line/json"),
 ("format"="json")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-empty/iceberg-empty.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-empty/iceberg-empty.00.ddl.sqlpp
index 127a2a9..3db9286 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-empty/iceberg-empty.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-empty/iceberg-empty.00.ddl.sqlpp
@@ -28,7 +28,7 @@
  ("accessKeyId"="dummyAccessKey"),
  ("secretAccessKey"="dummySecretKey"),
  ("region"="us-west-2"),
- ("serviceEndpoint"="http://localhost:8001"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
  ("container"="iceberg-container"),
  ("definition"="my-table-empty"),
  ("table-format"="apache-iceberg"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-metadata-invalid-location/iceberg-metadata-invalid-location.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-metadata-invalid-location/iceberg-metadata-invalid-location.00.ddl.sqlpp
index 5123c78..d337c1a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-metadata-invalid-location/iceberg-metadata-invalid-location.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-metadata-invalid-location/iceberg-metadata-invalid-location.00.ddl.sqlpp
@@ -28,7 +28,7 @@
  ("accessKeyId"="dummyAccessKey"),
  ("secretAccessKey"="dummySecretKey"),
  ("region"="us-west-2"),
- ("serviceEndpoint"="http://localhost:8001"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
  ("container"="iceberg-container"),
  ("definition"="my-table-invalid-path"),
  ("table-format"="apache-iceberg"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-metadata-specific-location/iceberg-load-selective-metadata.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-metadata-specific-location/iceberg-load-selective-metadata.00.ddl.sqlpp
index ee301bb..4505423 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-metadata-specific-location/iceberg-load-selective-metadata.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-metadata-specific-location/iceberg-load-selective-metadata.00.ddl.sqlpp
@@ -28,7 +28,7 @@
  ("accessKeyId"="dummyAccessKey"),
  ("secretAccessKey"="dummySecretKey"),
  ("region"="us-west-2"),
- ("serviceEndpoint"="http://localhost:8001"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
  ("container"="iceberg-container"),
  ("definition"="my-table#DATA_FILES"),
  ("table-format"="apache-iceberg"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-mixed-data-format/iceberg-mixed-data-format.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-mixed-data-format/iceberg-mixed-data-format.00.ddl.sqlpp
index f06c7ed..4f4ebd4 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-mixed-data-format/iceberg-mixed-data-format.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-mixed-data-format/iceberg-mixed-data-format.00.ddl.sqlpp
@@ -28,7 +28,7 @@
  ("accessKeyId"="dummyAccessKey"),
  ("secretAccessKey"="dummySecretKey"),
  ("region"="us-west-2"),
- ("serviceEndpoint"="http://localhost:8001"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
  ("container"="iceberg-container"),
  ("definition"="my-table-mixed-data-format"),
  ("table-format"="apache-iceberg"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-modified-data/iceberg-modified-data.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-modified-data/iceberg-modified-data.00.ddl.sqlpp
index 09f2849..afdcefd 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-modified-data/iceberg-modified-data.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-modified-data/iceberg-modified-data.00.ddl.sqlpp
@@ -28,7 +28,7 @@
  ("accessKeyId"="dummyAccessKey"),
  ("secretAccessKey"="dummySecretKey"),
  ("region"="us-west-2"),
- ("serviceEndpoint"="http://localhost:8001"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
  ("container"="iceberg-container"),
  ("definition"="my-table-modified-data"),
  ("table-format"="apache-iceberg"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-multiple-data-files/iceberg-multiple-data-files.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-multiple-data-files/iceberg-multiple-data-files.00.ddl.sqlpp
index 4e1b811..be140ea 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-multiple-data-files/iceberg-multiple-data-files.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-multiple-data-files/iceberg-multiple-data-files.00.ddl.sqlpp
@@ -28,7 +28,7 @@
  ("accessKeyId"="dummyAccessKey"),
  ("secretAccessKey"="dummySecretKey"),
  ("region"="us-west-2"),
- ("serviceEndpoint"="http://localhost:8001"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
  ("container"="iceberg-container"),
  ("definition"="my-table-multiple-data-files"),
  ("table-format"="apache-iceberg"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-unsupported-version/iceberg-unsupported-version.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-unsupported-version/iceberg-unsupported-version.00.ddl.sqlpp
index d37316f..71f5eba 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-unsupported-version/iceberg-unsupported-version.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg-unsupported-version/iceberg-unsupported-version.00.ddl.sqlpp
@@ -28,7 +28,7 @@
  ("accessKeyId"="dummyAccessKey"),
  ("secretAccessKey"="dummySecretKey"),
  ("region"="us-west-2"),
- ("serviceEndpoint"="http://localhost:8001"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
  ("container"="iceberg-container"),
  ("definition"="my-table-format-version-2"),
  ("table-format"="apache-iceberg"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg/iceberg-read-from-latest-snapshot.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg/iceberg-read-from-latest-snapshot.00.ddl.sqlpp
index 526e75e..6e61b9e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg/iceberg-read-from-latest-snapshot.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/iceberg/iceberg-read-from-latest-snapshot.00.ddl.sqlpp
@@ -28,7 +28,7 @@
  ("accessKeyId"="dummyAccessKey"),
  ("secretAccessKey"="dummySecretKey"),
  ("region"="us-west-2"),
- ("serviceEndpoint"="http://localhost:8001"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
  ("container"="iceberg-container"),
  ("definition"="my-table"),
  ("table-format"="apache-iceberg"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/non-s3-region/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/non-s3-region/external_dataset.000.ddl.sqlpp
index 4d0941d..1b91ae8 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/non-s3-region/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/non-s3-region/external_dataset.000.ddl.sqlpp
@@ -34,7 +34,7 @@
     ("accessKeyId"="dummyAccessKey"),
     ("secretAccessKey"="dummySecretKey"),
     ("region"="some-new-region"),
-    ("serviceEndpoint"="http://localhost:8001"),
+    ("serviceEndpoint"="http://127.0.0.1:8001"),
     ("container"="playground"),
     ("definition"="json-data/reviews/single-line/json"),
     ("format"="json")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/parquet-anonymous-access/parquet-anonymous-access.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/parquet-anonymous-access/parquet-anonymous-access.00.ddl.sqlpp
index 2337ac4..47167c2 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/parquet-anonymous-access/parquet-anonymous-access.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/parquet-anonymous-access/parquet-anonymous-access.00.ddl.sqlpp
@@ -26,7 +26,7 @@
 
 CREATE EXTERNAL DATASET ParquetDataset(ParquetType) USING S3 (
 ("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
 ("container"="playground"),
 ("definition"="parquet-data/reviews"),
 ("format"="parquet"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/parquet-temporary-access/parquet-temporary-access.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/parquet-temporary-access/parquet-temporary-access.00.ddl.sqlpp
index 15ec56a..ddad7f0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/parquet-temporary-access/parquet-temporary-access.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/parquet-temporary-access/parquet-temporary-access.00.ddl.sqlpp
@@ -29,7 +29,7 @@
 ("secretAccessKey"="dummySecretKey"),
 ("sessionToken"="dummySessionToken"),
 ("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
+("serviceEndpoint"="http://127.0.0.1:8001"),
 ("container"="playground"),
 ("definition"="parquet-data/reviews"),
 ("format"="parquet"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/cloud_storage/query/result.002.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/cloud_storage/query/result.002.adm
new file mode 100644
index 0000000..9f5f27f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/cloud_storage/query/result.002.adm
@@ -0,0 +1,2 @@
+{ "name": "foo" }
+{ "name": "bar" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/cloud_storage/query/result.003.ignore b/asterixdb/asterix-app/src/test/resources/runtimets/results/cloud_storage/query/result.003.ignore
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/cloud_storage/query/result.003.ignore
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage.xml
new file mode 100644
index 0000000..5360f43
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements.  See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership.  The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License.  You may obtain a copy of the License at
+ !
+ !   http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied.  See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
+  <test-group name="aws-s3-external-dataset">
+    <test-case FilePath="cloud_storage">
+      <compilation-unit name="query">
+        <output-dir compare="Text">query</output-dir>
+      </compilation-unit>
+    </test-case>
+  </test-group>
+</test-suite>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage_only.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage_only.xml
new file mode 100644
index 0000000..a28be6c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage_only.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements.  See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership.  The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License.  You may obtain a copy of the License at
+ !
+ !   http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied.  See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
+  <test-group name="cloud_storage">
+  </test-group>
+</test-suite>
diff --git a/asterixdb/asterix-cloud/pom.xml b/asterixdb/asterix-cloud/pom.xml
new file mode 100644
index 0000000..542e6ae
--- /dev/null
+++ b/asterixdb/asterix-cloud/pom.xml
@@ -0,0 +1,116 @@
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements.  See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership.  The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License.  You may obtain a copy of the License at
+ !
+ !   http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied.  See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <artifactId>apache-asterixdb</artifactId>
+    <groupId>org.apache.asterix</groupId>
+    <version>0.9.9-SNAPSHOT</version>
+  </parent>
+  <artifactId>asterix-cloud</artifactId>
+
+  <licenses>
+    <license>
+      <name>Apache License, Version 2.0</name>
+      <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+      <distribution>repo</distribution>
+      <comments>A business-friendly OSS license</comments>
+    </license>
+  </licenses>
+
+  <properties>
+    <root.dir>${basedir}/..</root.dir>
+  </properties>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>default</id>
+            <phase>validate</phase>
+            <goals>
+              <goal>check</goal>
+            </goals>
+            <configuration>
+              <licenses>
+                <license implementation="org.apache.rat.analysis.license.ApacheSoftwareLicense20"/>
+              </licenses>
+              <excludes combine.children="append">
+                <exclude>src/test/resources/result/**</exclude>
+              </excludes>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.asterix</groupId>
+      <artifactId>asterix-common</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <!-- aws s3 start -->
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>sdk-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>s3</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>regions</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>auth</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>s3-transfer-manager</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk.crt</groupId>
+      <artifactId>aws-crt</artifactId>
+      <version>0.21.10</version>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.findify</groupId>
+      <artifactId>s3mock_2.12</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.typesafe.akka</groupId>
+      <artifactId>akka-http-core_2.12</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <!-- aws s3 end -->
+  </dependencies>
+</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudFileHandle.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudFileHandle.java
similarity index 89%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudFileHandle.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudFileHandle.java
index c7ed9c0..24cb0bb 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudFileHandle.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudFileHandle.java
@@ -16,15 +16,15 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.hyracks.control.nc.io.cloud;
+package org.apache.asterix.cloud;
 
 import java.io.IOException;
 
+import org.apache.asterix.cloud.clients.ICloudBufferedWriter;
+import org.apache.asterix.cloud.clients.ICloudClient;
 import org.apache.hyracks.api.io.FileReference;
 import org.apache.hyracks.api.io.IIOManager;
 import org.apache.hyracks.control.nc.io.FileHandle;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudBufferedWriter;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudClient;
 
 public class CloudFileHandle extends FileHandle {
     private final CloudResettableInputStream inputStream;
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
similarity index 60%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudIOManager.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
index f753f4f..6f4ce69 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudIOManager.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
@@ -16,30 +16,40 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.hyracks.control.nc.io.cloud;
+package org.apache.asterix.cloud;
+
+import static org.apache.asterix.common.utils.StorageConstants.*;
 
 import java.io.File;
 import java.io.FilenameFilter;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.nio.channels.ClosedByInterruptException;
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
+import org.apache.asterix.cloud.clients.CloudClientProvider;
+import org.apache.asterix.cloud.clients.CloudClientProvider.ClientType;
+import org.apache.asterix.cloud.clients.ICloudClient;
+import org.apache.asterix.cloud.clients.ICloudClientCredentialsProvider.CredentialsType;
+import org.apache.asterix.cloud.storage.CloudStorageConfigurationProvider;
+import org.apache.asterix.cloud.storage.ICloudStorageConfiguration;
+import org.apache.asterix.cloud.storage.ICloudStorageConfiguration.ConfigurationType;
 import org.apache.commons.io.FileUtils;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IFileDeviceResolver;
 import org.apache.hyracks.api.io.IFileHandle;
-import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.io.IODeviceHandle;
 import org.apache.hyracks.api.util.IoUtil;
+import org.apache.hyracks.control.nc.io.FileHandle;
 import org.apache.hyracks.control.nc.io.IOManager;
-import org.apache.hyracks.control.nc.io.cloud.clients.CloudClientProvider;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudClient;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -48,26 +58,25 @@
     private final ICloudClient cloudClient;
     private final WriteBufferProvider writeBufferProvider;
     private final String bucket;
+    private IOManager localIoManager;
 
-    // TODO(htowaileb): temporary, will need to be read from somewhere
-    public static final String STORAGE_ROOT_DIR_NAME = "storage";
-
-    public CloudIOManager(IIOManager ioManager, int queueSize, int ioParallelism) throws HyracksDataException {
-        super((IOManager) ioManager, queueSize, ioParallelism);
-
-        // TODO(htowaileb): temporary, this needs to be provided somehow
-        try {
-            List<String> lines = FileUtils.readLines(new File("/etc/s3"), "UTF-8");
-            bucket = lines.get(0);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-
-        cloudClient = CloudClientProvider.getClient(CloudClientProvider.ClientType.S3);
-        int numOfThreads = ioManager.getIODevices().size() * ioParallelism;
+    private CloudIOManager(List<IODeviceHandle> devices, IFileDeviceResolver deviceComputer, int ioParallelism,
+            int queueSize) throws HyracksDataException {
+        super(devices, deviceComputer, ioParallelism, queueSize);
+        ICloudStorageConfiguration cloudStorageConfiguration =
+                CloudStorageConfigurationProvider.INSTANCE.getConfiguration(ConfigurationType.FILE);
+        this.bucket = cloudStorageConfiguration.getContainer();
+        cloudClient = CloudClientProvider.getClient(ClientType.S3, CredentialsType.FILE);
+        int numOfThreads = getIODevices().size() * getIoParallelism();
         writeBufferProvider = new WriteBufferProvider(numOfThreads);
     }
 
+    public CloudIOManager(IOManager ioManager) throws HyracksDataException {
+        this(ioManager.getIoDevices(), ioManager.getDeviceComputer(), ioManager.getIoParallelism(),
+                ioManager.getQueueSize());
+        this.localIoManager = ioManager;
+    }
+
     public String getBucket() {
         return bucket;
     }
@@ -82,36 +91,34 @@
             inputStream.abort();
             throw e;
         }
-
         return writtenBytes;
     }
 
     @Override
     public int doSyncWrite(IFileHandle fHandle, long offset, ByteBuffer dataArray) throws HyracksDataException {
         int writtenBytes = super.doSyncWrite(fHandle, offset, dataArray);
-        CloudResettableInputStream cloudInputStream = ((CloudFileHandle) fHandle).getInputStream();
+        CloudResettableInputStream inputStream = ((CloudFileHandle) fHandle).getInputStream();
         try {
-            cloudInputStream.write(dataArray);
+            inputStream.write(dataArray);
         } catch (HyracksDataException e) {
-            cloudInputStream.abort();
+            inputStream.abort();
             throw e;
         }
-
         return writtenBytes;
     }
 
     @Override
     public IFileHandle open(FileReference fileRef, FileReadWriteMode rwMode, FileSyncMode syncMode)
             throws HyracksDataException {
+
         CloudFileHandle fHandle = new CloudFileHandle(cloudClient, bucket, fileRef, writeBufferProvider);
         if (!super.exists(fileRef) && cloudClient.exists(bucket, fileRef.getRelativePath())) {
             ByteBuffer writeBuffer = writeBufferProvider.getBuffer();
             try {
-                // TODO: We need a proper caching mechanism
-                LOGGER.info("Downloading {} from cloud storage..", fileRef.getRelativePath());
-                LocalCacheUtil.download(cloudClient, this, fHandle, rwMode, syncMode, writeBuffer);
+                LOGGER.info("Downloading {} from S3..", fileRef.getRelativePath());
+                downloadFile(fHandle, rwMode, syncMode, writeBuffer);
                 super.close(fHandle);
-                LOGGER.info("Finished downloading {} from cloud storage..", fileRef.getRelativePath());
+                LOGGER.info("Finished downloading {} from S3..", fileRef.getRelativePath());
             } finally {
                 writeBufferProvider.recycle(writeBuffer);
             }
@@ -148,6 +155,7 @@
     public Set<FileReference> list(FileReference dir, FilenameFilter filter) throws HyracksDataException {
         Set<String> cloudFiles = cloudClient.listObjects(bucket, dir.getRelativePath(), filter);
         if (cloudFiles.isEmpty()) {
+            // TODO(htowaileb): Can we end up in a state where local has files but cloud does not?
             return Collections.emptySet();
         }
 
@@ -208,11 +216,10 @@
 
     @Override
     public long getSize(IFileHandle fileHandle) {
-        if (fileHandle.getFileReference().getFile().exists()) {
-            // This should always provide the correct size despite what is buffered in local disk
-            return super.getSize(fileHandle);
+        if (!fileHandle.getFileReference().getFile().exists()) {
+            return cloudClient.getObjectSize(bucket, fileHandle.getFileReference().getRelativePath());
         }
-        return cloudClient.getObjectSize(bucket, fileHandle.getFileReference().getRelativePath());
+        return super.getSize(fileHandle);
     }
 
     @Override
@@ -222,26 +229,24 @@
         cloudClient.write(bucket, fileRef.getRelativePath(), bytes);
     }
 
-    // TODO utilize locally stored files for reading
     @Override
     public int doSyncRead(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException {
-        if (fHandle.getFileReference().getFile().exists()) {
-            return super.doSyncRead(fHandle, offset, data);
-        }
-        return cloudClient.read(bucket, fHandle.getFileReference().getRelativePath(), offset, data);
+        return super.doSyncRead(fHandle, offset, data);
     }
 
     // TODO: We need to download this too
     @Override
     public byte[] readAllBytes(FileReference fileRef) throws HyracksDataException {
-        if (fileRef.getFile().exists()) {
-            return super.readAllBytes(fileRef);
+        if (!fileRef.getFile().exists()) {
+            // TODO(htowaileb): if it does not exist, download (lazy)
+            // TODO(htowaileb): make sure downloading the file is synchronous since many can request it at the same time
         }
-        return cloudClient.readAllBytes(bucket, fileRef.getRelativePath());
+        return super.readAllBytes(fileRef);
     }
 
     @Override
     public void deleteDirectory(FileReference fileRef) throws HyracksDataException {
+        // TODO(htowaileb): Should we delete the cloud first?
         super.deleteDirectory(fileRef);
         if (!STORAGE_ROOT_DIR_NAME.equals(IoUtil.getFileNameFromPath(fileRef.getAbsolutePath()))) {
             // Never delete the storage dir in cloud storage
@@ -250,16 +255,6 @@
     }
 
     @Override
-    public Collection<FileReference> getMatchingFiles(FileReference root, FilenameFilter filter) {
-        Set<String> paths = cloudClient.listObjects(bucket, root.getRelativePath(), filter);
-        List<FileReference> fileReferences = new ArrayList<>();
-        for (String path : paths) {
-            fileReferences.add(new FileReference(root.getDeviceHandle(), path));
-        }
-        return fileReferences;
-    }
-
-    @Override
     public boolean exists(FileReference fileRef) {
         // Check if the file exists locally first as newly created files (i.e., they are empty) are not stored in cloud storage
         return fileRef.getFile().exists() || cloudClient.exists(bucket, fileRef.getRelativePath());
@@ -286,4 +281,78 @@
     protected void syncLocally(IFileHandle fileHandle) throws HyracksDataException {
         super.sync(fileHandle, true);
     }
+
+    @Override
+    public void syncFiles(Set<Integer> activePartitions) throws HyracksDataException {
+        Map<String, String> cloudToLocalStoragePaths = new HashMap<>();
+        for (Integer partition : activePartitions) {
+            String partitionToFind = PARTITION_DIR_PREFIX + partition + "/";
+            IODeviceHandle deviceHandle = getDeviceComputer().resolve(partitionToFind, getIODevices());
+
+            String cloudStoragePath = STORAGE_ROOT_DIR_NAME + "/" + partitionToFind;
+            String localStoragePath = deviceHandle.getMount().getAbsolutePath() + "/" + cloudStoragePath;
+            cloudToLocalStoragePaths.put(cloudStoragePath, localStoragePath);
+        }
+        LOGGER.info("Resolved paths to io devices: {}", cloudToLocalStoragePaths);
+        cloudClient.syncFiles(bucket, cloudToLocalStoragePaths);
+    }
+
+    // TODO(htowaileb): the localIoManager is closed by the node controller service as well, check if we need this
+    @Override
+    public void close() throws IOException {
+        cloudClient.close();
+        super.close();
+        localIoManager.close();
+    }
+
+    private void downloadFile(FileHandle fileHandle, FileReadWriteMode rwMode, FileSyncMode syncMode,
+            ByteBuffer writeBuffer) throws HyracksDataException {
+        FileReference fileRef = fileHandle.getFileReference();
+        File file = fileRef.getFile();
+
+        try (InputStream inputStream = cloudClient.getObjectStream(bucket, fileRef.getRelativePath())) {
+            FileUtils.createParentDirectories(file);
+            if (!file.createNewFile()) {
+                throw new IllegalStateException("Couldn't create local file");
+            }
+
+            fileHandle.open(rwMode, syncMode);
+            writeToFile(fileHandle, inputStream, writeBuffer);
+        } catch (IOException e) {
+            throw HyracksDataException.create(e);
+        }
+    }
+
+    private void writeToFile(IFileHandle fileHandle, InputStream inStream, ByteBuffer writeBuffer)
+            throws HyracksDataException {
+        writeBuffer.clear();
+        try {
+            int position = 0;
+            long offset = 0;
+            int read;
+            while ((read = inStream.read(writeBuffer.array(), position, writeBuffer.remaining())) >= 0) {
+                position += read;
+                writeBuffer.position(position);
+                if (writeBuffer.remaining() == 0) {
+                    offset += writeBufferToFile(fileHandle, writeBuffer, offset);
+                    position = 0;
+                }
+            }
+
+            if (writeBuffer.position() > 0) {
+                writeBufferToFile(fileHandle, writeBuffer, offset);
+                syncLocally(fileHandle);
+            }
+        } catch (IOException e) {
+            throw HyracksDataException.create(e);
+        }
+    }
+
+    private long writeBufferToFile(IFileHandle fileHandle, ByteBuffer writeBuffer, long offset)
+            throws HyracksDataException {
+        writeBuffer.flip();
+        long written = writeLocally(fileHandle, offset, writeBuffer);
+        writeBuffer.clear();
+        return written;
+    }
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudResettableInputStream.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java
similarity index 96%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudResettableInputStream.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java
index 84af84f..3a7504a 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/CloudResettableInputStream.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java
@@ -16,14 +16,14 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.hyracks.control.nc.io.cloud;
+package org.apache.asterix.cloud;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
 
+import org.apache.asterix.cloud.clients.ICloudBufferedWriter;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudBufferedWriter;
 
 public class CloudResettableInputStream extends InputStream {
     // TODO: make configurable
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/WriteBufferProvider.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/WriteBufferProvider.java
similarity index 90%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/WriteBufferProvider.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/WriteBufferProvider.java
index 44adf45..5e49be3 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/WriteBufferProvider.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/WriteBufferProvider.java
@@ -16,9 +16,9 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.hyracks.control.nc.io.cloud;
+package org.apache.asterix.cloud;
 
-import static org.apache.hyracks.control.nc.io.cloud.CloudResettableInputStream.MIN_BUFFER_SIZE;
+import static org.apache.asterix.cloud.CloudResettableInputStream.MIN_BUFFER_SIZE;
 
 import java.nio.ByteBuffer;
 import java.util.concurrent.ArrayBlockingQueue;
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/CloudClientProvider.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
similarity index 62%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/CloudClientProvider.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
index 72aa566..d808ea5 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/CloudClientProvider.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
@@ -16,14 +16,13 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.hyracks.control.nc.io.cloud.clients;
+package org.apache.asterix.cloud.clients;
 
-import java.util.HashMap;
-
+import org.apache.asterix.cloud.clients.ICloudClientCredentialsProvider.CredentialsType;
+import org.apache.asterix.cloud.clients.aws.s3.S3CloudClient;
+import org.apache.asterix.cloud.clients.aws.s3.credentials.IS3Credentials;
+import org.apache.asterix.cloud.clients.aws.s3.credentials.S3CredentialsProvider;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.control.nc.io.cloud.clients.aws.s3.S3CloudClient;
-import org.apache.hyracks.control.nc.io.cloud.clients.azure.blob.AzureBlobCloudClient;
-import org.apache.hyracks.control.nc.io.cloud.clients.gcp.gcs.GCSCloudClient;
 
 public class CloudClientProvider {
 
@@ -38,17 +37,12 @@
         throw new AssertionError("do not instantiate");
     }
 
-    public static ICloudClient getClient(ClientType clientType) throws HyracksDataException {
+    public static ICloudClient getClient(ClientType clientType, CredentialsType credentialsType)
+            throws HyracksDataException {
         switch (clientType) {
-            case NO_OP:
-                return NoOpCloudClient.INSTANCE;
             case S3:
-                // TODO: map should have the config already
-                return new S3CloudClient(new HashMap<>());
-            case AZURE_BLOB:
-                return new AzureBlobCloudClient();
-            case GOOGLE_CLOUD_STORAGE:
-                return new GCSCloudClient();
+                IS3Credentials credentials = S3CredentialsProvider.INSTANCE.getCredentials(credentialsType);
+                return new S3CloudClient(credentials);
             default:
                 throw HyracksDataException.create(new IllegalArgumentException("Unknown cloud client type"));
         }
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/ICloudBufferedWriter.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudBufferedWriter.java
similarity index 95%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/ICloudBufferedWriter.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudBufferedWriter.java
index 8bb2d68..4f08111 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/ICloudBufferedWriter.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudBufferedWriter.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.hyracks.control.nc.io.cloud.clients;
+package org.apache.asterix.cloud.clients;
 
 import java.io.InputStream;
 
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/ICloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
similarity index 87%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/ICloudClient.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
index bb12769..fe2a2d6 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/ICloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
@@ -16,11 +16,12 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.hyracks.control.nc.io.cloud.clients;
+package org.apache.asterix.cloud.clients;
 
 import java.io.FilenameFilter;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
+import java.util.Map;
 import java.util.Set;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -58,7 +59,6 @@
      * @param path path
      * @param offset offset
      * @param buffer buffer
-     * TODO(htowaileb) should this be returning the buffer position or the total amount read?
      * @return returns the buffer position
      */
     int read(String bucket, String path, long offset, ByteBuffer buffer) throws HyracksDataException;
@@ -125,4 +125,18 @@
      * @return {@code true} if the object exists, {@code false} otherwise
      */
     boolean exists(String bucket, String path);
+
+    /**
+     * Syncs files by downloading them from cloud storage to local storage
+     *
+     * @param bucket bucket to sync from
+     * @param cloudToLocalStoragePaths map of cloud storage partition to local storage path
+     * @throws HyracksDataException HyracksDataException
+     */
+    void syncFiles(String bucket, Map<String, String> cloudToLocalStoragePaths) throws HyracksDataException;
+
+    /**
+     * Performs any necessary closing and cleaning up
+     */
+    void close();
 }
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClientCredentialsProvider.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClientCredentialsProvider.java
new file mode 100644
index 0000000..1bb94b2
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClientCredentialsProvider.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface ICloudClientCredentialsProvider {
+    enum CredentialsType {
+        FILE,
+        MOCK
+    }
+
+    ICredentials getCredentials(CredentialsType type) throws HyracksDataException;
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICredentials.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICredentials.java
new file mode 100644
index 0000000..abf5284
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICredentials.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients;
+
+public interface ICredentials {
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3BufferedWriter.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
similarity index 96%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3BufferedWriter.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
index a793f40..9104d9a 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3BufferedWriter.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
@@ -16,15 +16,15 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.hyracks.control.nc.io.cloud.clients.aws.s3;
+package org.apache.asterix.cloud.clients.aws.s3;
 
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.asterix.cloud.clients.ICloudBufferedWriter;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudBufferedWriter;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
new file mode 100644
index 0000000..b800b96
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -0,0 +1,344 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients.aws.s3;
+
+import static org.apache.asterix.cloud.clients.aws.s3.S3Utils.listS3Objects;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.asterix.cloud.clients.ICloudBufferedWriter;
+import org.apache.asterix.cloud.clients.ICloudClient;
+import org.apache.asterix.cloud.clients.aws.s3.credentials.IS3Credentials;
+import org.apache.commons.io.FileUtils;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.util.IoUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.ResponseInputStream;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
+import software.amazon.awssdk.services.s3.S3CrtAsyncClientBuilder;
+import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
+import software.amazon.awssdk.services.s3.model.Delete;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.transfer.s3.S3TransferManager;
+import software.amazon.awssdk.transfer.s3.model.CompletedDirectoryDownload;
+import software.amazon.awssdk.transfer.s3.model.DirectoryDownload;
+import software.amazon.awssdk.transfer.s3.model.DownloadDirectoryRequest;
+
+public class S3CloudClient implements ICloudClient {
+
+    private static final Logger LOGGER = LogManager.getLogger();
+
+    private final IS3Credentials credentials;
+    private final S3Client s3Client;
+    private S3TransferManager s3TransferManager;
+
+    // TODO(htowaileb): Temporary variables, can we get this from the used instance?
+    private static final double MAX_HOST_BANDWIDTH = 10.0; // in Gbps
+
+    public S3CloudClient(IS3Credentials credentials) throws HyracksDataException {
+        this.credentials = credentials;
+        s3Client = buildClient();
+    }
+
+    private S3Client buildClient() throws HyracksDataException {
+        AwsCredentialsProvider credentialsProvider = StaticCredentialsProvider
+                .create(AwsBasicCredentials.create(credentials.getAccessKeyId(), credentials.getSecretAccessKey()));
+        S3ClientBuilder builder = S3Client.builder();
+        builder.credentialsProvider(credentialsProvider);
+        builder.region(Region.of(credentials.getRegion()));
+
+        if (credentials.getEndpoint() != null && !credentials.getEndpoint().isEmpty()) {
+            try {
+                URI uri = new URI(credentials.getEndpoint());
+                builder.endpointOverride(uri);
+            } catch (Exception ex) {
+                throw HyracksDataException.create(ex);
+            }
+        }
+        return builder.build();
+    }
+
+    @Override
+    public ICloudBufferedWriter createBufferedWriter(String bucket, String path) {
+        return new S3BufferedWriter(s3Client, bucket, path);
+    }
+
+    @Override
+    public Set<String> listObjects(String bucket, String path, FilenameFilter filter) {
+        return filterAndGet(listS3Objects(s3Client, bucket, path), filter);
+    }
+
+    @Override
+    public int read(String bucket, String path, long offset, ByteBuffer buffer) throws HyracksDataException {
+        long readTo = offset + buffer.remaining();
+        GetObjectRequest rangeGetObjectRequest =
+                GetObjectRequest.builder().range("bytes=" + offset + "-" + readTo).bucket(bucket).key(path).build();
+
+        int totalRead = 0;
+        int read = 0;
+
+        // TODO(htowaileb): add retry logic here
+        try (ResponseInputStream<GetObjectResponse> response = s3Client.getObject(rangeGetObjectRequest)) {
+            while (buffer.remaining() > 0) {
+                read = response.read(buffer.array(), buffer.position(), buffer.remaining());
+                buffer.position(buffer.position() + read);
+                totalRead += read;
+            }
+        } catch (IOException ex) {
+            throw HyracksDataException.create(ex);
+        }
+
+        if (buffer.remaining() != 0) {
+            throw new IllegalStateException("Expected buffer remaining = 0, found: " + buffer.remaining());
+        }
+        return totalRead;
+    }
+
+    @Override
+    public byte[] readAllBytes(String bucket, String path) throws HyracksDataException {
+        GetObjectRequest getReq = GetObjectRequest.builder().bucket(bucket).key(path).build();
+        try {
+            ResponseInputStream<GetObjectResponse> stream = s3Client.getObject(getReq);
+            return stream.readAllBytes();
+        } catch (NoSuchKeyException e) {
+            return null;
+        } catch (IOException e) {
+            throw HyracksDataException.create(e);
+        }
+    }
+
+    @Override
+    public InputStream getObjectStream(String bucket, String path) {
+        GetObjectRequest getReq = GetObjectRequest.builder().bucket(bucket).key(path).build();
+        try {
+            return s3Client.getObject(getReq);
+        } catch (NoSuchKeyException e) {
+            // This should not happen at least from the only caller of this method
+            throw new IllegalStateException(e);
+        }
+    }
+
+    @Override
+    public void write(String bucket, String path, byte[] data) {
+        PutObjectRequest putReq = PutObjectRequest.builder().bucket(bucket).key(path).build();
+
+        // TODO(htowaileb): add retry logic here
+        s3Client.putObject(putReq, RequestBody.fromBytes(data));
+    }
+
+    @Override
+    public void copy(String bucket, String srcPath, FileReference destPath) {
+        List<S3Object> objects = listS3Objects(s3Client, bucket, srcPath);
+        for (S3Object object : objects) {
+            String srcKey = object.key();
+            String destKey = destPath.getChildPath(IoUtil.getFileNameFromPath(srcKey));
+            CopyObjectRequest copyReq = CopyObjectRequest.builder().sourceBucket(bucket).sourceKey(srcKey)
+                    .destinationBucket(bucket).destinationKey(destKey).build();
+            s3Client.copyObject(copyReq);
+        }
+    }
+
+    @Override
+    public void deleteObject(String bucket, String path) {
+        Set<String> fileList = listObjects(bucket, path, IoUtil.NO_OP_FILTER);
+        if (fileList.isEmpty()) {
+            return;
+        }
+
+        List<ObjectIdentifier> objectIdentifiers = new ArrayList<>();
+        for (String file : fileList) {
+            objectIdentifiers.add(ObjectIdentifier.builder().key(file).build());
+        }
+        Delete delete = Delete.builder().objects(objectIdentifiers).build();
+        DeleteObjectsRequest deleteReq = DeleteObjectsRequest.builder().bucket(bucket).delete(delete).build();
+        s3Client.deleteObjects(deleteReq);
+    }
+
+    // TODO(htowaileb): Use the following cheaper request to check if an object exists
+    // https://stackoverflow.com/questions/3910071/check-file-size-on-s3-without-downloading
+    @Override
+    public long getObjectSize(String bucket, String path) {
+        List<S3Object> objects = listS3Objects(s3Client, bucket, path);
+        if (objects.isEmpty()) {
+            return 0;
+        }
+        return objects.get(0).size();
+    }
+
+    // TODO(htowaileb): Use the following cheaper request to check if an object exists
+    // https://docs.aws.amazon.com/AmazonS3/latest/userguide/example_s3_HeadObject_section.html
+    @Override
+    public boolean exists(String bucket, String path) {
+        List<S3Object> objects = listS3Objects(s3Client, bucket, path);
+        return !objects.isEmpty();
+    }
+
+    private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter filter) {
+        Set<String> files = new HashSet<>();
+        for (S3Object s3Object : contents) {
+            String path = s3Object.key();
+            if (filter.accept(null, IoUtil.getFileNameFromPath(path))) {
+                files.add(path);
+            }
+        }
+        return files;
+    }
+
+    @Override
+    public void syncFiles(String bucket, Map<String, String> cloudToLocalStoragePaths) throws HyracksDataException {
+        LOGGER.info("Syncing cloud storage to local storage started");
+
+        S3TransferManager s3TransferManager = getS3TransferManager();
+
+        List<CompletableFuture<CompletedDirectoryDownload>> downloads = new ArrayList<>();
+        cloudToLocalStoragePaths.forEach((cloudStoragePath, localStoragePath) -> {
+            DownloadDirectoryRequest.Builder builder = DownloadDirectoryRequest.builder();
+            builder.bucket(bucket);
+            builder.destination(Paths.get(localStoragePath));
+            builder.listObjectsV2RequestTransformer(l -> l.prefix(cloudStoragePath));
+
+            LOGGER.info("TransferManager started downloading from cloud \"{}\" to local storage \"{}\"",
+                    cloudStoragePath, localStoragePath);
+            DirectoryDownload directoryDownload = s3TransferManager.downloadDirectory(builder.build());
+            downloads.add(directoryDownload.completionFuture());
+        });
+
+        try {
+            for (CompletableFuture<CompletedDirectoryDownload> download : downloads) {
+                download.join();
+                CompletedDirectoryDownload completedDirectoryDownload = download.get();
+
+                // if we have failed downloads with transfer manager, try to download them with GetObject
+                if (!completedDirectoryDownload.failedTransfers().isEmpty()) {
+                    LOGGER.warn("TransferManager failed to download file(s), will retry to download each separately");
+                    completedDirectoryDownload.failedTransfers().forEach(LOGGER::warn);
+
+                    Map<String, String> failedFiles = new HashMap<>();
+                    completedDirectoryDownload.failedTransfers().forEach(failed -> {
+                        String cloudStoragePath = failed.request().getObjectRequest().key();
+                        String localStoragePath = failed.request().destination().toAbsolutePath().toString();
+                        failedFiles.put(cloudStoragePath, localStoragePath);
+                    });
+                    downloadFiles(bucket, failedFiles);
+                }
+                LOGGER.info("TransferManager finished downloading {} to local storage", completedDirectoryDownload);
+            }
+        } catch (ExecutionException | InterruptedException e) {
+            throw HyracksDataException.create(e);
+        }
+        LOGGER.info("Syncing cloud storage to local storage successful");
+    }
+
+    private void downloadFiles(String bucket, Map<String, String> cloudToLocalStoragePaths)
+            throws HyracksDataException {
+        byte[] buffer = new byte[8 * 1024];
+        for (Map.Entry<String, String> entry : cloudToLocalStoragePaths.entrySet()) {
+            String cloudStoragePath = entry.getKey();
+            String localStoragePath = entry.getValue();
+
+            LOGGER.info("GetObject started downloading from cloud \"{}\" to local storage \"{}\"", cloudStoragePath,
+                    localStoragePath);
+
+            // TODO(htowaileb): add retry logic here
+            try {
+                File localFile = new File(localStoragePath);
+                FileUtils.createParentDirectories(localFile);
+                if (!localFile.createNewFile()) {
+                    // do nothing for now, a restart has the files when trying to flush, for testing
+                    //throw new IllegalStateException("Couldn't create local file");
+                }
+
+                try (InputStream inputStream = getObjectStream(bucket, cloudStoragePath);
+                        FileOutputStream outputStream = new FileOutputStream(localFile)) {
+                    int bytesRead;
+                    while ((bytesRead = inputStream.read(buffer)) != -1) {
+                        outputStream.write(buffer, 0, bytesRead);
+                    }
+                }
+            } catch (IOException ex) {
+                throw HyracksDataException.create(ex);
+            }
+            LOGGER.info("GetObject successful downloading from cloud \"{}\" to local storage \"{}\"", cloudStoragePath,
+                    localStoragePath);
+        }
+    }
+
+    private S3TransferManager getS3TransferManager() {
+        if (s3TransferManager != null) {
+            return s3TransferManager;
+        }
+
+        S3CrtAsyncClientBuilder builder = S3AsyncClient.crtBuilder();
+        builder.credentialsProvider(StaticCredentialsProvider
+                .create(AwsBasicCredentials.create(credentials.getAccessKeyId(), credentials.getSecretAccessKey())));
+        builder.region(Region.of(credentials.getRegion()));
+        builder.targetThroughputInGbps(MAX_HOST_BANDWIDTH);
+        builder.minimumPartSizeInBytes((long) 8 * 1024 * 1024);
+
+        if (credentials.getEndpoint() != null && !credentials.getEndpoint().isEmpty()) {
+            builder.endpointOverride(URI.create(credentials.getEndpoint()));
+        }
+
+        S3AsyncClient client = builder.build();
+        s3TransferManager = S3TransferManager.builder().s3Client(client).build();
+        return s3TransferManager;
+    }
+
+    @Override
+    public void close() {
+        if (s3Client != null) {
+            s3Client.close();
+        }
+
+        if (s3TransferManager != null) {
+            s3TransferManager.close();
+        }
+    }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3Utils.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
similarity index 63%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3Utils.java
rename to asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
index e29b611..996b22d 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3Utils.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.hyracks.control.nc.io.cloud.clients.aws.s3;
+package org.apache.asterix.cloud.clients.aws.s3;
 
 import java.util.List;
 
@@ -55,21 +55,4 @@
         }
         return listObjectsResponse.contents();
     }
-
-    // TODO(htowaileb): Test few runs with default client and see if any failures are encountered
-    //    private static SdkHttpClient buildHttpClient() {
-    //        ApacheHttpClient.Builder apacheClientBuilder = ApacheHttpClient.builder();
-    //
-    //        AttributeMap.Builder overriddenConfigBuilder = AttributeMap.builder();
-    //        overriddenConfigBuilder.put(SdkHttpConfigurationOption.MAX_CONNECTIONS, 128);
-    //        overriddenConfigBuilder.put(SdkHttpConfigurationOption.CONNECTION_TIMEOUT, Duration.ofMinutes(60));
-    //        overriddenConfigBuilder.put(SdkHttpConfigurationOption.CONNECTION_MAX_IDLE_TIMEOUT, Duration.ofMinutes(60));
-    //        overriddenConfigBuilder.put(SdkHttpConfigurationOption.CONNECTION_ACQUIRE_TIMEOUT, Duration.ofMinutes(60));
-    //        overriddenConfigBuilder.put(SdkHttpConfigurationOption.READ_TIMEOUT, Duration.ofMinutes(60));
-    //        overriddenConfigBuilder.put(SdkHttpConfigurationOption.WRITE_TIMEOUT, Duration.ofMinutes(60));
-    //        overriddenConfigBuilder.put(SdkHttpConfigurationOption.TCP_KEEPALIVE, Boolean.TRUE);
-    //        AttributeMap configuration = overriddenConfigBuilder.build();
-    //
-    //        return apacheClientBuilder.buildWithDefaults(configuration);
-    //    }
 }
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/FileCredentials.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/FileCredentials.java
new file mode 100644
index 0000000..342e1a1
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/FileCredentials.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients.aws.s3.credentials;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class FileCredentials implements IS3Credentials {
+
+    private final String accessKeyId;
+    private final String secretAccessKey;
+    private final String region;
+    private String endpoint;
+
+    // TODO(htowaileb): change the credential file to be json object instead of reading per line
+    public FileCredentials(File file) throws HyracksDataException {
+        if (!file.exists()) {
+            throw new IllegalStateException("No cloud configuration file found");
+        }
+
+        try {
+            List<String> lines = FileUtils.readLines(file, "UTF-8");
+            this.accessKeyId = lines.get(1);
+            this.secretAccessKey = lines.get(2);
+            this.region = lines.get(3);
+
+            if (lines.size() > 4) {
+                this.endpoint = lines.get(4);
+            }
+        } catch (IOException ex) {
+            throw HyracksDataException.create(ex);
+        }
+    }
+
+    @Override
+    public String getAccessKeyId() {
+        return accessKeyId;
+    }
+
+    @Override
+    public String getSecretAccessKey() {
+        return secretAccessKey;
+    }
+
+    @Override
+    public String getRegion() {
+        return region;
+    }
+
+    @Override
+    public String getEndpoint() {
+        return endpoint;
+    }
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/IS3Credentials.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/IS3Credentials.java
new file mode 100644
index 0000000..32da3da
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/IS3Credentials.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients.aws.s3.credentials;
+
+import org.apache.asterix.cloud.clients.ICredentials;
+
+public interface IS3Credentials extends ICredentials {
+
+    /**
+     * @return access key id
+     */
+    String getAccessKeyId();
+
+    /**
+     * @return secret access key
+     */
+    String getSecretAccessKey();
+
+    /**
+     * @return region
+     */
+    String getRegion();
+
+    /**
+     * @return endpoint
+     */
+    String getEndpoint();
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3CredentialsProvider.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3CredentialsProvider.java
new file mode 100644
index 0000000..e605ea1
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3CredentialsProvider.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients.aws.s3.credentials;
+
+import java.io.File;
+
+import org.apache.asterix.cloud.clients.ICloudClientCredentialsProvider;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class S3CredentialsProvider implements ICloudClientCredentialsProvider {
+
+    public static S3CredentialsProvider INSTANCE = new S3CredentialsProvider();
+
+    private S3CredentialsProvider() {
+    }
+
+    @Override
+    public IS3Credentials getCredentials(CredentialsType type) throws HyracksDataException {
+        switch (type) {
+            case FILE:
+                File file = new File("/etc/s3");
+                if (file.exists()) {
+                    return new FileCredentials(file);
+                }
+            default:
+                return S3MockCredentials.INSTANCE;
+        }
+    }
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3MockCredentials.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3MockCredentials.java
new file mode 100644
index 0000000..56bb08a
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3MockCredentials.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients.aws.s3.credentials;
+
+public class S3MockCredentials implements IS3Credentials {
+
+    private static final String ACCESS_KEY_ID = "dummyAccessKeyId";
+    private static final String SECRET_ACCESS_KEY = "dummySecretAccessKey";
+    private static final String REGION = "us-west-2";
+    private static final String ENDPOINT = "http://127.0.0.1:8001";
+
+    public static final S3MockCredentials INSTANCE = new S3MockCredentials();
+
+    private S3MockCredentials() {
+    }
+
+    @Override
+    public String getAccessKeyId() {
+        return ACCESS_KEY_ID;
+    }
+
+    @Override
+    public String getSecretAccessKey() {
+        return SECRET_ACCESS_KEY;
+    }
+
+    @Override
+    public String getRegion() {
+        return REGION;
+    }
+
+    @Override
+    public String getEndpoint() {
+        return ENDPOINT;
+    }
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/CloudStorageConfigurationProvider.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/CloudStorageConfigurationProvider.java
new file mode 100644
index 0000000..7a94f98
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/CloudStorageConfigurationProvider.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.storage;
+
+import java.io.File;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class CloudStorageConfigurationProvider {
+
+    public static CloudStorageConfigurationProvider INSTANCE = new CloudStorageConfigurationProvider();
+
+    private CloudStorageConfigurationProvider() {
+    }
+
+    public ICloudStorageConfiguration getConfiguration(ICloudStorageConfiguration.ConfigurationType type)
+            throws HyracksDataException {
+        switch (type) {
+            case FILE:
+                File file = new File("/etc/storage");
+                if (file.exists()) {
+                    return new FileCloudStorageConfiguration(file);
+                }
+            default:
+                return MockCloudStorageConfiguration.INSTANCE;
+        }
+    }
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/FileCloudStorageConfiguration.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/FileCloudStorageConfiguration.java
new file mode 100644
index 0000000..7cee20a
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/FileCloudStorageConfiguration.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.storage;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class FileCloudStorageConfiguration implements ICloudStorageConfiguration {
+
+    private final String containerName;
+    private final int storagePartitionsCount;
+
+    public FileCloudStorageConfiguration(File file) throws HyracksDataException {
+        if (!file.exists()) {
+            throw new IllegalStateException("No cloud configuration file found");
+        }
+
+        try {
+            List<String> lines = FileUtils.readLines(file, "UTF-8");
+            this.containerName = lines.get(0);
+            this.storagePartitionsCount = Integer.parseInt(lines.get(1));
+        } catch (IOException ex) {
+            throw HyracksDataException.create(ex);
+        }
+    }
+
+    @Override
+    public String getContainer() {
+        return containerName;
+    }
+
+    @Override
+    public int getPartitionsCount() {
+        return storagePartitionsCount;
+    }
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/ICloudStorageConfiguration.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/ICloudStorageConfiguration.java
new file mode 100644
index 0000000..e140c7c
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/ICloudStorageConfiguration.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.storage;
+
+public interface ICloudStorageConfiguration {
+
+    enum ConfigurationType {
+        FILE,
+        MOCK
+    }
+
+    /**
+     * @return returns the container name used for the storage
+     */
+    String getContainer();
+
+    /**
+     * @return returns the number of storage partitions
+     */
+    int getPartitionsCount();
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/MockCloudStorageConfiguration.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/MockCloudStorageConfiguration.java
new file mode 100644
index 0000000..cf4f46f
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/MockCloudStorageConfiguration.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.storage;
+
+public class MockCloudStorageConfiguration implements ICloudStorageConfiguration {
+
+    private static final String CONTAINER_NAME = "cloud-storage-container";
+    private static final int NUMBER_OF_STORAGE_PARTITIONS = 8;
+
+    public static MockCloudStorageConfiguration INSTANCE = new MockCloudStorageConfiguration();
+
+    private MockCloudStorageConfiguration() {
+
+    }
+
+    @Override
+    public String getContainer() {
+        return CONTAINER_NAME;
+    }
+
+    @Override
+    public int getPartitionsCount() {
+        return NUMBER_OF_STORAGE_PARTITIONS;
+    }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/test/java/org/apache/hyracks/control/nc/lsm/LSMTest.java b/asterixdb/asterix-cloud/src/test/java/org/apach/asterix/cloud/LSMTest.java
similarity index 91%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/test/java/org/apache/hyracks/control/nc/lsm/LSMTest.java
rename to asterixdb/asterix-cloud/src/test/java/org/apach/asterix/cloud/LSMTest.java
index d62242f..bd0ab28 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/test/java/org/apache/hyracks/control/nc/lsm/LSMTest.java
+++ b/asterixdb/asterix-cloud/src/test/java/org/apach/asterix/cloud/LSMTest.java
@@ -16,16 +16,16 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.hyracks.control.nc.lsm;
+package org.apach.asterix.cloud;
 
 import java.io.FilenameFilter;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 
-import org.apache.hyracks.control.nc.io.cloud.CloudResettableInputStream;
-import org.apache.hyracks.control.nc.io.cloud.WriteBufferProvider;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudBufferedWriter;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudClient;
+import org.apache.asterix.cloud.CloudResettableInputStream;
+import org.apache.asterix.cloud.WriteBufferProvider;
+import org.apache.asterix.cloud.clients.ICloudBufferedWriter;
+import org.apache.asterix.cloud.clients.ICloudClient;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.junit.FixMethodOrder;
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/test/java/org/apache/hyracks/control/nc/lsm/aws/s3/LSMS3Test.java b/asterixdb/asterix-cloud/src/test/java/org/apach/asterix/cloud/s3/LSMS3Test.java
similarity index 80%
rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/test/java/org/apache/hyracks/control/nc/lsm/aws/s3/LSMS3Test.java
rename to asterixdb/asterix-cloud/src/test/java/org/apach/asterix/cloud/s3/LSMS3Test.java
index e9d0689..2424ced 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/test/java/org/apache/hyracks/control/nc/lsm/aws/s3/LSMS3Test.java
+++ b/asterixdb/asterix-cloud/src/test/java/org/apach/asterix/cloud/s3/LSMS3Test.java
@@ -16,14 +16,13 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.hyracks.control.nc.lsm.aws.s3;
+package org.apach.asterix.cloud.s3;
 
 import java.net.URI;
-import java.util.HashMap;
-import java.util.Map;
 
-import org.apache.hyracks.control.nc.io.cloud.clients.aws.s3.S3CloudClient;
-import org.apache.hyracks.control.nc.lsm.LSMTest;
+import org.apach.asterix.cloud.LSMTest;
+import org.apache.asterix.cloud.clients.aws.s3.S3CloudClient;
+import org.apache.asterix.cloud.clients.aws.s3.credentials.S3MockCredentials;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -40,7 +39,7 @@
     private static S3Client client;
     private static S3Mock s3MockServer;
     private static final int MOCK_SERVER_PORT = 8001;
-    private static final String MOCK_SERVER_HOSTNAME = "http://localhost:" + MOCK_SERVER_PORT;
+    private static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:" + MOCK_SERVER_PORT;
     private static final String MOCK_SERVER_REGION = "us-west-2"; // does not matter the value
 
     @BeforeClass
@@ -48,7 +47,11 @@
         LOGGER.info("LSMS3Test setup");
         LOGGER.info("Starting S3 mock server");
         s3MockServer = new S3Mock.Builder().withPort(MOCK_SERVER_PORT).withInMemoryBackend().build();
-        s3MockServer.start();
+        try {
+            s3MockServer.start();
+        } catch (Exception ex) {
+            // it might already be started, do nothing
+        }
         LOGGER.info("S3 mock server started successfully");
 
         // Create a client and add some files to the S3 mock server
@@ -62,12 +65,7 @@
         client.createBucket(CreateBucketRequest.builder().bucket(PLAYGROUND_CONTAINER).build());
         LOGGER.info("Client created successfully");
 
-        Map<String, String> clientConfiguration = new HashMap<>();
-        clientConfiguration.put("accessKeyId", "randomValue");
-        clientConfiguration.put("secretAccessKey", "randomValue");
-        clientConfiguration.put("region", "randomValue");
-        clientConfiguration.put("endpoint", MOCK_SERVER_HOSTNAME);
-        CLOUD_CLIENT = new S3CloudClient(clientConfiguration);
+        CLOUD_CLIENT = new S3CloudClient(S3MockCredentials.INSTANCE);
     }
 
     private static void cleanup() {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
index f4e241c..def3634 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
@@ -90,4 +90,9 @@
      * @return the extension manager instance
      */
     Object getExtensionManager();
+
+    /**
+     * @return true if running in cloud deployment, false otherwise.
+     */
+    boolean isCloudDeployment();
 }
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java
index 5475b97..434d61c 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java
@@ -49,6 +49,8 @@
 
     IIOManager getIoManager();
 
+    IIOManager getCloudIoManager();
+
     Executor getThreadExecutor();
 
     ITransactionSubsystem getTransactionSubsystem();
diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml
index e223d54..e068a2a 100644
--- a/asterixdb/asterix-server/pom.xml
+++ b/asterixdb/asterix-server/pom.xml
@@ -217,6 +217,10 @@
               </gavs>
               <noticeUrl>https://raw.githubusercontent.com/aws/aws-sdk-java-v2/2.10.83/NOTICE.txt</noticeUrl>
             </override>
+            <override>
+              <gav>software.amazon.awssdk.crt:aws-crt:0.21.10</gav>
+              <noticeUrl>https://raw.githubusercontent.com/awslabs/aws-crt-java/v0.21.10/NOTICE</noticeUrl>
+            </override>
             <!-- Hadoop AWS SDK -->
             <override>
               <gavs>
@@ -679,6 +683,7 @@
                 <aliasUrl>https://raw.githubusercontent.com/apache/orc/v1.8.0/LICENSE</aliasUrl>
                 <aliasUrl>https://raw.githubusercontent.com/RoaringBitmap/RoaringBitmap/0.9.39/LICENSE</aliasUrl>
                 <aliasUrl>https://raw.githubusercontent.com/JetBrains/java-annotations/master/LICENSE.txt</aliasUrl>
+                <aliasUrl>https://raw.githubusercontent.com/awslabs/aws-crt-java/v0.21.10/LICENSE</aliasUrl>
               </aliasUrls>
               <metric>1</metric>
             </license>
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index 0d6c70b..47544bb 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -87,7 +87,7 @@
     <hadoop.version>3.3.4</hadoop.version>
     <jacoco.version>0.7.6.201602180812</jacoco.version>
     <log4j.version>2.19.0</log4j.version>
-    <awsjavasdk.version>2.17.218</awsjavasdk.version>
+    <awsjavasdk.version>2.20.37</awsjavasdk.version>
     <parquet.version>1.12.3</parquet.version>
     <hadoop-awsjavasdk.version>1.12.402</hadoop-awsjavasdk.version>
     <azureblobjavasdk.version>12.14.2</azureblobjavasdk.version>
@@ -943,6 +943,7 @@
     <module>asterix-geo</module>
     <module>asterix-spidersilk</module>
     <module>asterix-column</module>
+    <module>asterix-cloud</module>
   </modules>
 
   <dependencyManagement>
@@ -1601,6 +1602,11 @@
         <artifactId>http-client-spi</artifactId>
         <version>${awsjavasdk.version}</version>
       </dependency>
+      <dependency>
+        <groupId>software.amazon.awssdk</groupId>
+        <artifactId>s3-transfer-manager</artifactId>
+        <version>${awsjavasdk.version}</version>
+      </dependency>
       <!-- Mock for AWS S3 -->
       <dependency>
         <groupId>io.findify</groupId>
diff --git a/asterixdb/src/main/appended-resources/supplemental-models.xml b/asterixdb/src/main/appended-resources/supplemental-models.xml
index c0555ec..3727fee 100644
--- a/asterixdb/src/main/appended-resources/supplemental-models.xml
+++ b/asterixdb/src/main/appended-resources/supplemental-models.xml
@@ -575,6 +575,21 @@
       </properties>
     </project>
   </supplement>
+
+  <!-- software.amazon.eventstream is ALv2, and does not contain any embedded LICENSE or NOTICE file -->
+  <!-- license override not needed, ALv2 is specified in its pom.xml -->
+  <!-- see https://github.com/aws/aws-sdk-java -->
+  <supplement>
+    <project>
+      <groupId>software.amazon.awssdk.crt</groupId>
+      <artifactId>aws-crt</artifactId>
+      <properties>
+        <license.ignoreMissingEmbeddedLicense>0.21.10</license.ignoreMissingEmbeddedLicense>
+        <license.ignoreMissingEmbeddedNotice>0.21.10</license.ignoreMissingEmbeddedNotice>
+        <license.ignoreNoticeOverride>0.21.10</license.ignoreNoticeOverride>
+      </properties>
+    </project>
+  </supplement>
   <!-- AWS SDK end -->
 
   <!-- AWS Hadoop SDK start -->
diff --git a/asterixdb/src/main/licenses/content/raw.githubusercontent.com_awslabs_aws-crt-java_v0.21.10_NOTICE.txt b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_awslabs_aws-crt-java_v0.21.10_NOTICE.txt
new file mode 100644
index 0000000..9a649a2
--- /dev/null
+++ b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_awslabs_aws-crt-java_v0.21.10_NOTICE.txt
@@ -0,0 +1,3 @@
+AWS Crt Java
+Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+SPDX-License-Identifier: Apache-2.0.
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
index 20e9ff6..cec1598 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
@@ -100,7 +100,7 @@
     public void register() {
         if (registrationTime != 0) {
             throw new IllegalStateException(
-                    "File " + toString() + " was already registered at " + new Date(registrationTime));
+                    "File " + this + " was already registered at " + new Date(registrationTime));
         }
         registrationTime = System.currentTimeMillis();
     }
@@ -111,7 +111,7 @@
 
     public void unregister() {
         if (registrationTime == 0) {
-            throw new IllegalStateException("File " + toString() + " wasn't registered before");
+            throw new IllegalStateException("File " + this + " wasn't registered before");
         }
         registrationTime = 0;
     }
@@ -119,4 +119,13 @@
     public boolean isCompressed() {
         return false;
     }
+
+    public FileReference getParent() {
+        int parentIndex = path.lastIndexOf(File.separatorChar);
+        if (parentIndex < 0) {
+            return new FileReference(dev, "");
+        }
+        String parentPath = path.substring(parentIndex);
+        return new FileReference(dev, parentPath);
+    }
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
index c3ff70a..5f93c21 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
@@ -134,10 +134,11 @@
 
     void deleteDirectory(FileReference root) throws HyracksDataException;
 
-    // TODO: Remove and use list
     Collection<FileReference> getMatchingFiles(FileReference root, FilenameFilter filter) throws HyracksDataException;
 
     boolean exists(FileReference fileRef);
 
     void create(FileReference fileRef) throws HyracksDataException;
+
+    void syncFiles(Set<Integer> activePartitions) throws HyracksDataException;
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
index 223c71c..d68c291 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
@@ -95,36 +95,5 @@
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
     </dependency>
-    <dependency>
-      <groupId>software.amazon.awssdk</groupId>
-      <artifactId>sdk-core</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>software.amazon.awssdk</groupId>
-      <artifactId>s3</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>software.amazon.awssdk</groupId>
-      <artifactId>regions</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>software.amazon.awssdk</groupId>
-      <artifactId>auth</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>io.findify</groupId>
-      <artifactId>s3mock_2.12</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>com.typesafe.akka</groupId>
-      <artifactId>akka-http-core_2.12</artifactId>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
 </project>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index 984ab08..4eebf6b 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -67,6 +67,8 @@
     /*
      * Finals
      */
+    private final int queueSize;
+    private final int ioParallelism;
     private final ExecutorService executor;
     private final BlockingQueue<IoRequest> submittedRequests;
     private final BlockingQueue<IoRequest> freeRequests;
@@ -81,6 +83,8 @@
     public IOManager(List<IODeviceHandle> devices, IFileDeviceResolver deviceComputer, int ioParallelism, int queueSize)
             throws HyracksDataException {
         this.ioDevices = Collections.unmodifiableList(devices);
+        this.queueSize = queueSize;
+        this.ioParallelism = ioParallelism;
         checkDeviceValidity(devices);
         workspaces = new ArrayList<>();
         for (IODeviceHandle d : ioDevices) {
@@ -98,27 +102,29 @@
         }
         workspaceIndex = 0;
         this.deviceComputer = deviceComputer;
-        submittedRequests = new ArrayBlockingQueue<>(queueSize);
-        freeRequests = new ArrayBlockingQueue<>(queueSize);
-        int numIoThreads = ioDevices.size() * ioParallelism;
+        submittedRequests = new ArrayBlockingQueue<>(this.queueSize);
+        freeRequests = new ArrayBlockingQueue<>(this.queueSize);
+        int numIoThreads = ioDevices.size() * this.ioParallelism;
         executor = Executors.newFixedThreadPool(numIoThreads);
         for (int i = 0; i < numIoThreads; i++) {
             executor.execute(new IoRequestHandler(i, submittedRequests));
         }
     }
 
-    protected IOManager(IOManager ioManager, int queueSize, int ioParallelism) {
-        this.ioDevices = ioManager.ioDevices;
-        workspaces = ioManager.workspaces;
-        workspaceIndex = 0;
-        this.deviceComputer = ioManager.deviceComputer;
-        submittedRequests = new ArrayBlockingQueue<>(queueSize);
-        freeRequests = new ArrayBlockingQueue<>(queueSize);
-        int numIoThreads = ioDevices.size() * ioParallelism;
-        executor = Executors.newFixedThreadPool(numIoThreads);
-        for (int i = 0; i < numIoThreads; i++) {
-            executor.execute(new IoRequestHandler(i, submittedRequests));
-        }
+    public int getQueueSize() {
+        return queueSize;
+    }
+
+    public int getIoParallelism() {
+        return ioParallelism;
+    }
+
+    public List<IODeviceHandle> getIoDevices() {
+        return ioDevices;
+    }
+
+    public IFileDeviceResolver getDeviceComputer() {
+        return deviceComputer;
     }
 
     public IoRequest getOrAllocRequest() {
@@ -570,7 +576,12 @@
     @Override
     public Collection<FileReference> getMatchingFiles(FileReference root, FilenameFilter filter)
             throws HyracksDataException {
-        Collection<File> files = IoUtil.getMatchingFiles(root.getFile().toPath(), filter);
+        File rootFile = root.getFile();
+        if (!rootFile.exists()) {
+            return Collections.emptyList();
+        }
+
+        Collection<File> files = IoUtil.getMatchingFiles(rootFile.toPath(), filter);
         Set<FileReference> fileReferences = new HashSet<>();
         for (File file : files) {
             fileReferences.add(resolveAbsolutePath(file.getAbsolutePath()));
@@ -597,4 +608,9 @@
             throw HyracksDataException.create(e);
         }
     }
+
+    @Override
+    public void syncFiles(Set<Integer> activePartitions) throws HyracksDataException {
+        // do nothing
+    }
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/LocalCacheUtil.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/LocalCacheUtil.java
deleted file mode 100644
index ada32b6..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/LocalCacheUtil.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.io.cloud;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IFileHandle;
-import org.apache.hyracks.api.io.IIOManager;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudClient;
-import org.apache.hyracks.util.file.FileUtil;
-
-// TODO replace with a proper caching mechanism
-public class LocalCacheUtil {
-    private LocalCacheUtil() {
-
-    }
-
-    public static void writeToFile(FileReference fileRef, byte[] bytes) throws HyracksDataException {
-        try {
-            File file = fileRef.getFile();
-            FileUtils.createParentDirectories(file);
-            FileUtil.writeAndForce(file.toPath(), bytes);
-        } catch (IOException e) {
-            throw HyracksDataException.create(e);
-        }
-    }
-
-    // TODO: replace with proper caching policy
-    public static void download(ICloudClient cloudClient, CloudIOManager ioManager, CloudFileHandle fileHandle,
-            IIOManager.FileReadWriteMode rwMode, IIOManager.FileSyncMode syncMode, ByteBuffer writeBuffer)
-            throws HyracksDataException {
-        FileReference fileRef = fileHandle.getFileReference();
-        // write the file locally (the call to open is synchronized hence only one thread can perform this call)
-        try (InputStream inStream = cloudClient.getObjectStream(ioManager.getBucket(), fileRef.getRelativePath())) {
-            File file = fileRef.getFile();
-            FileUtils.createParentDirectories(fileRef.getFile());
-            if (!file.createNewFile()) {
-                throw new IllegalStateException("Couldn't create local file");
-            }
-            fileHandle.open(rwMode, syncMode);
-            LocalCacheUtil.writeToFile(ioManager, fileHandle, inStream, writeBuffer);
-        } catch (IOException e) {
-            throw HyracksDataException.create(e);
-        }
-    }
-
-    private static void writeToFile(CloudIOManager ioManager, IFileHandle fileHandle, InputStream inStream,
-            ByteBuffer writeBuffer) throws HyracksDataException {
-        writeBuffer.clear();
-        try {
-            int position = 0;
-            long offset = 0;
-            int read;
-            while ((read = inStream.read(writeBuffer.array(), position, writeBuffer.remaining())) >= 0) {
-                position += read;
-                writeBuffer.position(position);
-                if (writeBuffer.remaining() == 0) {
-                    offset += writeBufferToFile(ioManager, fileHandle, writeBuffer, offset);
-                    position = 0;
-                }
-            }
-
-            if (writeBuffer.position() > 0) {
-                writeBufferToFile(ioManager, fileHandle, writeBuffer, offset);
-                ioManager.syncLocally(fileHandle);
-            }
-        } catch (IOException e) {
-            throw HyracksDataException.create(e);
-        }
-
-    }
-
-    private static long writeBufferToFile(CloudIOManager ioManager, IFileHandle fileHandle, ByteBuffer writeBuffer,
-            long offset) throws HyracksDataException {
-        writeBuffer.flip();
-        long written = ioManager.writeLocally(fileHandle, offset, writeBuffer);
-        writeBuffer.clear();
-        return written;
-    }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/NoOpCloudClient.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/NoOpCloudClient.java
deleted file mode 100644
index 2751181..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/NoOpCloudClient.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.io.cloud.clients;
-
-import java.io.FilenameFilter;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.util.Set;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-
-public class NoOpCloudClient implements ICloudClient {
-
-    public static final NoOpCloudClient INSTANCE = new NoOpCloudClient();
-
-    private NoOpCloudClient() {
-        // do not instantiate
-    }
-
-    @Override
-    public ICloudBufferedWriter createBufferedWriter(String bucket, String path) {
-        return null;
-    }
-
-    @Override
-    public Set<String> listObjects(String bucket, String path, FilenameFilter filter) {
-        return null;
-    }
-
-    @Override
-    public int read(String bucket, String path, long offset, ByteBuffer buffer) throws HyracksDataException {
-        return 0;
-    }
-
-    @Override
-    public byte[] readAllBytes(String bucket, String path) throws HyracksDataException {
-        return new byte[0];
-    }
-
-    @Override
-    public InputStream getObjectStream(String bucket, String path) {
-        return null;
-    }
-
-    @Override
-    public void write(String bucket, String path, byte[] data) {
-
-    }
-
-    @Override
-    public void copy(String bucket, String srcPath, FileReference destPath) {
-
-    }
-
-    @Override
-    public void deleteObject(String bucket, String path) {
-
-    }
-
-    @Override
-    public long getObjectSize(String bucket, String path) {
-        return 0;
-    }
-
-    @Override
-    public boolean exists(String bucket, String path) {
-        return false;
-    }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3CloudClient.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3CloudClient.java
deleted file mode 100644
index af1441e..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/aws/s3/S3CloudClient.java
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.io.cloud.clients.aws.s3;
-
-import static org.apache.hyracks.control.nc.io.cloud.clients.aws.s3.S3Utils.listS3Objects;
-
-import java.io.File;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.util.IoUtil;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudBufferedWriter;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudClient;
-
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import software.amazon.awssdk.core.ResponseInputStream;
-import software.amazon.awssdk.core.sync.RequestBody;
-import software.amazon.awssdk.regions.Region;
-import software.amazon.awssdk.services.s3.S3Client;
-import software.amazon.awssdk.services.s3.S3ClientBuilder;
-import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
-import software.amazon.awssdk.services.s3.model.Delete;
-import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
-import software.amazon.awssdk.services.s3.model.GetObjectRequest;
-import software.amazon.awssdk.services.s3.model.GetObjectResponse;
-import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
-import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
-import software.amazon.awssdk.services.s3.model.PutObjectRequest;
-import software.amazon.awssdk.services.s3.model.S3Object;
-
-public class S3CloudClient implements ICloudClient {
-
-    private static final String ACCESS_KEY_ID_FIELD = "accessKeyId";
-    private static final String SECRET_ACCESS_KEY_FIELD = "secretAccessKey";
-    private static final String REGION_FIELD = "region";
-    private final static String ENDPOINT_FIELD = "endpoint";
-
-    private final S3Client s3Client;
-
-    // TODO fix the throws exception
-    public S3CloudClient(Map<String, String> clientConfiguration) throws HyracksDataException {
-        setClientConfig(clientConfiguration); // TODO: remove later, this is temporary
-        s3Client = buildClient(clientConfiguration);
-    }
-
-    private S3Client buildClient(Map<String, String> clientConfiguration) throws HyracksDataException {
-        String accessKeyId = clientConfiguration.get(ACCESS_KEY_ID_FIELD);
-        String secretAccessKey = clientConfiguration.get(SECRET_ACCESS_KEY_FIELD);
-        String region = clientConfiguration.get(REGION_FIELD);
-        String endpoint = clientConfiguration.get(ENDPOINT_FIELD);
-
-        AwsCredentialsProvider credentialsProvider =
-                StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey));
-        S3ClientBuilder builder = S3Client.builder();
-        builder.credentialsProvider(credentialsProvider);
-        builder.region(Region.of(region));
-
-        if (endpoint != null) {
-            try {
-                URI uri = new URI(endpoint);
-                builder.endpointOverride(uri);
-            } catch (Exception ex) {
-                throw HyracksDataException.create(ex);
-            }
-        }
-        return builder.build();
-    }
-
-    // TODO: temporarily setting the client config, this should be provided
-    private void setClientConfig(Map<String, String> clientConfiguration) throws HyracksDataException {
-        if (!clientConfiguration.isEmpty()) {
-            return;
-        }
-
-        try {
-            List<String> lines = FileUtils.readLines(new File("/etc/s3"), "UTF-8");
-            String accessKeyId = lines.get(1);
-            String secretAccessKey = lines.get(2);
-            String region = lines.get(3);
-
-            clientConfiguration.put(ACCESS_KEY_ID_FIELD, accessKeyId);
-            clientConfiguration.put(SECRET_ACCESS_KEY_FIELD, secretAccessKey);
-            clientConfiguration.put(REGION_FIELD, region);
-
-            if (lines.size() > 4) {
-                String serviceEndpoint = lines.get(4);
-                clientConfiguration.put(ENDPOINT_FIELD, serviceEndpoint);
-            }
-        } catch (IOException ex) {
-            throw HyracksDataException.create(ex);
-        }
-    }
-
-    @Override
-    public ICloudBufferedWriter createBufferedWriter(String bucket, String path) {
-        return new S3BufferedWriter(s3Client, bucket, path);
-    }
-
-    @Override
-    public Set<String> listObjects(String bucket, String path, FilenameFilter filter) {
-        return filterAndGet(listS3Objects(s3Client, bucket, path), filter);
-    }
-
-    @Override
-    public int read(String bucket, String path, long offset, ByteBuffer buffer) throws HyracksDataException {
-        long readTo = offset + buffer.remaining();
-        GetObjectRequest rangeGetObjectRequest =
-                GetObjectRequest.builder().range("bytes=" + offset + "-" + readTo).bucket(bucket).key(path).build();
-
-        int totalRead = 0;
-        int read = 0;
-
-        // TODO(htowaileb): add retry logic here
-        try (ResponseInputStream<GetObjectResponse> response = s3Client.getObject(rangeGetObjectRequest)) {
-            while (buffer.remaining() > 0) {
-                read = response.read(buffer.array(), buffer.position(), buffer.remaining());
-                buffer.position(buffer.position() + read);
-                totalRead += read;
-            }
-        } catch (IOException ex) {
-            throw HyracksDataException.create(ex);
-        }
-
-        if (buffer.remaining() != 0) {
-            throw new IllegalStateException("Expected buffer remaining = 0, found: " + buffer.remaining());
-        }
-        return totalRead;
-    }
-
-    @Override
-    public byte[] readAllBytes(String bucket, String path) throws HyracksDataException {
-        GetObjectRequest getReq = GetObjectRequest.builder().bucket(bucket).key(path).build();
-        try {
-            ResponseInputStream<GetObjectResponse> stream = s3Client.getObject(getReq);
-            return stream.readAllBytes();
-        } catch (NoSuchKeyException e) {
-            return null;
-        } catch (IOException e) {
-            throw HyracksDataException.create(e);
-        }
-    }
-
-    @Override
-    public InputStream getObjectStream(String bucket, String path) {
-        GetObjectRequest getReq = GetObjectRequest.builder().bucket(bucket).key(path).build();
-        try {
-            return s3Client.getObject(getReq);
-        } catch (NoSuchKeyException e) {
-            // This should not happen at least from the only caller of this method
-            throw new IllegalStateException(e);
-        }
-    }
-
-    @Override
-    public void write(String bucket, String path, byte[] data) {
-        PutObjectRequest putReq = PutObjectRequest.builder().bucket(bucket).key(path).build();
-
-        // TODO(htowaileb): add retry logic here
-        s3Client.putObject(putReq, RequestBody.fromBytes(data));
-    }
-
-    @Override
-    public void copy(String bucket, String srcPath, FileReference destPath) {
-        List<S3Object> objects = listS3Objects(s3Client, bucket, srcPath);
-        for (S3Object object : objects) {
-            String srcKey = object.key();
-            String destKey = destPath.getChildPath(IoUtil.getFileNameFromPath(srcKey));
-            CopyObjectRequest copyReq = CopyObjectRequest.builder().sourceBucket(bucket).sourceKey(srcKey)
-                    .destinationBucket(bucket).destinationKey(destKey).build();
-            s3Client.copyObject(copyReq);
-        }
-    }
-
-    @Override
-    public void deleteObject(String bucket, String path) {
-        Set<String> fileList = listObjects(bucket, path, IoUtil.NO_OP_FILTER);
-        if (fileList.isEmpty()) {
-            return;
-        }
-
-        List<ObjectIdentifier> objectIdentifiers = new ArrayList<>();
-        for (String file : fileList) {
-            objectIdentifiers.add(ObjectIdentifier.builder().key(file).build());
-        }
-        Delete delete = Delete.builder().objects(objectIdentifiers).build();
-        DeleteObjectsRequest deleteReq = DeleteObjectsRequest.builder().bucket(bucket).delete(delete).build();
-        s3Client.deleteObjects(deleteReq);
-    }
-
-    @Override
-    public long getObjectSize(String bucket, String path) {
-        List<S3Object> objects = listS3Objects(s3Client, bucket, path);
-        if (objects.isEmpty()) {
-            return 0;
-        }
-        return objects.get(0).size();
-    }
-
-    @Override
-    public boolean exists(String bucket, String path) {
-        List<S3Object> objects = listS3Objects(s3Client, bucket, path);
-        return !objects.isEmpty();
-    }
-
-    private Set<String> filterAndGet(List<S3Object> contents, FilenameFilter filter) {
-        Set<String> files = new HashSet<>();
-        for (S3Object s3Object : contents) {
-            String path = s3Object.key();
-            if (filter.accept(null, IoUtil.getFileNameFromPath(path))) {
-                files.add(path);
-            }
-        }
-        return files;
-    }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/azure/blob/AzureBlobCloudClient.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/azure/blob/AzureBlobCloudClient.java
deleted file mode 100644
index 6dfe05a..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/azure/blob/AzureBlobCloudClient.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.io.cloud.clients.azure.blob;
-
-import java.io.FilenameFilter;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.util.Set;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudBufferedWriter;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudClient;
-
-public class AzureBlobCloudClient implements ICloudClient {
-
-    public AzureBlobCloudClient() {
-        throw new IllegalStateException("NYI");
-    }
-
-    @Override
-    public ICloudBufferedWriter createBufferedWriter(String bucket, String path) {
-        return null;
-    }
-
-    @Override
-    public Set<String> listObjects(String bucket, String path, FilenameFilter filter) {
-        return null;
-    }
-
-    @Override
-    public int read(String bucket, String path, long offset, ByteBuffer buffer) throws HyracksDataException {
-        return 0;
-    }
-
-    @Override
-    public byte[] readAllBytes(String bucket, String path) throws HyracksDataException {
-        return new byte[0];
-    }
-
-    @Override
-    public InputStream getObjectStream(String bucket, String path) {
-        return null;
-    }
-
-    @Override
-    public void write(String bucket, String path, byte[] data) {
-
-    }
-
-    @Override
-    public void copy(String bucket, String srcPath, FileReference destPath) {
-
-    }
-
-    @Override
-    public void deleteObject(String bucket, String path) {
-
-    }
-
-    @Override
-    public long getObjectSize(String bucket, String path) {
-        return 0;
-    }
-
-    @Override
-    public boolean exists(String bucket, String path) {
-        return false;
-    }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/gcp/gcs/GCSCloudClient.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/gcp/gcs/GCSCloudClient.java
deleted file mode 100644
index a9a6f06..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/cloud/clients/gcp/gcs/GCSCloudClient.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.io.cloud.clients.gcp.gcs;
-
-import java.io.FilenameFilter;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.util.Set;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudBufferedWriter;
-import org.apache.hyracks.control.nc.io.cloud.clients.ICloudClient;
-
-public class GCSCloudClient implements ICloudClient {
-
-    public GCSCloudClient() {
-        throw new IllegalStateException("NYI");
-    }
-
-    @Override
-    public ICloudBufferedWriter createBufferedWriter(String bucket, String path) {
-        return null;
-    }
-
-    @Override
-    public Set<String> listObjects(String bucket, String path, FilenameFilter filter) {
-        return null;
-    }
-
-    @Override
-    public int read(String bucket, String path, long offset, ByteBuffer buffer) throws HyracksDataException {
-        return 0;
-    }
-
-    @Override
-    public byte[] readAllBytes(String bucket, String path) throws HyracksDataException {
-        return new byte[0];
-    }
-
-    @Override
-    public InputStream getObjectStream(String bucket, String path) {
-        return null;
-    }
-
-    @Override
-    public void write(String bucket, String path, byte[] data) {
-
-    }
-
-    @Override
-    public void copy(String bucket, String srcPath, FileReference destPath) {
-
-    }
-
-    @Override
-    public void deleteObject(String bucket, String path) {
-
-    }
-
-    @Override
-    public long getObjectSize(String bucket, String path) {
-        return 0;
-    }
-
-    @Override
-    public boolean exists(String bucket, String path) {
-        return false;
-    }
-}
diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml
index c1b8bfc..348b5bf 100644
--- a/hyracks-fullstack/pom.xml
+++ b/hyracks-fullstack/pom.xml
@@ -77,7 +77,6 @@
     <jackson.version>2.14.1</jackson.version>
     <jackson-databind.version>${jackson.version}</jackson-databind.version>
     <netty.version>4.1.87.Final</netty.version>
-    <awsjavasdk.version>2.17.218</awsjavasdk.version>
 
     <implementation.title>Apache Hyracks and Algebricks - ${project.name}</implementation.title>
     <implementation.url>https://asterixdb.apache.org/</implementation.url>
@@ -481,38 +480,6 @@
         <artifactId>jetty-util-ajax</artifactId>
         <version>9.4.48.v20220622</version>
       </dependency>
-      <dependency>
-        <groupId>software.amazon.awssdk</groupId>
-        <artifactId>sdk-core</artifactId>
-        <version>${awsjavasdk.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>software.amazon.awssdk</groupId>
-        <artifactId>s3</artifactId>
-        <version>${awsjavasdk.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>software.amazon.awssdk</groupId>
-        <artifactId>regions</artifactId>
-        <version>${awsjavasdk.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>software.amazon.awssdk</groupId>
-        <artifactId>auth</artifactId>
-        <version>${awsjavasdk.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>io.findify</groupId>
-        <artifactId>s3mock_2.12</artifactId>
-        <version>0.2.5</version>
-        <scope>test</scope>
-      </dependency>
-      <dependency>
-        <groupId>com.typesafe.akka</groupId>
-        <artifactId>akka-http-core_2.12</artifactId>
-        <version>10.1.0</version>
-        <scope>test</scope>
-      </dependency>
     </dependencies>
   </dependencyManagement>
   <build>