[ASTERIXDB-3592][OTH] Increase S3 client HTTP pool size
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Increase S3 client HTTP pool size from 50 to 1000.
Ext-ref: MB-66048
Change-Id: I4ac5bdc21d9577513f777b3fedf6ffb96b03754a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19628
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Tested-by: Murtadha Hubail <mhubail@apache.org>
Integration-Tests: Murtadha Hubail <mhubail@apache.org>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index c6d7ba9..29061ec 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -17,6 +17,7 @@
"cloud.max.read.requests.per.second" : 1500,
"cloud.max.write.requests.per.second" : 250,
"cloud.profiler.log.interval" : 5,
+ "cloud.requests.max.http.connections" : 1000,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
"cloud.storage.bucket" : "",
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 562e195..f2ea15b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -17,6 +17,7 @@
"cloud.max.read.requests.per.second" : 1500,
"cloud.max.write.requests.per.second" : 250,
"cloud.profiler.log.interval" : 5,
+ "cloud.requests.max.http.connections" : 1000,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
"cloud.storage.bucket" : "",
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 132fa5b..685d28b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -17,6 +17,7 @@
"cloud.max.read.requests.per.second" : 1500,
"cloud.max.write.requests.per.second" : 250,
"cloud.profiler.log.interval" : 5,
+ "cloud.requests.max.http.connections" : 1000,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
"cloud.storage.bucket" : "",
diff --git a/asterixdb/asterix-cloud/pom.xml b/asterixdb/asterix-cloud/pom.xml
index 6f952b0..419ac4d 100644
--- a/asterixdb/asterix-cloud/pom.xml
+++ b/asterixdb/asterix-cloud/pom.xml
@@ -247,6 +247,10 @@
<artifactId>aws-crt</artifactId>
</dependency>
<dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>apache-client</artifactId>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
index 0b9b15c..20727de 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
@@ -41,15 +41,16 @@
private final long tokenAcquireTimeout;
private final int readMaxRequestsPerSeconds;
private final int writeMaxRequestsPerSeconds;
+ private final int requestsMaxHttpConnections;
public S3ClientConfig(String region, String endpoint, String prefix, boolean anonymousAuth,
long profilerLogInterval, int writeBufferSize) {
- this(region, endpoint, prefix, anonymousAuth, profilerLogInterval, writeBufferSize, 1, 0, 0);
+ this(region, endpoint, prefix, anonymousAuth, profilerLogInterval, writeBufferSize, 1, 0, 0, 0);
}
private S3ClientConfig(String region, String endpoint, String prefix, boolean anonymousAuth,
long profilerLogInterval, int writeBufferSize, long tokenAcquireTimeout, int writeMaxRequestsPerSeconds,
- int readMaxRequestsPerSeconds) {
+ int readMaxRequestsPerSeconds, int requestsMaxHttpConnections) {
this.region = Objects.requireNonNull(region, "region");
this.endpoint = endpoint;
this.prefix = Objects.requireNonNull(prefix, "prefix");
@@ -59,6 +60,7 @@
this.tokenAcquireTimeout = tokenAcquireTimeout;
this.writeMaxRequestsPerSeconds = writeMaxRequestsPerSeconds;
this.readMaxRequestsPerSeconds = readMaxRequestsPerSeconds;
+ this.requestsMaxHttpConnections = requestsMaxHttpConnections;
}
public static S3ClientConfig of(CloudProperties cloudProperties) {
@@ -66,7 +68,7 @@
cloudProperties.getStoragePrefix(), cloudProperties.isStorageAnonymousAuth(),
cloudProperties.getProfilerLogInterval(), cloudProperties.getWriteBufferSize(),
cloudProperties.getTokenAcquireTimeout(), cloudProperties.getWriteMaxRequestsPerSecond(),
- cloudProperties.getReadMaxRequestsPerSecond());
+ cloudProperties.getReadMaxRequestsPerSecond(), cloudProperties.getRequestsMaxHttpConnections());
}
public static S3ClientConfig of(Map<String, String> configuration, int writeBufferSize) {
@@ -124,6 +126,10 @@
return readMaxRequestsPerSeconds;
}
+ public int getRequestsMaxHttpConnections() {
+ return requestsMaxHttpConnections;
+ }
+
private boolean isS3Mock() {
return endpoint != null && !endpoint.isEmpty();
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index 224ede4..01a8b02 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -63,6 +63,7 @@
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3ClientBuilder;
@@ -325,6 +326,10 @@
S3ClientBuilder builder = S3Client.builder();
builder.credentialsProvider(config.createCredentialsProvider());
builder.region(Region.of(config.getRegion()));
+ if (config.getRequestsMaxHttpConnections() > 0) {
+ builder.httpClientBuilder(
+ ApacheHttpClient.builder().maxConnections(config.getRequestsMaxHttpConnections()));
+ }
if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
URI uri;
try {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
index 7b6fb6e..1e3fe75 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
@@ -64,7 +64,8 @@
CLOUD_WRITE_BUFFER_SIZE(
getRangedIntegerType(5, Integer.MAX_VALUE),
StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.MEGABYTE)),
- CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD(POSITIVE_INTEGER, 50);
+ CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD(POSITIVE_INTEGER, 50),
+ CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS(POSITIVE_INTEGER, 1000);
private final IOptionType interpreter;
private final Object defaultValue;
@@ -96,6 +97,7 @@
case CLOUD_MAX_READ_REQUESTS_PER_SECOND:
case CLOUD_WRITE_BUFFER_SIZE:
case CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD:
+ case CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS:
return Section.COMMON;
default:
return Section.NC;
@@ -163,6 +165,8 @@
return "The write buffer size in bytes. (default: 8MB, min: 5MB)";
case CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD:
return "The number of cloud reads for re-evaluating an eviction plan. (default: 50)";
+ case CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS:
+ return "The maximum number of HTTP connections to use for cloud requests per node. (default: 1000)";
default:
throw new IllegalStateException("NYI: " + this);
}
@@ -258,4 +262,8 @@
public int getEvictionPlanReevaluationThreshold() {
return accessor.getInt(Option.CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD);
}
+
+ public int getRequestsMaxHttpConnections() {
+ return accessor.getInt(Option.CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS);
+ }
}
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index e90283d..35f9219 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -1635,6 +1635,11 @@
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
+ <artifactId>apache-client</artifactId>
+ <version>${awsjavasdk.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
<artifactId>http-client-spi</artifactId>
<version>${awsjavasdk.version}</version>
</dependency>