[ASTERIXDB-3619][CONF] Add cloud properties to configure HTTP client used by cloud client
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Add the following cloud properties:
CLOUD_REQUESTS_MAX_PENDING_HTTP_CONNECTIONS.
CLOUD_REQUESTS_HTTP_CONNECTION_ACQUIRE_TIMEOUT.
- Apply the HTTP configurations to the (non-Crt) S3AsyncClient similar to the S3Client:
- Now S3AsyncClient will use the default 1000 instead of 50 for
the number of concurrent connections.
- Allow configuring the HTTP connection acquire timeout:
- Default is increased to 2 minutes.
- Allow configuring the max HTTP pending connections:
- Default is 10000 (the same as S3 SDK default).
Ext-ref: MB-67039
Change-Id: I25ac7b00fea6433f0cb72b6bbd0f00b5356addb9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19870
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index d71912c..7ea1cb4 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -17,7 +17,9 @@
"cloud.max.read.requests.per.second" : 1500,
"cloud.max.write.requests.per.second" : 250,
"cloud.profiler.log.interval" : 5,
+ "cloud.requests.http.connection.acquire.timeout" : 120,
"cloud.requests.max.http.connections" : 1000,
+ "cloud.requests.max.pending.http.connections" : 10000,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
"cloud.storage.bucket" : "",
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index f16ae94c..e5e0da6 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -17,7 +17,9 @@
"cloud.max.read.requests.per.second" : 1500,
"cloud.max.write.requests.per.second" : 250,
"cloud.profiler.log.interval" : 5,
+ "cloud.requests.http.connection.acquire.timeout" : 120,
"cloud.requests.max.http.connections" : 1000,
+ "cloud.requests.max.pending.http.connections" : 10000,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
"cloud.storage.bucket" : "",
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 9871dfc..a218009 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -17,7 +17,9 @@
"cloud.max.read.requests.per.second" : 1500,
"cloud.max.write.requests.per.second" : 250,
"cloud.profiler.log.interval" : 5,
+ "cloud.requests.http.connection.acquire.timeout" : 120,
"cloud.requests.max.http.connections" : 1000,
+ "cloud.requests.max.pending.http.connections" : 10000,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
"cloud.storage.bucket" : "",
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
index e0449b6..025af08 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
@@ -42,6 +42,8 @@
private final int readMaxRequestsPerSeconds;
private final int writeMaxRequestsPerSeconds;
private final int requestsMaxHttpConnections;
+ private final int requestsMaxPendingHttpConnections;
+ private final int requestsHttpConnectionAcquireTimeout;
private final boolean forcePathStyle;
private final boolean disableSslVerify;
private final boolean storageListEventuallyConsistent;
@@ -49,13 +51,14 @@
public S3ClientConfig(String region, String endpoint, String prefix, boolean anonymousAuth,
long profilerLogInterval, int writeBufferSize) {
this(region, endpoint, prefix, anonymousAuth, profilerLogInterval, writeBufferSize, 1, 0, 0, 0, false, false,
- false);
+ false, 0, 0);
}
private S3ClientConfig(String region, String endpoint, String prefix, boolean anonymousAuth,
long profilerLogInterval, int writeBufferSize, long tokenAcquireTimeout, int writeMaxRequestsPerSeconds,
int readMaxRequestsPerSeconds, int requestsMaxHttpConnections, boolean forcePathStyle,
- boolean disableSslVerify, boolean storageListEventuallyConsistent) {
+ boolean disableSslVerify, boolean storageListEventuallyConsistent, int requestsMaxPendingHttpConnections,
+ int requestsHttpConnectionAcquireTimeout) {
this.region = Objects.requireNonNull(region, "region");
this.endpoint = endpoint;
this.prefix = Objects.requireNonNull(prefix, "prefix");
@@ -66,6 +69,8 @@
this.writeMaxRequestsPerSeconds = writeMaxRequestsPerSeconds;
this.readMaxRequestsPerSeconds = readMaxRequestsPerSeconds;
this.requestsMaxHttpConnections = requestsMaxHttpConnections;
+ this.requestsMaxPendingHttpConnections = requestsMaxPendingHttpConnections;
+ this.requestsHttpConnectionAcquireTimeout = requestsHttpConnectionAcquireTimeout;
this.forcePathStyle = forcePathStyle;
this.disableSslVerify = disableSslVerify;
this.storageListEventuallyConsistent = storageListEventuallyConsistent;
@@ -78,7 +83,9 @@
cloudProperties.getTokenAcquireTimeout(), cloudProperties.getWriteMaxRequestsPerSecond(),
cloudProperties.getReadMaxRequestsPerSecond(), cloudProperties.getRequestsMaxHttpConnections(),
cloudProperties.isStorageForcePathStyle(), cloudProperties.isStorageDisableSSLVerify(),
- cloudProperties.isStorageListEventuallyConsistent());
+ cloudProperties.isStorageListEventuallyConsistent(),
+ cloudProperties.getRequestsMaxPendingHttpConnections(),
+ cloudProperties.getRequestsHttpConnectionAcquireTimeout());
}
public static S3ClientConfig of(Map<String, String> configuration, int writeBufferSize) {
@@ -140,6 +147,14 @@
return requestsMaxHttpConnections;
}
+ public int getRequestsMaxPendingHttpConnections() {
+ return requestsMaxPendingHttpConnections;
+ }
+
+ public int getRequestsHttpConnectionAcquireTimeout() {
+ return requestsHttpConnectionAcquireTimeout;
+ }
+
public boolean isDisableSslVerify() {
return disableSslVerify;
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index 8a28f53..81b96d7 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -27,6 +27,7 @@
import java.io.InputStream;
import java.net.URI;
import java.nio.ByteBuffer;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
@@ -352,6 +353,14 @@
customHttpConfigBuilder.put(SdkHttpConfigurationOption.MAX_CONNECTIONS,
config.getRequestsMaxHttpConnections());
}
+ if (config.getRequestsMaxPendingHttpConnections() > 0) {
+ customHttpConfigBuilder.put(SdkHttpConfigurationOption.MAX_PENDING_CONNECTION_ACQUIRES,
+ config.getRequestsMaxPendingHttpConnections());
+ }
+ if (config.getRequestsHttpConnectionAcquireTimeout() > 0) {
+ customHttpConfigBuilder.put(SdkHttpConfigurationOption.CONNECTION_ACQUIRE_TIMEOUT,
+ Duration.ofSeconds(config.getRequestsHttpConnectionAcquireTimeout()));
+ }
if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
builder.endpointOverride(URI.create(config.getEndpoint()));
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
index b5259e9..4d27c5a 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.net.URI;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -193,6 +194,18 @@
if (config.isDisableSslVerify()) {
customHttpConfigBuilder.put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES, true);
}
+ if (config.getRequestsMaxHttpConnections() > 0) {
+ customHttpConfigBuilder.put(SdkHttpConfigurationOption.MAX_CONNECTIONS,
+ config.getRequestsMaxHttpConnections());
+ }
+ if (config.getRequestsMaxPendingHttpConnections() > 0) {
+ customHttpConfigBuilder.put(SdkHttpConfigurationOption.MAX_PENDING_CONNECTION_ACQUIRES,
+ config.getRequestsMaxPendingHttpConnections());
+ }
+ if (config.getRequestsHttpConnectionAcquireTimeout() > 0) {
+ customHttpConfigBuilder.put(SdkHttpConfigurationOption.CONNECTION_ACQUIRE_TIMEOUT,
+ Duration.ofSeconds(config.getRequestsHttpConnectionAcquireTimeout()));
+ }
SdkAsyncHttpClient nettyHttpClient =
NettyNioAsyncHttpClient.builder().buildWithDefaults(customHttpConfigBuilder.build());
builder.httpClient(nettyHttpClient);
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
index 4354e63..87eace3 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
@@ -66,6 +66,8 @@
StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.MEGABYTE)),
CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD(POSITIVE_INTEGER, 50),
CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS(POSITIVE_INTEGER, 1000),
+ CLOUD_REQUESTS_MAX_PENDING_HTTP_CONNECTIONS(POSITIVE_INTEGER, 10000),
+ CLOUD_REQUESTS_HTTP_CONNECTION_ACQUIRE_TIMEOUT(POSITIVE_INTEGER, 120),
CLOUD_STORAGE_FORCE_PATH_STYLE(BOOLEAN, false),
CLOUD_STORAGE_DISABLE_SSL_VERIFY(BOOLEAN, false),
CLOUD_STORAGE_LIST_EVENTUALLY_CONSISTENT(BOOLEAN, false);
@@ -101,6 +103,8 @@
case CLOUD_WRITE_BUFFER_SIZE:
case CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD:
case CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS:
+ case CLOUD_REQUESTS_MAX_PENDING_HTTP_CONNECTIONS:
+ case CLOUD_REQUESTS_HTTP_CONNECTION_ACQUIRE_TIMEOUT:
case CLOUD_STORAGE_FORCE_PATH_STYLE:
case CLOUD_STORAGE_DISABLE_SSL_VERIFY:
case CLOUD_STORAGE_LIST_EVENTUALLY_CONSISTENT:
@@ -131,7 +135,7 @@
+ " request to open it. 'selective' caching will act as the 'lazy' policy; however, "
+ " it allows to use the local disk(s) as a cache, where pages and indexes can be "
+ " cached or evicted according to the pressure imposed on the local disks."
- + " (default: 'lazy')";
+ + " (default: 'selective')";
case CLOUD_STORAGE_ALLOCATION_PERCENTAGE:
return "The percentage of the total disk space that should be allocated for data storage when the"
+ " 'selective' caching policy is used. The remaining will act as a buffer for "
@@ -157,9 +161,8 @@
+ " CLOUD_STORAGE_SWEEP_THRESHOLD_PERCENTAGE."
+ " (default: 0. I.e., CLOUD_STORAGE_SWEEP_THRESHOLD_PERCENTAGE will be used by default)";
case CLOUD_PROFILER_LOG_INTERVAL:
- return "The waiting time (in minutes) to log cloud request statistics (default: 0, which means"
- + " the profiler is disabled by default). The minimum is 1 minute."
- + " NOTE: Enabling the profiler could perturb the performance of cloud requests";
+ return "The waiting time (in minutes) to log cloud request statistics. The minimum is 1 minute."
+ + " Note: by default, the logging is disabled. Enabling it could perturb the performance of cloud requests";
case CLOUD_ACQUIRE_TOKEN_TIMEOUT:
return "The waiting time (in milliseconds) if a requesting thread failed to acquire a token if the"
+ " rate limit of cloud requests exceeded (default: 100, min: 1, and max: 5000)";
@@ -172,7 +175,12 @@
case CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD:
return "The number of cloud reads for re-evaluating an eviction plan. (default: 50)";
case CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS:
- return "The maximum number of HTTP connections to use for cloud requests per node. (default: 1000)";
+ return "The maximum number of HTTP connections to use concurrently for cloud requests per node. (default: 1000)";
+ case CLOUD_REQUESTS_MAX_PENDING_HTTP_CONNECTIONS:
+ return "The maximum number of HTTP connections allowed to wait for a connection per node. (default: 10000)";
+ case CLOUD_REQUESTS_HTTP_CONNECTION_ACQUIRE_TIMEOUT:
+ return "The waiting time (in seconds) to acquire an HTTP connection before failing the request."
+ + " (default: 120 seconds)";
case CLOUD_STORAGE_FORCE_PATH_STYLE:
return "Indicates whether or not to force path style when accessing the cloud storage. (default:"
+ " false)";
@@ -282,6 +290,14 @@
return accessor.getInt(Option.CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS);
}
+ public int getRequestsMaxPendingHttpConnections() {
+ return accessor.getInt(Option.CLOUD_REQUESTS_MAX_PENDING_HTTP_CONNECTIONS);
+ }
+
+ public int getRequestsHttpConnectionAcquireTimeout() {
+ return accessor.getInt(Option.CLOUD_REQUESTS_HTTP_CONNECTION_ACQUIRE_TIMEOUT);
+ }
+
public boolean isStorageForcePathStyle() {
return accessor.getBoolean(Option.CLOUD_STORAGE_FORCE_PATH_STYLE);
}