[ASTERIXDB-3423][CONF] Add compiler property for COPY TO buffer size
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add property 'COMPILER_COPY_TO_WRITE_BUFFER_SIZE' to configure
COPY TO write buffer size. Default is 8MB with a minimum of 5MB.
- Enforce a minimum of 5MB for 'CLOUD_WRITE_BUFFER_SIZE'.
- Remove property 'COMPILER_ENABLE_DB_RESOLUTION'.
Change-Id: I50bebc4b8b683889855cb5dd048ab27d7193ff76
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18370
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Tested-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
index 8cd2487..f19dea6 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
@@ -73,7 +73,6 @@
import org.apache.asterix.common.cluster.IGlobalTxManager;
import org.apache.asterix.common.config.AsterixExtension;
import org.apache.asterix.common.config.CloudProperties;
-import org.apache.asterix.common.config.CompilerProperties;
import org.apache.asterix.common.config.ExtensionProperties;
import org.apache.asterix.common.config.ExternalProperties;
import org.apache.asterix.common.config.GlobalConfig;
@@ -171,10 +170,8 @@
new ReplicationProperties(PropertiesAccessor.getInstance(ccServiceCtx.getAppConfig()));
INcLifecycleCoordinator lifecycleCoordinator = createNcLifeCycleCoordinator(repProp.isReplicationEnabled());
componentProvider = new StorageComponentProvider();
- boolean isDbResolutionEnabled =
- ccServiceCtx.getAppConfig().getBoolean(CompilerProperties.Option.COMPILER_ENABLE_DB_RESOLUTION);
boolean cloudDeployment = ccServiceCtx.getAppConfig().getBoolean(CLOUD_DEPLOYMENT);
- boolean useDatabaseResolution = cloudDeployment && isDbResolutionEnabled;
+ boolean useDatabaseResolution = cloudDeployment;
INamespaceResolver namespaceResolver = createNamespaceResolver(useDatabaseResolution);
INamespacePathResolver namespacePathResolver = new NamespacePathResolver(useDatabaseResolution);
ccExtensionManager = new CCExtensionManager(new ArrayList<>(getExtensions()), namespaceResolver, ccServiceCtx);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index d02dc4f..0ff8796 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -61,7 +61,6 @@
import org.apache.asterix.common.api.IPropertiesFactory;
import org.apache.asterix.common.api.IReceptionistFactory;
import org.apache.asterix.common.config.AsterixExtension;
-import org.apache.asterix.common.config.CompilerProperties;
import org.apache.asterix.common.config.ExtensionProperties;
import org.apache.asterix.common.config.ExternalProperties;
import org.apache.asterix.common.config.GlobalConfig;
@@ -160,10 +159,8 @@
}
MetadataBuiltinFunctions.init();
- boolean isDbResolutionEnabled =
- ncServiceCtx.getAppConfig().getBoolean(CompilerProperties.Option.COMPILER_ENABLE_DB_RESOLUTION);
boolean cloudDeployment = ncServiceCtx.getAppConfig().getBoolean(CLOUD_DEPLOYMENT);
- boolean useDatabaseResolution = cloudDeployment && isDbResolutionEnabled;
+ boolean useDatabaseResolution = cloudDeployment;
INamespaceResolver namespaceResolver = createNamespaceResolver(useDatabaseResolution);
NamespacePathResolver namespacePathResolver = new NamespacePathResolver(useDatabaseResolution);
ncExtensionManager = new NCExtensionManager(new ArrayList<>(getExtensions()), cloudDeployment,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 93b92bc..107de00 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -29,6 +29,7 @@
"compiler.batch.lookup" : true,
"compiler.cbo" : true,
"compiler.column.filter" : true,
+ "compiler.copy.to.write.buffer.size" : 8388608,
"compiler\.external\.field\.pushdown" : true,
"compiler.forcejoinorder" : false,
"compiler\.framesize" : 32768,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index a5e70a5..bdeedd6 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -29,6 +29,7 @@
"compiler.batch.lookup" : true,
"compiler.cbo" : true,
"compiler.column.filter" : true,
+ "compiler.copy.to.write.buffer.size" : 8388608,
"compiler\.external\.field\.pushdown" : true,
"compiler.forcejoinorder" : false,
"compiler\.framesize" : 32768,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index cc1db05..96fa23c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -29,6 +29,7 @@
"compiler.batch.lookup" : true,
"compiler.cbo" : true,
"compiler.column.filter" : true,
+ "compiler.copy.to.write.buffer.size" : 8388608,
"compiler\.external\.field\.pushdown" : true,
"compiler.forcejoinorder" : false,
"compiler\.framesize" : 32768,
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
index c98e0e8..4357b37 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
@@ -22,7 +22,6 @@
import org.apache.asterix.common.config.CloudProperties;
import org.apache.asterix.external.util.aws.s3.S3Constants;
-import org.apache.hyracks.util.StorageUtil;
import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
@@ -55,7 +54,7 @@
cloudProperties.getProfilerLogInterval(), cloudProperties.getWriteBufferSize());
}
- public static S3ClientConfig of(Map<String, String> configuration) {
+ public static S3ClientConfig of(Map<String, String> configuration, int writeBufferSize) {
// Used to determine local vs. actual S3
String endPoint = configuration.getOrDefault(S3Constants.SERVICE_END_POINT_FIELD_NAME, "");
// Disabled
@@ -65,7 +64,6 @@
String region = "";
String prefix = "";
boolean anonymousAuth = false;
- int writeBufferSize = StorageUtil.getIntSizeInBytes(5, StorageUtil.StorageUnit.MEGABYTE);
return new S3ClientConfig(region, endPoint, prefix, anonymousAuth, profilerLogInterval, writeBufferSize);
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java
index d73957e..589ee79 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java
@@ -50,12 +50,14 @@
protected final Map<String, String> configuration;
protected final SourceLocation pathSourceLocation;
protected final String staticPath;
+ protected final int writeBufferSize;
protected transient ICloudClient cloudClient;
AbstractCloudExternalFileWriterFactory(ExternalFileWriterConfiguration externalConfig) {
configuration = externalConfig.getConfiguration();
pathSourceLocation = externalConfig.getPathSourceLocation();
staticPath = externalConfig.getStaticPath();
+ writeBufferSize = externalConfig.getWriteBufferSize();
}
abstract ICloudClient createCloudClient() throws CompilationException;
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
index dcaf488..e07acc0 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
@@ -62,7 +62,7 @@
@Override
ICloudClient createCloudClient() throws CompilationException {
- S3ClientConfig config = S3ClientConfig.of(configuration);
+ S3ClientConfig config = S3ClientConfig.of(configuration, writeBufferSize);
return new S3CloudClient(config, S3Utils.buildAwsS3Client(configuration),
ICloudGuardian.NoOpCloudGuardian.INSTANCE);
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
index 5c612dc..7e7ac69 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
@@ -24,6 +24,7 @@
import static org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER;
import static org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER;
import static org.apache.hyracks.control.common.config.OptionTypes.STRING;
+import static org.apache.hyracks.control.common.config.OptionTypes.getRangedIntegerType;
import static org.apache.hyracks.util.StorageUtil.StorageUnit.GIGABYTE;
import java.util.concurrent.TimeUnit;
@@ -57,7 +58,9 @@
CLOUD_STORAGE_DEBUG_MODE_ENABLED(BOOLEAN, false),
CLOUD_STORAGE_DEBUG_SWEEP_THRESHOLD_SIZE(LONG_BYTE_UNIT, StorageUtil.getLongSizeInBytes(1, GIGABYTE)),
CLOUD_PROFILER_LOG_INTERVAL(NONNEGATIVE_INTEGER, 5),
- CLOUD_WRITE_BUFFER_SIZE(POSITIVE_INTEGER, StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.MEGABYTE));
+ CLOUD_WRITE_BUFFER_SIZE(
+ getRangedIntegerType(5, Integer.MAX_VALUE),
+ StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.MEGABYTE));
private final IOptionType interpreter;
private final Object defaultValue;
@@ -142,7 +145,7 @@
+ " the profiler is disabled by default). The minimum is 1 minute."
+ " NOTE: Enabling the profiler could perturb the performance of cloud requests";
case CLOUD_WRITE_BUFFER_SIZE:
- return "The write buffer size in bytes. (default: 8MB)";
+ return "The write buffer size in bytes. (default: 8MB, min: 5MB)";
default:
throw new IllegalStateException("NYI: " + this);
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 042516e..cfe7ce8 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -25,6 +25,7 @@
import static org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER;
import static org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER;
import static org.apache.hyracks.control.common.config.OptionTypes.STRING;
+import static org.apache.hyracks.control.common.config.OptionTypes.getRangedIntegerType;
import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE;
@@ -124,12 +125,14 @@
BOOLEAN,
AlgebricksConfig.COLUMN_FILTER_DEFAULT,
"Enable/disable the use of column min/max filters"),
- //TODO(DB): remove after
- COMPILER_ENABLE_DB_RESOLUTION(BOOLEAN, true, "Enable/disable the resolution of namespaces to database"),
COMPILER_RUNTIME_MEMORY_OVERHEAD(
NONNEGATIVE_INTEGER,
5,
- "A percentage of the job's required memory to be added to account for runtime memory overhead");
+ "A percentage of the job's required memory to be added to account for runtime memory overhead"),
+ COMPILER_COPY_TO_WRITE_BUFFER_SIZE(
+ getRangedIntegerType(5, Integer.MAX_VALUE),
+ StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.MEGABYTE),
+ "The COPY TO write buffer size in bytes. (default: 8MB, min: 5MB)");
private final IOptionType type;
private final Object defaultValue;
@@ -163,8 +166,7 @@
@Override
public boolean hidden() {
- return this == COMPILER_EXTERNALSCANMEMORY || this == COMPILER_CBOTEST
- || this == COMPILER_ENABLE_DB_RESOLUTION;
+ return this == COMPILER_EXTERNALSCANMEMORY || this == COMPILER_CBOTEST;
}
}
@@ -323,4 +325,8 @@
public int getRuntimeMemoryOverheadPercentage() {
return accessor.getInt(Option.COMPILER_RUNTIME_MEMORY_OVERHEAD);
}
+
+ public int getCopyToWriteBufferSize() {
+ return accessor.getInt(Option.COMPILER_COPY_TO_WRITE_BUFFER_SIZE);
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
index 8ca428b..b8583d0 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
@@ -86,8 +86,10 @@
IWriteDataSink sink, String staticPath, SourceLocation pathExpressionLocation) {
Map<String, String> params = sink.getConfiguration();
boolean singleNodeCluster = isSingleNodeCluster(appCtx);
+ int copyToWriteBufferSize = appCtx.getCompilerProperties().getCopyToWriteBufferSize();
- return new ExternalFileWriterConfiguration(params, pathExpressionLocation, staticPath, singleNodeCluster);
+ return new ExternalFileWriterConfiguration(params, pathExpressionLocation, staticPath, singleNodeCluster,
+ copyToWriteBufferSize);
}
private static boolean isSingleNodeCluster(ICcApplicationContext appCtx) {
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalFileWriterConfiguration.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalFileWriterConfiguration.java
index b62a07a..080fa82 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalFileWriterConfiguration.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalFileWriterConfiguration.java
@@ -27,13 +27,15 @@
private final SourceLocation pathSourceLocation;
private final String staticPath;
private final boolean singleNodeCluster;
+ private final int writeBufferSize;
public ExternalFileWriterConfiguration(Map<String, String> configuration, SourceLocation pathSourceLocation,
- String staticPath, boolean singleNodeCluster) {
+ String staticPath, boolean singleNodeCluster, int writeBufferSize) {
this.configuration = configuration;
this.pathSourceLocation = pathSourceLocation;
this.staticPath = staticPath;
this.singleNodeCluster = singleNodeCluster;
+ this.writeBufferSize = writeBufferSize;
}
public Map<String, String> getConfiguration() {
@@ -51,4 +53,8 @@
public boolean isSingleNodeCluster() {
return singleNodeCluster;
}
+
+ public int getWriteBufferSize() {
+ return writeBufferSize;
+ }
}