ASTERIXDB-1569: the config file now supports unit, comma, and decimal.
- This patch fixes an issue that the config file only supports byte
value for the byte required parameters. Now, we can use unit (KB, MB, GB ...),
comma, and decimal.
Change-Id: I763d1b06e93f75775c0294da0d609c2bf3e102c9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1064
Reviewed-by: Yingyi Bu <buyingyi@gmail.com>
Reviewed-by: Till Westmann <tillw@apache.org>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
index 1d693f9..cbd5c93 100644
--- a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
+++ b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
@@ -52,19 +52,19 @@
</property>
<property>
<name>compiler.framesize</name>
- <value>32768</value>
+ <value>32KB</value>
</property>
<property>
<name>compiler.sortmemory</name>
- <value>327680</value>
+ <value>320KB</value>
</property>
<property>
<name>compiler.groupmemory</name>
- <value>163840</value>
+ <value>160KB</value>
</property>
<property>
<name>compiler.joinmemory</name>
- <value>163840</value>
+ <value>160KB</value>
</property>
<property>
<name>compiler.pregelix.home</name>
@@ -72,14 +72,14 @@
</property>
<property>
<name>storage.buffercache.pagesize</name>
- <value>32768</value>
+ <value>32KB</value>
<description>The page size in bytes for pages in the buffer cache.
(Default = "32768" // 32KB)
</description>
</property>
<property>
<name>storage.buffercache.size</name>
- <value>33554432</value>
+ <value>32MB</value>
<description>The size of memory allocated to the disk buffer cache.
The value should be a multiple of the buffer cache page size(Default
= "33554432" // 32MB)
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java
index 42f612a..5bfb5f8 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java
@@ -40,22 +40,22 @@
public long getSortMemorySize() {
return accessor.getProperty(COMPILER_SORTMEMORY_KEY, COMPILER_SORTMEMORY_DEFAULT,
- PropertyInterpreters.getLongPropertyInterpreter());
+ PropertyInterpreters.getLongBytePropertyInterpreter());
}
public long getJoinMemorySize() {
return accessor.getProperty(COMPILER_JOINMEMORY_KEY, COMPILER_JOINMEMORY_DEFAULT,
- PropertyInterpreters.getLongPropertyInterpreter());
+ PropertyInterpreters.getLongBytePropertyInterpreter());
}
public long getGroupMemorySize() {
return accessor.getProperty(COMPILER_GROUPMEMORY_KEY, COMPILER_GROUPMEMORY_DEFAULT,
- PropertyInterpreters.getLongPropertyInterpreter());
+ PropertyInterpreters.getLongBytePropertyInterpreter());
}
public int getFrameSize() {
return accessor.getProperty(COMPILER_FRAMESIZE_KEY, COMPILER_FRAMESIZE_DEFAULT,
- PropertyInterpreters.getIntegerPropertyInterpreter());
+ PropertyInterpreters.getIntegerBytePropertyInterpreter());
}
public String getPregelixHome() {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixFeedProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixFeedProperties.java
index e4d2913..0f3951e 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixFeedProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixFeedProperties.java
@@ -41,7 +41,7 @@
public long getMemoryComponentGlobalBudget() {
return accessor.getProperty(FEED_MEMORY_GLOBALBUDGET_KEY, FEED_MEMORY_GLOBALBUDGET_DEFAULT,
- PropertyInterpreters.getLongPropertyInterpreter());
+ PropertyInterpreters.getLongBytePropertyInterpreter());
}
public long getMemoryAvailableWaitTimeout() {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixReplicationProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixReplicationProperties.java
index 7f51bbd..eec4947 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixReplicationProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixReplicationProperties.java
@@ -260,7 +260,7 @@
public int getLogBufferPageSize() {
return accessor.getProperty(REPLICATION_LOG_BUFFER_PAGE_SIZE_KEY, REPLICATION_LOG_BUFFER_PAGE_SIZE_DEFAULT,
- PropertyInterpreters.getIntegerPropertyInterpreter());
+ PropertyInterpreters.getIntegerBytePropertyInterpreter());
}
public int getLogBufferNumOfPages() {
@@ -270,6 +270,6 @@
public int getLogBatchSize() {
return accessor.getProperty(REPLICATION_LOG_BATCH_SIZE_KEY, REPLICATION_LOG_BATCH_SIZE_DEFAULT,
- PropertyInterpreters.getIntegerPropertyInterpreter());
+ PropertyInterpreters.getIntegerBytePropertyInterpreter());
}
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
index 9fb4e94..5fdbc1c 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
@@ -55,12 +55,12 @@
public int getBufferCachePageSize() {
return accessor.getProperty(STORAGE_BUFFERCACHE_PAGESIZE_KEY, STORAGE_BUFFERCACHE_PAGESIZE_DEFAULT,
- PropertyInterpreters.getIntegerPropertyInterpreter());
+ PropertyInterpreters.getIntegerBytePropertyInterpreter());
}
public long getBufferCacheSize() {
return accessor.getProperty(STORAGE_BUFFERCACHE_SIZE_KEY, STORAGE_BUFFERCACHE_SIZE_DEFAULT,
- PropertyInterpreters.getLongPropertyInterpreter());
+ PropertyInterpreters.getLongBytePropertyInterpreter());
}
public int getBufferCacheNumPages() {
@@ -74,7 +74,7 @@
public int getMemoryComponentPageSize() {
return accessor.getProperty(STORAGE_MEMORYCOMPONENT_PAGESIZE_KEY, STORAGE_MEMORYCOMPONENT_PAGESIZE_DEFAULT,
- PropertyInterpreters.getIntegerPropertyInterpreter());
+ PropertyInterpreters.getIntegerBytePropertyInterpreter());
}
public int getMemoryComponentNumPages() {
@@ -96,7 +96,7 @@
public long getMemoryComponentGlobalBudget() {
return accessor.getProperty(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY,
- STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT, PropertyInterpreters.getLongPropertyInterpreter());
+ STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT, PropertyInterpreters.getLongBytePropertyInterpreter());
}
public double getBloomFilterFalsePositiveRate() {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java
index 356dad3..00bcdc9 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java
@@ -81,12 +81,12 @@
public int getLogBufferPageSize() {
return accessor.getProperty(TXN_LOG_BUFFER_PAGESIZE_KEY, TXN_LOG_BUFFER_PAGESIZE_DEFAULT,
- PropertyInterpreters.getIntegerPropertyInterpreter());
+ PropertyInterpreters.getIntegerBytePropertyInterpreter());
}
public long getLogPartitionSize() {
return accessor.getProperty(TXN_LOG_PARTITIONSIZE_KEY, TXN_LOG_PARTITIONSIZE_DEFAULT,
- PropertyInterpreters.getLongPropertyInterpreter());
+ PropertyInterpreters.getLongBytePropertyInterpreter());
}
public int getCheckpointLSNThreshold() {
@@ -131,6 +131,6 @@
public long getJobRecoveryMemorySize() {
return accessor.getProperty(TXN_JOB_RECOVERY_MEMORY_SIZE_KEY, TXN_JOB_RECOVERY_MEMORY_SIZE_DEFAULT,
- PropertyInterpreters.getLongPropertyInterpreter());
+ PropertyInterpreters.getLongBytePropertyInterpreter());
}
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertyInterpreters.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertyInterpreters.java
index b54bcc3..4dd07d8 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertyInterpreters.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertyInterpreters.java
@@ -20,6 +20,8 @@
import java.util.logging.Level;
+import org.apache.hyracks.util.StorageUtil;
+
public class PropertyInterpreters {
public static IPropertyInterpreter<Integer> getIntegerPropertyInterpreter() {
@@ -88,4 +90,30 @@
};
}
+ public static IPropertyInterpreter<Long> getLongBytePropertyInterpreter() {
+ return new IPropertyInterpreter<Long>() {
+ @Override
+ public Long interpret(String s) throws IllegalArgumentException {
+ try {
+ return StorageUtil.getByteValue(s);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+ };
+ }
+
+ public static IPropertyInterpreter<Integer> getIntegerBytePropertyInterpreter() {
+ return new IPropertyInterpreter<Integer>() {
+ @Override
+ public Integer interpret(String s) throws IllegalArgumentException {
+ try {
+ return (int) StorageUtil.getByteValue(s);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+ };
+ }
+
}
diff --git a/asterixdb/asterix-docker/docker/asterix-configuration.xml b/asterixdb/asterix-docker/docker/asterix-configuration.xml
index dd8d571..ea43da9 100644
--- a/asterixdb/asterix-docker/docker/asterix-configuration.xml
+++ b/asterixdb/asterix-docker/docker/asterix-configuration.xml
@@ -66,7 +66,7 @@
<property>
<name>storage.buffercache.pagesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The page size in bytes for pages in the buffer cache.
(Default = "131072" // 128KB)
</description>
@@ -74,7 +74,7 @@
<property>
<name>storage.buffercache.size</name>
- <value>536870912</value>
+ <value>512MB</value>
<description>The size of memory allocated to the disk buffer cache.
The value should be a multiple of the buffer cache page size(Default
= "536870912" // 512MB)
@@ -91,7 +91,7 @@
<property>
<name>storage.memorycomponent.pagesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The page size in bytes for pages allocated to memory
components. (Default = "131072" // 128KB)
</description>
@@ -122,7 +122,7 @@
<property>
<name>storage.memorycomponent.globalbudget</name>
- <value>536870912</value>
+ <value>512MB</value>
<description>The total size of memory in bytes that the sum of all
open memory
components cannot exceed. (Default = "536870192" // 512MB)
@@ -146,7 +146,7 @@
<property>
<name>txn.log.buffer.pagesize</name>
- <value>524288</value>
+ <value>512KB</value>
<description>The size of pages in the in-memory log buffer. (Default =
"524288" // 512KB)
</description>
@@ -154,7 +154,7 @@
<property>
<name>txn.log.partitionsize</name>
- <value>2147483648</value>
+ <value>2GB</value>
<description>The maximum size of a log file partition allowed before
rotating the log to the next partition. (Default = "2147483648" //
2GB)
@@ -221,7 +221,7 @@
<property>
<name>compiler.sortmemory</name>
- <value>33554432</value>
+ <value>32MB</value>
<description>The amount of memory in bytes given to sort operations.
(Default = "33554432" // 32mb)
</description>
@@ -229,7 +229,7 @@
<property>
<name>compiler.joinmemory</name>
- <value>33554432</value>
+ <value>32MB</value>
<description>The amount of memory in bytes given to join operations.
(Default = "33554432" // 32mb)
</description>
@@ -237,7 +237,7 @@
<property>
<name>compiler.framesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The Hyracks frame size that the compiler configures per
job. (Default = "131072" // 128KB)
</description>
diff --git a/asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml b/asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml
index af37d0b..c642cbb 100644
--- a/asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml
+++ b/asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml
@@ -37,7 +37,7 @@
<property>
<name>storage.buffercache.pagesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The page size in bytes for pages in the buffer cache.
(Default = "131072" // 128KB)
</description>
@@ -45,7 +45,7 @@
<property>
<name>storage.buffercache.size</name>
- <value>3221225472</value>
+ <value>3GB</value>
<description>[3GB] The size of memory allocated to the disk buffer cache.
The value should be a multiple of the buffer cache page size(Default
= "536870912" // 512MB)
@@ -63,7 +63,7 @@
<!-- Buffer size per dataset for in-memory components: 1GB -->
<property>
<name>storage.memorycomponent.pagesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The page size in bytes for pages allocated to memory
components. (Default = "131072" // 128KB)
</description>
@@ -95,8 +95,8 @@
<property>
<name>storage.memorycomponent.globalbudget</name>
- <value>4399824896</value>
- <description>[4GB + 8MB]The total size of memory in bytes that the sum of all
+ <value>4196MB</value>
+ <description>[4GB + 100MB]The total size of memory in bytes that the sum of all
open memory
components cannot exceed. (Default = "536870192" // 512MB)
</description>
@@ -119,7 +119,7 @@
<property>
<name>txn.log.buffer.pagesize</name>
- <value>8388608</value>
+ <value>8MB</value>
<description>[8MB]The size of pages in the in-memory log buffer. (Default =
"524288" // 512KB)
</description>
@@ -127,7 +127,7 @@
<property>
<name>txn.log.partitionsize</name>
- <value>2147483648</value>
+ <value>2GB</value>
<description>The maximum size of a log file partition allowed before
rotating the log to the next partition. (Default = "2147483648" //
2GB)
@@ -194,7 +194,7 @@
<property>
<name>compiler.sortmemory</name>
- <value>67108864</value>
+ <value>64MB</value>
<description>[64MB]The amount of memory in bytes given to sort operations.
(Default = "33554432" // 32mb)
</description>
@@ -202,7 +202,7 @@
<property>
<name>compiler.joinmemory</name>
- <value>67108864</value>
+ <value>64MB</value>
<description>[64MB]The amount of memory in bytes given to join operations.
(Default = "33554432" // 32mb)
</description>
@@ -210,7 +210,7 @@
<property>
<name>compiler.framesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The Hyracks frame size that the compiler configures per
job. (Default = "131072" // 128KB)
</description>
diff --git a/asterixdb/asterix-installer/src/main/resources/conf/asterix-configuration.xml b/asterixdb/asterix-installer/src/main/resources/conf/asterix-configuration.xml
index f6da341..a7c5c6d 100644
--- a/asterixdb/asterix-installer/src/main/resources/conf/asterix-configuration.xml
+++ b/asterixdb/asterix-installer/src/main/resources/conf/asterix-configuration.xml
@@ -41,7 +41,7 @@
<property>
<name>storage.buffercache.pagesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The page size in bytes for pages in the buffer cache.
(Default = "131072" // 128KB)
</description>
@@ -49,7 +49,7 @@
<property>
<name>storage.buffercache.size</name>
- <value>536870912</value>
+ <value>512MB</value>
<description>The size of memory allocated to the disk buffer cache.
The value should be a multiple of the buffer cache page size(Default
= "536870912" // 512MB)
@@ -66,7 +66,7 @@
<property>
<name>storage.memorycomponent.pagesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The page size in bytes for pages allocated to memory
components. (Default = "131072" // 128KB)
</description>
@@ -98,7 +98,7 @@
<property>
<name>storage.memorycomponent.globalbudget</name>
- <value>1073741824</value>
+ <value>1GB</value>
<description>The total size of memory in bytes that the sum of all
open memory
components cannot exceed. (Default = "536870192" // 512MB)
@@ -122,7 +122,7 @@
<property>
<name>txn.log.buffer.pagesize</name>
- <value>524288</value>
+ <value>512KB</value>
<description>The size of pages in the in-memory log buffer. (Default =
"524288" // 512KB)
</description>
@@ -130,7 +130,7 @@
<property>
<name>txn.log.partitionsize</name>
- <value>2147483648</value>
+ <value>2GB</value>
<description>The maximum size of a log file partition allowed before
rotating the log to the next partition. (Default = "2147483648" //
2GB)
@@ -197,7 +197,7 @@
<property>
<name>txn.job.recovery.memorysize</name>
- <value>67108864</value>
+ <value>64MB</value>
<description>The memory allocated per job during recovery.
(Default = "67108864" // 64MB)
</description>
@@ -205,23 +205,23 @@
<property>
<name>compiler.sortmemory</name>
- <value>33554432</value>
+ <value>32MB</value>
<description>The amount of memory in bytes given to sort operations.
- (Default = "33554432" // 32mb)
+ (Default = "33554432" // 32MB)
</description>
</property>
<property>
<name>compiler.joinmemory</name>
- <value>33554432</value>
+ <value>32MB</value>
<description>The amount of memory in bytes given to join operations.
- (Default = "33554432" // 32mb)
+ (Default = "33554432" // 32MB)
</description>
</property>
<property>
<name>compiler.framesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The Hyracks frame size that the compiler configures per
job. (Default = "131072" // 128KB)
</description>
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/asterix-configuration.xml b/asterixdb/asterix-installer/src/test/resources/integrationts/asterix-configuration.xml
index 4cd7fff..e36c33b 100644
--- a/asterixdb/asterix-installer/src/test/resources/integrationts/asterix-configuration.xml
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/asterix-configuration.xml
@@ -41,7 +41,7 @@
<property>
<name>storage.buffercache.pagesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The page size in bytes for pages in the buffer cache.
(Default = "131072" // 128KB)
</description>
@@ -49,7 +49,7 @@
<property>
<name>storage.buffercache.size</name>
- <value>536870912</value>
+ <value>512MB</value>
<description>The size of memory allocated to the disk buffer cache.
The value should be a multiple of the buffer cache page size(Default
= "536870912" // 512MB)
@@ -66,7 +66,7 @@
<property>
<name>storage.memorycomponent.pagesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The page size in bytes for pages allocated to memory
components. (Default = "131072" // 128KB)
</description>
@@ -98,7 +98,7 @@
<property>
<name>storage.memorycomponent.globalbudget</name>
- <value>1073741824</value>
+ <value>1GB</value>
<description>The total size of memory in bytes that the sum of all
open memory
components cannot exceed. (Default = "536870192" // 512MB)
@@ -122,7 +122,7 @@
<property>
<name>txn.log.buffer.pagesize</name>
- <value>524288</value>
+ <value>512KB</value>
<description>The size of pages in the in-memory log buffer. (Default =
"524288" // 512KB)
</description>
@@ -130,7 +130,7 @@
<property>
<name>txn.log.partitionsize</name>
- <value>2147483648</value>
+ <value>2GB</value>
<description>The maximum size of a log file partition allowed before
rotating the log to the next partition. (Default = "2147483648" //
2GB)
@@ -197,23 +197,31 @@
<property>
<name>compiler.sortmemory</name>
- <value>33554432</value>
+ <value>32MB</value>
<description>The amount of memory in bytes given to sort operations.
- (Default = "33554432" // 32mb)
+ (Default = "33554432" // 32MB)
</description>
</property>
<property>
<name>compiler.joinmemory</name>
- <value>33554432</value>
+ <value>32MB</value>
<description>The amount of memory in bytes given to join operations.
- (Default = "33554432" // 32mb)
+ (Default = "33554432" // 32MB)
+ </description>
+ </property>
+
+ <property>
+ <name>compiler.groupmemory</name>
+ <value>32MB</value>
+ <description>The amount of memory in bytes given to group-by operations.
+ (Default = "33554432" // 32MB)
</description>
</property>
<property>
<name>compiler.framesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The Hyracks frame size that the compiler configures per
job. (Default = "131072" // 128KB)
</description>
diff --git a/asterixdb/asterix-yarn/src/main/resources/base-asterix-configuration.xml b/asterixdb/asterix-yarn/src/main/resources/base-asterix-configuration.xml
index 778e51f..788cb76 100644
--- a/asterixdb/asterix-yarn/src/main/resources/base-asterix-configuration.xml
+++ b/asterixdb/asterix-yarn/src/main/resources/base-asterix-configuration.xml
@@ -41,7 +41,7 @@
<property>
<name>storage.buffercache.pagesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The page size in bytes for pages in the buffer cache.
(Default = "131072" // 128KB)
</description>
@@ -49,7 +49,7 @@
<property>
<name>storage.buffercache.size</name>
- <value>536870912</value>
+ <value>512MB</value>
<description>The size of memory allocated to the disk buffer cache.
The value should be a multiple of the buffer cache page size(Default
= "536870912" // 512MB)
@@ -66,7 +66,7 @@
<property>
<name>storage.memorycomponent.pagesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The page size in bytes for pages allocated to memory
components. (Default = "131072" // 128KB)
</description>
@@ -98,7 +98,7 @@
<property>
<name>storage.memorycomponent.globalbudget</name>
- <value>536870912</value>
+ <value>512MB</value>
<description>The total size of memory in bytes that the sum of all
open memory
components cannot exceed. (Default = "536870192" // 512MB)
@@ -122,7 +122,7 @@
<property>
<name>txn.log.buffer.pagesize</name>
- <value>524288</value>
+ <value>512KB</value>
<description>The size of pages in the in-memory log buffer. (Default =
"524288" // 512KB)
</description>
@@ -130,7 +130,7 @@
<property>
<name>txn.log.partitionsize</name>
- <value>2147483648</value>
+ <value>2GB</value>
<description>The maximum size of a log file partition allowed before
rotating the log to the next partition. (Default = "2147483648" //
2GB)
@@ -197,23 +197,23 @@
<property>
<name>compiler.sortmemory</name>
- <value>33554432</value>
+ <value>32MB</value>
<description>The amount of memory in bytes given to sort operations.
- (Default = "33554432" // 32mb)
+ (Default = "33554432" // 32MB)
</description>
</property>
<property>
<name>compiler.joinmemory</name>
- <value>33554432</value>
+ <value>32MB</value>
<description>The amount of memory in bytes given to join operations.
- (Default = "33554432" // 32mb)
+ (Default = "33554432" // 32MB)
</description>
</property>
<property>
<name>compiler.framesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The Hyracks frame size that the compiler configures per
job. (Default = "131072" // 128KB)
</description>
diff --git a/asterixdb/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml b/asterixdb/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml
index 778e51f..788cb76 100644
--- a/asterixdb/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml
+++ b/asterixdb/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml
@@ -41,7 +41,7 @@
<property>
<name>storage.buffercache.pagesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The page size in bytes for pages in the buffer cache.
(Default = "131072" // 128KB)
</description>
@@ -49,7 +49,7 @@
<property>
<name>storage.buffercache.size</name>
- <value>536870912</value>
+ <value>512MB</value>
<description>The size of memory allocated to the disk buffer cache.
The value should be a multiple of the buffer cache page size(Default
= "536870912" // 512MB)
@@ -66,7 +66,7 @@
<property>
<name>storage.memorycomponent.pagesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The page size in bytes for pages allocated to memory
components. (Default = "131072" // 128KB)
</description>
@@ -98,7 +98,7 @@
<property>
<name>storage.memorycomponent.globalbudget</name>
- <value>536870912</value>
+ <value>512MB</value>
<description>The total size of memory in bytes that the sum of all
open memory
components cannot exceed. (Default = "536870192" // 512MB)
@@ -122,7 +122,7 @@
<property>
<name>txn.log.buffer.pagesize</name>
- <value>524288</value>
+ <value>512KB</value>
<description>The size of pages in the in-memory log buffer. (Default =
"524288" // 512KB)
</description>
@@ -130,7 +130,7 @@
<property>
<name>txn.log.partitionsize</name>
- <value>2147483648</value>
+ <value>2GB</value>
<description>The maximum size of a log file partition allowed before
rotating the log to the next partition. (Default = "2147483648" //
2GB)
@@ -197,23 +197,23 @@
<property>
<name>compiler.sortmemory</name>
- <value>33554432</value>
+ <value>32MB</value>
<description>The amount of memory in bytes given to sort operations.
- (Default = "33554432" // 32mb)
+ (Default = "33554432" // 32MB)
</description>
</property>
<property>
<name>compiler.joinmemory</name>
- <value>33554432</value>
+ <value>32MB</value>
<description>The amount of memory in bytes given to join operations.
- (Default = "33554432" // 32mb)
+ (Default = "33554432" // 32MB)
</description>
</property>
<property>
<name>compiler.framesize</name>
- <value>131072</value>
+ <value>128KB</value>
<description>The Hyracks frame size that the compiler configures per
job. (Default = "131072" // 128KB)
</description>
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java
index 4607214..7025f15 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java
@@ -23,12 +23,23 @@
private static final int BASE = 1024, KB = BASE, MB = KB * BASE, GB = MB * BASE, TB = GB * BASE, PB = TB * BASE;
public enum StorageUnit {
- BYTE,
- KILOBYTE,
- MEGABYTE,
- GIGABYTE,
- TERABYTE,
- PETABYTE
+ BYTE("B"),
+ KILOBYTE("KB"),
+ MEGABYTE("MB"),
+ GIGABYTE("GB"),
+ TERABYTE("TB"),
+ PETABYTE("PB");
+
+ private final String unitTypeInLetter;
+
+ private StorageUnit(String unitTypeInLetter) {
+ this.unitTypeInLetter = unitTypeInLetter;
+ }
+
+ @Override
+ public String toString() {
+ return this.unitTypeInLetter;
+ }
}
private StorageUtil() {
@@ -36,23 +47,24 @@
}
public static int getSizeInBytes(final int size, final StorageUnit unit) {
- switch (unit) {
- case BYTE:
- return size;
- case KILOBYTE:
- return size * KB;
- case MEGABYTE:
- return size * MB;
- case GIGABYTE:
- return size * GB;
- case TERABYTE:
- return size * TB;
- default:
- throw new IllegalStateException("Unsupported unti: " + unit);
+ double result = getSizeInBytes((double) size, unit);
+ if (result > Integer.MAX_VALUE) {
+ throw new IllegalStateException("The given value:" + result + " is not within the integer range.");
+ } else {
+ return (int) result;
}
}
public static long getSizeInBytes(final long size, final StorageUnit unit) {
+ double result = getSizeInBytes((double) size, unit);
+ if (result > Long.MAX_VALUE) {
+ throw new IllegalStateException("The given value:" + result + " is not within the long range.");
+ } else {
+ return (long) result;
+ }
+ }
+
+ public static double getSizeInBytes(final double size, final StorageUnit unit) {
switch (unit) {
case BYTE:
return size;
@@ -67,7 +79,87 @@
case PETABYTE:
return size * PB;
default:
- throw new IllegalStateException("Unsupported unti: " + unit);
+ throw new IllegalStateException("Unsupported unit: " + unit);
+ }
+ }
+
+ /**
+ * Helper method to parse a byte unit string to its double value and unit
+ * (e.g., 10,345.8MB becomes Pair<10345.8, StorageUnit.MB>.)
+ *
+ * @throws HyracksException
+ */
+ public static ByteValueStringInfo parseByteUnitString(String s) {
+ String sSpaceRemoved = s.replaceAll(" ", "");
+ String sUpper = sSpaceRemoved.toUpperCase();
+ ByteValueStringInfo valueAndUnitType = new ByteValueStringInfo();
+
+ // Default type
+ StorageUtil.StorageUnit unitType = StorageUnit.BYTE;
+
+ // If the length is 1, it should only contain a digit number.
+ if (sUpper.length() == 1) {
+ if (Character.isDigit(sUpper.charAt(0))) {
+ unitType = StorageUnit.BYTE;
+ } else {
+ throw new IllegalStateException(
+ "The given string: " + s + " is not a byte unit string (e.g., 320KB or 1024).");
+ }
+ } else if (sUpper.length() > 1) {
+ String checkStr = sUpper.substring(sUpper.length() - 2);
+ boolean found = false;
+ for (StorageUnit unit : StorageUnit.values()) {
+ if (checkStr.equals(unit.toString())) {
+ unitType = unit;
+ found = true;
+ break;
+ }
+ }
+
+ if (!found) {
+ // The last suffix should be at least "B" or a digit to be qualified as byte unit string.
+ char lastChar = sUpper.charAt(sUpper.length() - 1);
+ if (sUpper.substring(sUpper.length() - 1).equals(StorageUnit.BYTE.toString())
+ || Character.isDigit(lastChar)) {
+ unitType = StorageUnit.BYTE;
+ } else {
+ throw new IllegalStateException(
+ "The given string: " + s + " is not a byte unit string (e.g., 320KB or 1024).");
+ }
+ }
+ } else {
+ // String length is zero. We can't parse this string.
+ throw new IllegalStateException(
+ "The given string: " + s + " is not a byte unit string (e.g., 320KB or 1024).");
+ }
+
+ // Strip all unit suffixes such as KB, MB ...
+ String sFinalVal = sUpper.replaceAll("[^\\.0123456789]", "");
+
+ // Return the digit and its unit type.
+ valueAndUnitType.value = Double.parseDouble(sFinalVal);
+ valueAndUnitType.unitType = unitType;
+
+ return valueAndUnitType;
+ }
+
+ /**
+ * Return byte value for the given string (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...)
+ *
+ * @throws HyracksException
+ */
+ public static long getByteValue(String s) {
+ try {
+ ByteValueStringInfo valueAndUnitType = parseByteUnitString(s);
+
+ double result = getSizeInBytes(valueAndUnitType.value, valueAndUnitType.unitType);
+ if (result > Long.MAX_VALUE) {
+ throw new IllegalStateException("The given value:" + result + " is not within the long range.");
+ } else {
+ return (long) result;
+ }
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(e);
}
}
@@ -85,4 +177,14 @@
final int baseValue = (63 - Long.numberOfLeadingZeros(bytes)) / 10;
return String.format("%.2f %sB", (double) bytes / (1L << (baseValue * 10)), " kMGTPE".charAt(baseValue));
}
+
+ private static class ByteValueStringInfo {
+ double value;
+ StorageUnit unitType;
+
+ ByteValueStringInfo() {
+ value = 0.0;
+ unitType = StorageUnit.BYTE;
+ }
+ }
}