ASTERIXDB-1455: Scale CharRecord Buffer Expansions
Reading a 30 MB record requires > 1500 buffer reallocations when increasing by
a fixed amount. Updated to scale up the buffer increases to reduce the number
of reallocations in case of large records.
Change-Id: I4822b2a78e1f411de94d2c717523df6b2b4a381e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/861
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
index 6b8e10d..f174962 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
@@ -69,10 +69,8 @@
throw new IOException(
"Record is too large!. Maximum record size is " + ExternalDataConstants.MAX_RECORD_SIZE);
}
- int newSize = len + ExternalDataConstants.DEFAULT_BUFFER_INCREMENT;
- if (newSize > ExternalDataConstants.MAX_RECORD_SIZE) {
- newSize = ExternalDataConstants.MAX_RECORD_SIZE;
- }
+ int newSize = Math.min((int)(len * ExternalDataConstants.DEFAULT_BUFFER_INCREMENT_FACTOR),
+ ExternalDataConstants.MAX_RECORD_SIZE);
value = Arrays.copyOf(value, newSize);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
index 45d01e9..f9ec114 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
@@ -64,7 +64,7 @@
reusableByteBuffer.clear();
if (reusableByteBuffer.remaining() < value.getLength()) {
reusableByteBuffer = ByteBuffer
- .allocateDirect(value.getLength() + ExternalDataConstants.DEFAULT_BUFFER_INCREMENT);
+ .allocateDirect((int)(value.getLength() * ExternalDataConstants.DEFAULT_BUFFER_INCREMENT_FACTOR));
}
reusableByteBuffer.put(value.getBytes(), 0, value.getLength());
reusableByteBuffer.flip();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 0fd76cc..cf2b7f3 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -206,7 +206,7 @@
* Size default values
*/
public static final int DEFAULT_BUFFER_SIZE = 4096;
- public static final int DEFAULT_BUFFER_INCREMENT = 2048;
+ public static final float DEFAULT_BUFFER_INCREMENT_FACTOR = 1.5F;
public static final int DEFAULT_QUEUE_SIZE = 64;
public static final int MAX_RECORD_SIZE = 32000000;