[ASTERIXDB-3437][STO] Make eviction plan reevaluation threshold configurable
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
This patch makes eviction plan reevaluation threshold configurable.
The default value is set to 50 operation that require cloud read,
(e.g., running 50 query that require cloud read).
Change-Id: I6b937610982c13f16552f8aff4acc7608b45ee7d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18384
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ritik Raj <raj.ritik9835@gmail.com>
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 107de00..805f785 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -10,6 +10,7 @@
"active\.suspend\.timeout" : 3600,
"azure.request.timeout" : 120,
"cloud.deployment" : false,
+ "cloud.eviction.plan.reevaluate.threshold" : 50,
"cloud.profiler.log.interval" : 5,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index bdeedd6..35a488a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -10,6 +10,7 @@
"active\.suspend\.timeout" : 3600,
"azure.request.timeout" : 120,
"cloud.deployment" : false,
+ "cloud.eviction.plan.reevaluate.threshold" : 50,
"cloud.profiler.log.interval" : 5,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 96fa23c..13e7a4e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -10,6 +10,7 @@
"active\.suspend\.timeout" : 3600,
"azure.request.timeout" : 120,
"cloud.deployment" : false,
+ "cloud.eviction.plan.reevaluate.threshold" : 50,
"cloud.profiler.log.interval" : 5,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudConfigurator.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudConfigurator.java
index 3c0f1df..dced5b0 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudConfigurator.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudConfigurator.java
@@ -124,8 +124,8 @@
resourceCacheManager, cloudIOManager, numOfSweepThreads, maxSweepQueueSize, physicalDrive,
diskBufferCache, fileInfoMap, inactiveThreshold);
- IDiskCacheMonitoringService diskCacheService =
- new CloudDiskCacheMonitoringAndPrefetchingService(executor, physicalDrive, monitorThread);
+ IDiskCacheMonitoringService diskCacheService = new CloudDiskCacheMonitoringAndPrefetchingService(executor,
+ physicalDrive, monitorThread, cloudProperties.getEvictionPlanReevaluationThreshold());
localIoManager.setSpaceMaker(monitorThread);
return diskCacheService;
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
index 7e7ac69..a4d3b7a 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
@@ -60,7 +60,8 @@
CLOUD_PROFILER_LOG_INTERVAL(NONNEGATIVE_INTEGER, 5),
CLOUD_WRITE_BUFFER_SIZE(
getRangedIntegerType(5, Integer.MAX_VALUE),
- StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.MEGABYTE));
+ StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.MEGABYTE)),
+ CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD(POSITIVE_INTEGER, 50);
private final IOptionType interpreter;
private final Object defaultValue;
@@ -88,6 +89,7 @@
case CLOUD_STORAGE_DEBUG_MODE_ENABLED:
case CLOUD_PROFILER_LOG_INTERVAL:
case CLOUD_WRITE_BUFFER_SIZE:
+ case CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD:
return Section.COMMON;
default:
return Section.NC;
@@ -146,6 +148,8 @@
+ " NOTE: Enabling the profiler could perturb the performance of cloud requests";
case CLOUD_WRITE_BUFFER_SIZE:
return "The write buffer size in bytes. (default: 8MB, min: 5MB)";
+ case CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD:
+ return "The number of cloud reads for re-evaluating an eviction plan. (default: 50)";
default:
throw new IllegalStateException("NYI: " + this);
}
@@ -224,4 +228,8 @@
public int getWriteBufferSize() {
return accessor.getInt(Option.CLOUD_WRITE_BUFFER_SIZE);
}
+
+ public int getEvictionPlanReevaluationThreshold() {
+ return accessor.getInt(Option.CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/cache/service/CloudDiskCacheMonitoringAndPrefetchingService.java b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/cache/service/CloudDiskCacheMonitoringAndPrefetchingService.java
index 4564856..b58a6bf 100644
--- a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/cache/service/CloudDiskCacheMonitoringAndPrefetchingService.java
+++ b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/cache/service/CloudDiskCacheMonitoringAndPrefetchingService.java
@@ -33,12 +33,14 @@
private final ExecutorService executor;
private final DiskCacheSweeperThread monitorThread;
private final IPhysicalDrive drive;
+ private final int evictionPlanReevaluationThreshold;
public CloudDiskCacheMonitoringAndPrefetchingService(ExecutorService executor, IPhysicalDrive drive,
- DiskCacheSweeperThread monitorThread) {
+ DiskCacheSweeperThread monitorThread, int evictionPlanReevaluationThreshold) {
this.executor = executor;
this.drive = drive;
this.monitorThread = monitorThread;
+ this.evictionPlanReevaluationThreshold = evictionPlanReevaluationThreshold;
}
@Override
@@ -81,4 +83,9 @@
public void request(AbstractPrefetchRequest request) throws HyracksDataException {
// TODO implement
}
+
+ @Override
+ public int getEvictionPlanReevaluationThreshold() {
+ return evictionPlanReevaluationThreshold;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/CloudColumnIndexDiskCacheManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/CloudColumnIndexDiskCacheManager.java
index 7187496..417322e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/CloudColumnIndexDiskCacheManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/CloudColumnIndexDiskCacheManager.java
@@ -48,11 +48,11 @@
private final ColumnSweepPlanner planner;
private final ColumnSweeper sweeper;
- public CloudColumnIndexDiskCacheManager(int numberOfPrimaryKeys, IColumnTupleProjector sweepProjector,
- IPhysicalDrive drive) {
+ public CloudColumnIndexDiskCacheManager(int numberOfPrimaryKeys, int evictionPlanReevaluationThreshold,
+ IColumnTupleProjector sweepProjector, IPhysicalDrive drive) {
this.sweepProjector = sweepProjector;
this.drive = drive;
- planner = new ColumnSweepPlanner(numberOfPrimaryKeys, System::nanoTime);
+ planner = new ColumnSweepPlanner(numberOfPrimaryKeys, evictionPlanReevaluationThreshold, System::nanoTime);
sweeper = new ColumnSweeper(numberOfPrimaryKeys);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlanner.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlanner.java
index 3875812..9d52603 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlanner.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlanner.java
@@ -57,6 +57,7 @@
private final BitSet reevaluatedPlan;
private final IntSet indexedColumns;
private final ISweepClock clock;
+ private final int evictionPlanReevaluationThreshold;
private int numberOfColumns;
private long lastAccess;
private int maxSize;
@@ -67,7 +68,7 @@
private int numberOfSweptColumns;
private int numberOfCloudRequests;
- public ColumnSweepPlanner(int numberOfPrimaryKeys, ISweepClock clock) {
+ public ColumnSweepPlanner(int numberOfPrimaryKeys, int evictionPlanReevaluationThreshold, ISweepClock clock) {
this.clock = clock;
active = new AtomicBoolean(false);
this.numberOfPrimaryKeys = numberOfPrimaryKeys;
@@ -77,6 +78,7 @@
plan = new BitSet();
reevaluatedPlan = new BitSet();
punchableThreshold = INITIAL_PUNCHABLE_THRESHOLD;
+ this.evictionPlanReevaluationThreshold = evictionPlanReevaluationThreshold;
}
public boolean isActive() {
@@ -235,7 +237,7 @@
}
private void resetPlanIfNeeded() {
- if (numberOfCloudRequests < REEVALUATE_PLAN_THRESHOLD) {
+ if (numberOfCloudRequests < evictionPlanReevaluationThreshold) {
return;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/LSMColumnBTreeUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/LSMColumnBTreeUtil.java
index 6335acd..04b4984 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/LSMColumnBTreeUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/utils/LSMColumnBTreeUtil.java
@@ -76,6 +76,7 @@
IColumnManager columnManager = columnManagerFactory.createColumnManager();
IColumnIndexDiskCacheManager diskCacheManager = diskCacheEnabled
? new CloudColumnIndexDiskCacheManager(columnManager.getNumberOfPrimaryKeys(),
+ diskCacheService.getEvictionPlanReevaluationThreshold(),
columnManager.getMergeColumnProjector(), diskCacheService.getPhysicalDrive())
: NoOpColumnIndexDiskCacheManager.INSTANCE;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/test/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlannerTest.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/test/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlannerTest.java
index a5a87c4..e4b3449 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/test/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlannerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/test/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/ColumnSweepPlannerTest.java
@@ -38,6 +38,7 @@
public class ColumnSweepPlannerTest {
private static final int MAX_MEGA_LEAF_NODE_SIZE = getIntSizeInBytes(10, StorageUtil.StorageUnit.MEGABYTE);
+ private static final int REEVALUATE_PLAN_THRESHOLD = 50;
private static final Random RANDOM = new Random(0);
private final DummySweepClock clock = new DummySweepClock();
@@ -46,7 +47,7 @@
int numberOfPrimaryKeys = 1;
int numberOfColumns = numberOfPrimaryKeys + 10;
int[] columnSizes = createNormalColumnSizes(numberOfPrimaryKeys, numberOfColumns);
- ColumnSweepPlanner planner = new ColumnSweepPlanner(numberOfPrimaryKeys, clock);
+ ColumnSweepPlanner planner = new ColumnSweepPlanner(numberOfPrimaryKeys, REEVALUATE_PLAN_THRESHOLD, clock);
IntList projectedColumns = new IntArrayList();
DummyColumnProjectionInfo info = new DummyColumnProjectionInfo(numberOfPrimaryKeys, QUERY, projectedColumns);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/disk/IDiskCacheMonitoringService.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/disk/IDiskCacheMonitoringService.java
index 4151c77..9497463 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/disk/IDiskCacheMonitoringService.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/disk/IDiskCacheMonitoringService.java
@@ -62,4 +62,9 @@
* @return physical drive
*/
IPhysicalDrive getPhysicalDrive();
+
+ /**
+ * @return threshold for re-evaluation of eviction plan
+ */
+ int getEvictionPlanReevaluationThreshold();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/disk/NoOpDiskCacheMonitoringService.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/disk/NoOpDiskCacheMonitoringService.java
index 5cbf5cb..ad153fe 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/disk/NoOpDiskCacheMonitoringService.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/disk/NoOpDiskCacheMonitoringService.java
@@ -62,4 +62,9 @@
public IPhysicalDrive getPhysicalDrive() {
return DummyPhysicalDrive.INSTANCE;
}
+
+ @Override
+ public int getEvictionPlanReevaluationThreshold() {
+ return 0;
+ }
}