[NO ISSUE][OTH] Add some logging when join build fails to insert tuple
- user model changes: no
- storage format changes: no
- interface changes: no
Change-Id: I5e342019d4831bfef0a10d6154a080c3cb524256
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17363
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/PreferToSpillFullyOccupiedFramePolicy.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/PreferToSpillFullyOccupiedFramePolicy.java
index 12985c0..613a396 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/PreferToSpillFullyOccupiedFramePolicy.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/PreferToSpillFullyOccupiedFramePolicy.java
@@ -98,4 +98,14 @@
}
};
}
+
+ public String partitionsStatus() {
+ StringBuilder sb = new StringBuilder();
+ int numPartitions = bufferManager.getNumPartitions();
+ for (int p = 0; p < numPartitions; p++) {
+ sb.append("p:").append(p).append(",#t:").append(bufferManager.getNumTuples(p)).append(",s:")
+ .append(spilledStatus.get(p)).append(",s:").append(bufferManager.getPhysicalSize(p)).append('\n');
+ }
+ return sb.toString();
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java
index 7a9bb25..02a381d 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java
@@ -49,6 +49,8 @@
import org.apache.hyracks.dataflow.std.structures.ISerializableTable;
import org.apache.hyracks.dataflow.std.structures.SerializableHashTable;
import org.apache.hyracks.dataflow.std.structures.TuplePointer;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
/**
* This class mainly applies one level of HHJ on a pair of
@@ -56,6 +58,7 @@
*/
public class OptimizedHybridHashJoin {
+ private static final Logger LOGGER = LogManager.getLogger();
// Used for special probe BigObject which can not be held into the Join memory
private FrameTupleAppender bigFrameAppender;
@@ -155,12 +158,19 @@
int recordSize = VPartitionTupleBufferManager.calculateActualSize(null, accessorBuild.getTupleLength(tid));
double numFrames = (double) recordSize / (double) jobletCtx.getInitialFrameSize();
int victimPartition;
- if (numFrames > bufferManager.getConstrain().frameLimit(pid)
- || (victimPartition = spillPolicy.selectVictimPartition(pid)) < 0) {
+ int partitionFrameLimit = bufferManager.getConstrain().frameLimit(pid);
+ if (numFrames > partitionFrameLimit || (victimPartition = spillPolicy.selectVictimPartition(pid)) < 0) {
// insert request can never be satisfied
if (numFrames > memSizeInFrames || recordSize < jobletCtx.getInitialFrameSize()) {
// the tuple is greater than the memory budget or although the record is small we could not find
// a frame for it (possibly due to a bug)
+ String details = String.format(
+ "partition %s, tuple size %s, needed # frames %s, partition frame limit %s, join "
+ + "memory in frames %s, initial frame size %s",
+ pid, recordSize, numFrames, partitionFrameLimit, memSizeInFrames,
+ jobletCtx.getInitialFrameSize());
+ LOGGER.debug("can't insert tuple in join memory. {}", details);
+ LOGGER.debug("partitions status:\n{}", spillPolicy.partitionsStatus());
throw HyracksDataException.create(ErrorCode.INSUFFICIENT_MEMORY);
}
// Record is large but insertion failed either 1) we could not satisfy the request because of the