Skip to content

Commit

Permalink
[SPARK-23432][UI] Add executor peak jvm memory metrics in executors page
Browse files Browse the repository at this point in the history 
### What changes were proposed in this pull request?
Add executor peak jvm memory metrics in executors page

![image](https://user-images.githubusercontent.com/1633312/97767765-9121bf00-1adb-11eb-93c7-7912d9fe7826.png)

### Why are the changes needed?
Users can know executor peak jvm metrics on in executors page

### Does this PR introduce _any_ user-facing change?
Users can know executor peak jvm metrics on in executors page

### How was this patch tested?
Manually tested

Closes #30186 from warrenzhu25/23432.

Authored-by: Warren Zhu <warren.zhu25@gmail.com>
Signed-off-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
  • Loading branch information
@warrenzhu25 @HeartSaVioR
warrenzhu25 authored and HeartSaVioR committed Nov 6, 2020
1 parent 68c032c commit 93ad26b
Show file tree
Hide file tree
Showing 2 changed files with 64 additions and 4 deletions.
16 changes: 16 additions & 0 deletions core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,22 @@ <h4 class="title-table">Executors</h4>
<span data-toggle="tooltip" data-placement="top"
title="Memory used / total available memory for off heap storage of data like RDD partitions cached in memory.">
Off Heap Storage Memory</span></th>
<th>
<span data-toggle="tooltip" data-placement="top"
title="Peak onHeap / offHeap memory used by JVM.">
Peak JVM Memory OnHeap / OffHeap</span></th>
<th>
<span data-toggle="tooltip" data-placement="top"
title="Peak onHeap / offHeap memory used for execution. This refers to memory used for computation in shuffles, joins, user data structures, etc. See the Memory Management Overview documentation for more details.">
Peak Execution Memory OnHeap / OffHeap</span></th>
<th>
<span data-toggle="tooltip" data-placement="top"
title="Peak storage onHeap / offHeap memory used for storage of data like RDD partitions cached in memory.">
Peak Storage Memory OnHeap / OffHeap</span></th>
<th>
<span data-toggle="tooltip" data-placement="top"
title="Peak direct byte buffer / memory-mapped buffer pool memory used by JVM. This refers to BufferPoolMXBean with form 'java.nio:type=BufferPool,name=direct' and 'java.nio:type=BufferPool,name=mapped'.">
Peak Pool Memory Direct / Mapped</span></th>
<th>Disk Used</th>
<th>Cores</th>
<th>Resources</th>
Expand Down
52 changes: 48 additions & 4 deletions core/src/main/resources/org/apache/spark/ui/static/executorspage.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ function totalDurationColor(totalGCTime, totalDuration) {
}

var sumOptionalColumns = [3, 4];
var execOptionalColumns = [5, 6, 9, 10];
var execOptionalColumns = [5, 6, 7, 8, 9, 10, 13, 14];
var execDataTable;
var sumDataTable;

Expand Down Expand Up @@ -412,6 +412,42 @@ $(document).ready(function () {
formatBytes(row.memoryMetrics.totalOffHeapStorageMemory, type));
}
},
{
data: function (row, type) {
if (type !== 'display')
return row.peakMemoryMetrics.JVMHeapMemory;
else
return (formatBytes(row.peakMemoryMetrics.JVMHeapMemory, type) + ' / ' +
formatBytes(row.peakMemoryMetrics.JVMOffHeapMemory, type));
}
},
{
data: function (row, type) {
if (type !== 'display')
return row.peakMemoryMetrics.OnHeapExecutionMemory;
else
return (formatBytes(row.peakMemoryMetrics.OnHeapExecutionMemory, type) + ' / ' +
formatBytes(row.peakMemoryMetrics.OffHeapExecutionMemory, type));
}
},
{
data: function (row, type) {
if (type !== 'display')
return row.peakMemoryMetrics.OnHeapStorageMemory;
else
return (formatBytes(row.peakMemoryMetrics.OnHeapStorageMemory, type) + ' / ' +
formatBytes(row.peakMemoryMetrics.OffHeapStorageMemory, type));
}
},
{
data: function (row, type) {
if (type !== 'display')
return row.peakMemoryMetrics.DirectPoolMemory;
else
return (formatBytes(row.peakMemoryMetrics.DirectPoolMemory, type) + ' / ' +
formatBytes(row.peakMemoryMetrics.MappedPoolMemory, type));
}
},
{data: 'diskUsed', render: formatBytes},
{data: 'totalCores'},
{name: 'resourcesCol', data: 'resources', render: formatResourceCells, orderable: false},
Expand Down Expand Up @@ -462,8 +498,12 @@ $(document).ready(function () {
"columnDefs": [
{"visible": false, "targets": 5},
{"visible": false, "targets": 6},
{"visible": false, "targets": 7},
{"visible": false, "targets": 8},
{"visible": false, "targets": 9},
{"visible": false, "targets": 10}
{"visible": false, "targets": 10},
{"visible": false, "targets": 13},
{"visible": false, "targets": 14}
],
"deferRender": true
};
Expand Down Expand Up @@ -571,8 +611,12 @@ $(document).ready(function () {
"<div><input type='checkbox' class='toggle-vis' id='select-all-box'>Select All</div>" +
"<div id='on_heap_memory' class='on-heap-memory-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='3' data-exec-col-idx='5'>On Heap Memory</div>" +
"<div id='off_heap_memory' class='off-heap-memory-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='4' data-exec-col-idx='6'>Off Heap Memory</div>" +
"<div id='extra_resources' class='resources-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='' data-exec-col-idx='9'>Resources</div>" +
"<div id='resource_prof_id' class='resource-prof-id-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='' data-exec-col-idx='10'>Resource Profile Id</div>" +
"<div id='jvm_on_off_heap_memory' class='jvm_on_off_heap_memory-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='' data-exec-col-idx='7'>Peak JVM Memory OnHeap / OffHeap</div>" +
"<div id='on_off_heap_execution_memory' class='on_off_heap_execution_memory-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='' data-exec-col-idx='8'>Peak Execution Memory OnHeap / OffHeap</div>" +
"<div id='on_off_heap_storage_memory' class='on_off_heap_storage_memory'><input type='checkbox' class='toggle-vis' data-sum-col-idx='' data-exec-col-idx='9'>Peak Storage Memory OnHeap / OffHeap</div>" +
"<div id='direct_mapped_pool_memory' class='direct_mapped_pool_memory-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='' data-exec-col-idx='10'>Peak Pool Memory Direct / Mapped</div>" +
"<div id='extra_resources' class='resources-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='' data-exec-col-idx='13'>Resources</div>" +
"<div id='resource_prof_id' class='resource-prof-id-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='' data-exec-col-idx='14'>Resource Profile Id</div>" +
"</div>");

reselectCheckboxesBasedOnTaskTableState();
Expand Down

0 comments on commit 93ad26b

Please sign in to comment.