Skip to content

Commit

Permalink
Include total number of tasks in the performance report (#3822)
Browse files Browse the repository at this point in the history
* Include total number of tasks in the performance report

* Include tasks' timings in the performance report
  • Loading branch information
abduhbm authored May 26, 2020
1 parent 342b6a6 commit 0041cc1
Showing 1 changed file with 21 additions and 5 deletions.
26 changes: 21 additions & 5 deletions distributed/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1069,7 +1069,7 @@ def __init__(
preload=None,
preload_argv=(),
plugins=(),
**kwargs
**kwargs,
):
self._setup_logging(logger)

Expand Down Expand Up @@ -1352,7 +1352,7 @@ def __init__(
connection_limit=connection_limit,
deserialize=False,
connection_args=self.connection_args,
**kwargs
**kwargs,
)

if self.worker_ttl:
Expand Down Expand Up @@ -2089,7 +2089,7 @@ def stimulus_task_erred(
exception=exception,
traceback=traceback,
worker=worker,
**kwargs
**kwargs,
)
else:
recommendations = {}
Expand Down Expand Up @@ -3389,7 +3389,7 @@ async def retire_workers(
close_workers=False,
names=None,
lock=True,
**kwargs
**kwargs,
):
""" Gracefully retire workers from cluster
Expand Down Expand Up @@ -4090,7 +4090,7 @@ def transition_processing_memory(
typename=None,
worker=None,
startstops=None,
**kwargs
**kwargs,
):
try:
ts = self.tasks[key]
Expand Down Expand Up @@ -5055,6 +5055,15 @@ def profile_to_figure(state):

# Task stream
task_stream = self.get_task_stream(start=start)
total_tasks = len(task_stream)
timespent = defaultdict(int)
for d in task_stream:
for x in d["startstops"]:
timespent[x["action"]] += x["stop"] - x["start"]
tasks_timings = ""
for k in sorted(timespent.keys()):
tasks_timings += f"\n<li> {k} time: {format_time(timespent[k])} </li>"

from .diagnostics.task_stream import rectangles
from .dashboard.components.scheduler import task_stream_figure

Expand All @@ -5081,6 +5090,11 @@ def profile_to_figure(state):
<i> Select different tabs on the top for additional information </i>
<h2> Duration: {time} </h2>
<h2> Tasks Information </h2>
<ul>
<li> number of tasks: {ntasks} </li>
{tasks_timings}
</ul>
<h2> Scheduler Information </h2>
<ul>
Expand All @@ -5096,6 +5110,8 @@ def profile_to_figure(state):
</pre>
""".format(
time=format_time(stop - start),
ntasks=total_tasks,
tasks_timings=tasks_timings,
address=self.address,
nworkers=len(self.workers),
threads=sum(w.nthreads for w in self.workers.values()),
Expand Down

0 comments on commit 0041cc1

Please sign in to comment.