Merge pull request #117 from sybrenjansen/dashboard-progressbar-impro…

…vements

Dashboard progressbar improvements

docs/changelog.rst

@@ -1,6 +1,18 @@
 Changelog
 =========
 
+Unreleased
+----------
+
+* Use function details from the `__call__` method on the dashboard in case the
+  callable being executed is a class instance (`#117`_)
+* Use (global) average rate for the estimate on the dashboard when smoothing=0
+  (`#117`_)
+* Make it possible to reuse the same `progress_bar_options` without raising warnings (`#117`_)
+
+.. _#117: https://github.com/sybrenjansen/mpire/pull/117
+
+
 2.9.0
 -----
 

docs/conf.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # mpire documentation build configuration file, created by
 # sphinx-quickstart on Tue Jun  27 15:35:20 2017.

mpire/async_result.py

@@ -9,7 +9,7 @@
 job_counter = itertools.count()
 
 
-class AsyncResult(object):
+class AsyncResult:
 
     """ Adapted from ``multiprocessing.pool.ApplyResult``. """
 

mpire/dashboard/dashboard.py

@@ -79,7 +79,7 @@ def progress_bar_new() -> str:
     # Obtain progress bar details. Only show the user@host part if it doesn't equal the user@host of this process
     # (in case someone connected to this dashboard from another machine or user)
     progress_bar_details = _DASHBOARD_TQDM_DETAILS_DICT.get(pb_id)
-    if progress_bar_details['user'] == '{}@{}'.format(getpass.getuser(), socket.gethostname()):
+    if progress_bar_details['user'] == f'{getpass.getuser()}@{socket.gethostname()}':
         progress_bar_details['user'] = ''
     else:
         progress_bar_details['user'] = '{}:'.format(progress_bar_details['user'])

mpire/dashboard/utils.py

@@ -3,6 +3,7 @@
 import socket
 from functools import partial
 from typing import Callable, Dict, List, Sequence, Tuple, Union
+import types
 
 
 def get_two_available_ports(port_range: Sequence) -> Tuple[int, int]:
@@ -13,6 +14,7 @@ def get_two_available_ports(port_range: Sequence) -> Tuple[int, int]:
     :raises OSError: If there are not enough ports available
     :return: Two available ports
     """
+
     def _port_available(port_nr: int) -> bool:
         """
         Checks if a port is available
@@ -27,23 +29,23 @@ def _port_available(port_nr: int) -> bool:
             return True
         except OSError:
             return False
-    
+
     available_ports = set()
     for port_nr in port_range:
         if _port_available(port_nr):
             available_ports.add(port_nr)
             break
-    
+
     for port_nr in reversed(port_range):
         if _port_available(port_nr):
             available_ports.add(port_nr)
             break
-    
+
     if len(available_ports) != 2:
         raise OSError(f"Dashboard Manager Server: there are not enough ports available: {port_range}")
-    
+
     return tuple(sorted(available_ports))
-     
+
 
 def get_function_details(func: Callable) -> Dict[str, Union[str, int]]:
     """
@@ -84,9 +86,12 @@ def get_function_details(func: Callable) -> Dict[str, Union[str, int]]:
     else:
         invoked_line_no = 'N/A'
 
-    # If we're dealing with a partial, obtain the function within
     if isinstance(func, partial):
+        # If we're dealing with a partial, obtain the function within
         func = func.func
+    elif hasattr(func, '__call__') and not isinstance(func, (type, types.FunctionType, types.MethodType)):
+        # If we're dealing with a callable class instance, use its __call__ method
+        func = func.__call__
 
     # We use a try/except block as some constructs don't allow this. E.g., in the case the function is a MagicMock
     # (i.e., in unit tests) these inspections will fail
@@ -100,7 +105,7 @@ def get_function_details(func: Callable) -> Dict[str, Union[str, int]]:
         function_name = 'n/a'
 
     # Populate details
-    func_details = {'user': '{}@{}'.format(getpass.getuser(), socket.gethostname()),
+    func_details = {'user': f'{getpass.getuser()}@{socket.gethostname()}',
                     'function_filename': function_filename,
                     'function_line_no': function_line_no,
                     'function_name': function_name,

mpire/params.py

@@ -293,19 +293,26 @@ def check_progress_bar_options(progress_bar_options: Optional[Dict[str, Any]], p
                           "precedence", RuntimeWarning, stacklevel=2)
         else:
             progress_bar_options["position"] = progress_bar_position
-            
+
     # We currently do not support the position parameter for rich progress bars. Although this can be implemented by
     # using a single rich progress bar for all workers and using `add_task`, but this is not trivial to implement.
     if progress_bar_style == "rich" and "position" in progress_bar_options:
         raise NotImplementedError("The 'position' parameter is currently not supported for rich progress bars")
 
     # Set some defaults and overwrite others
-    progress_bar_options["total"] = n_tasks
-    progress_bar_options["leave"] = True
-    progress_bar_options.setdefault("position", 0)
-    progress_bar_options.setdefault("dynamic_ncols", True)
-    progress_bar_options.setdefault("mininterval", 0.1)
-    progress_bar_options.setdefault("maxinterval", 0.5)
+    # NB We make a copy of the progress bar options, so that if the original dict is reused, redoing this check doesn't
+    # raise warnings due to the "total" and "leave" being added.
+    progress_bar_options = {
+        # defaults
+        "position": 0,
+        "dynamic_ncols": True,
+        "mininterval": 0.1,
+        "maxinterval": 0.5,
+        **progress_bar_options,
+        # overrides
+        "total": n_tasks,
+        "leave": True
+    }
 
     # Check if the tqdm progress bar style is valid
     tqdm = get_tqdm(progress_bar_style)

mpire/progress_bar.py

@@ -226,7 +226,8 @@ def _get_progress_bar_update_dict(self, progress_bar: tqdm_type, failed: bool,
         n = details["n"]
         total = details["total"]
         now = datetime.now()
-        remaining_time = (total - n) / details["rate"] if total and details["rate"] else None
+        rate = details["rate"] if details["rate"] else n / details["elapsed"] if details["elapsed"] else None
+        remaining_time = (total - n) / rate if total and rate else None
 
         return {"id": self.progress_bar_id,
                 "success": not failed,

mpire/worker.py

@@ -713,4 +713,4 @@ def worker_factory(start_method: str, use_dill: bool) -> Type[Union[AbstractWork
         elif start_method == 'spawn':
             return SpawnWorker
         else:
-            raise ValueError("Unknown start method: '{}'".format(start_method))
+            raise ValueError(f"Unknown start method: '{start_method}'")

tests/test_params.py

@@ -185,13 +185,15 @@ def test_check_progress_bar_options(self):
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
 
+            defaults = {"position": 0, "dynamic_ncols": True, "mininterval": 0.1, "maxinterval": 0.5}
+            overrides = {"total": None, "leave": True}
+
             # Should work fine. We're ignoring warnings
             for progress_bar_options in [{}, {"position": 0}, {"desc": "hello", "total": 100},
                                          {"unit": "seconds", "mininterval": 0.1}]:
                 with self.subTest(progress_bar_options=progress_bar_options):
                     returned_progress_bar_options = check_progress_bar_options(progress_bar_options, None, None, None)
-                    self.assertEqual({k: None if k == "total" else returned_progress_bar_options[k]
-                                      for k in progress_bar_options.keys()}, progress_bar_options)
+                    self.assertEqual(returned_progress_bar_options, {**defaults, **progress_bar_options, **overrides})
 
             # progress_bar_options should be a dictionary
             for progress_bar_options in ['hello', {8}, 3.14]: