From c5ee784a9975572973c666ff2ff48bb524bbf4c9 Mon Sep 17 00:00:00 2001 From: Christopher Whelan Date: Sun, 16 Dec 2018 20:52:45 -0800 Subject: [PATCH] BUG: change default timer to timeit.default_timer to fix resolution issues on Windows --- CHANGES.rst | 3 +++ asv/benchmark.py | 22 ++++++++++++---------- docs/source/benchmarks.rst | 25 +++++++++++-------------- docs/source/writing_benchmarks.rst | 12 +++++++++--- 4 files changed, 35 insertions(+), 27 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 79fdd1149..aaf134c70 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,9 @@ New Features API Changes ^^^^^^^^^^^ +- Default timer changed from ``process_time()`` to + ``timeit.default_timer()`` to fix resolution issues on Windows. + Old behavior can be restored by setting ``Benchmark.timer = time.process_time`` Bug Fixes ^^^^^^^^^ diff --git a/asv/benchmark.py b/asv/benchmark.py index 01dc78595..62cce5193 100644 --- a/asv/benchmark.py +++ b/asv/benchmark.py @@ -152,6 +152,8 @@ def process_time(): # Fallback to default timer process_time = timeit.default_timer +wall_timer = timeit.default_timer + def get_maxrss(): # Fallback function, in case we don't have one that works on the @@ -487,7 +489,7 @@ def _load_vars(self): self.number = int(_get_first_attr(self._attr_sources, 'number', 0)) self.sample_time = _get_first_attr(self._attr_sources, 'sample_time', 0.01) self.warmup_time = _get_first_attr(self._attr_sources, 'warmup_time', -1) - self.timer = _get_first_attr(self._attr_sources, 'timer', process_time) + self.timer = _get_first_attr(self._attr_sources, 'timer', wall_timer) def do_setup(self): result = Benchmark.do_setup(self) @@ -547,7 +549,7 @@ def benchmark_timing(self, timer, min_repeat, max_repeat, max_time, warmup_time, number, min_run_count): sample_time = self.sample_time - start_time = time.time() + start_time = wall_timer() run_count = 0 samples = [] @@ -558,7 +560,7 @@ def too_slow(num_samples): return False if num_samples < min_repeat: return False - return time.time() > start_time + warmup_time + max_time + return wall_timer() > start_time + warmup_time + max_time if number == 0: # Select number & warmup. @@ -569,14 +571,14 @@ def too_slow(num_samples): number = 1 while True: self._redo_setup_next = False - start = time.time() + start = wall_timer() timing = timer.timeit(number) - wall_time = time.time() - start + wall_time = wall_timer() - start actual_timing = max(wall_time, timing) run_count += number if actual_timing >= sample_time: - if time.time() > start_time + warmup_time: + if wall_timer() > start_time + warmup_time: break else: try: @@ -593,7 +595,7 @@ def too_slow(num_samples): self._redo_setup_next = False timing = timer.timeit(number) run_count += number - if time.time() >= start_time + warmup_time: + if wall_timer() >= start_time + warmup_time: break if too_slow(1): @@ -1089,14 +1091,14 @@ def main_run_server(args): # Wait for results # (Poll in a loop is simplest --- also used by subprocess.py) - start_time = time.time() + start_time = wall_timer() is_timeout = False while True: res, status = os.waitpid(pid, os.WNOHANG) if res != 0: break - if timeout is not None and time.time() > start_time + timeout: + if timeout is not None and wall_timer() > start_time + timeout: # Timeout if is_timeout: os.kill(pid, signal.SIGKILL) @@ -1142,7 +1144,7 @@ def main_timing(argv): parser.add_argument("--number", action="store", type=int, default=0) parser.add_argument("--repeat", action="store", type=int, default=0) parser.add_argument("--timer", action="store", choices=("process_time", "perf_counter"), - default="process_time") + default="perf_counter") parser.add_argument("--json", action="store_true") parser.add_argument("statement") args = parser.parse_args(argv) diff --git a/docs/source/benchmarks.rst b/docs/source/benchmarks.rst index bbb89ea7a..441b7bbdb 100644 --- a/docs/source/benchmarks.rst +++ b/docs/source/benchmarks.rst @@ -138,20 +138,17 @@ Timing benchmarks - ``timer``: The timing function to use, which can be any source of monotonically increasing numbers, such as `time.clock`, `time.time` or ``time.process_time``. If it's not provided, it defaults to - ``time.process_time`` (or a backported version of it for versions of - Python prior to 3.3), but other useful values are - `timeit.default_timer` to use the default ``timeit`` behavior on - your version of Python. - - On Windows, `time.clock` has microsecond granularity, but - `time.time`'s granularity is 1/60th of a second. On Unix, - `time.clock` has 1/100th of a second granularity, and `time.time` is - much more precise. On either platform, `timeit.default_timer` - measures wall clock time, not the CPU time. This means that other - processes running on the same computer may interfere with the - timing. That's why the default of ``time.process_time``, which only - measures the time used by the current process, is often the best - choice. + ``timeit.default_timer``, but other useful values are + ``process_time``, for which ``asv`` provides a backported version for + versions of Python prior to 3.3. + + .. versionchanged:: 0.4 + + Previously, the default timer measured process time, which was chosen + to minimize noise from other processes. However, on Windows, this is + only available at a resolution of 15.6ms, which is greater than the + recommended benchmark runtime of 10ms. Therefore, we default to the + highest resolution clock on any platform. The ``sample_time``, ``number``, ``repeat``, and ``timer`` attributes can be adjusted in the ``setup()`` routine, which can be useful for diff --git a/docs/source/writing_benchmarks.rst b/docs/source/writing_benchmarks.rst index 26de36571..f5335ce8a 100644 --- a/docs/source/writing_benchmarks.rst +++ b/docs/source/writing_benchmarks.rst @@ -266,11 +266,17 @@ stolen from IPython's `%timeit magic function. This means that in most cases the benchmark function itself will be run many times to achieve accurate timing. -The default timing function is `time.process_time` (POSIX +The default timing function is `timeit.default_timer`, which uses the +highest resolution clock available on a given platform to measure the +elapsed wall time. This has the consequence of being more susceptible +to noise from other processes, but the increase in resolution is more +significant for shorter duration tests (particularly on Windows). + +Process timing is provided by the function `time.process_time` (POSIX ``CLOCK_PROCESS_CPUTIME``), which measures the CPU time used only by the current process. You can change the timer by setting the -benchmark's ``timer`` attribute, for example to `timeit.default_timer` -to measure wall clock time. +benchmark's ``timer`` attribute, for example to `time.process_time` +to measure process time. .. note::