BUG: change default timer to timeit.default_timer to fix resolution on Windows

CHANGES.rst

@@ -6,6 +6,9 @@ New Features
 
 API Changes
 ^^^^^^^^^^^
+- Default timer changed from ``process_time()`` to
+  ``timeit.default_timer()`` to fix resolution issues on Windows.
+  Old behavior can be restored by setting ``Benchmark.timer = time.process_time``
 
 Bug Fixes
 ^^^^^^^^^

asv/benchmark.py

@@ -152,6 +152,8 @@ def process_time():
         # Fallback to default timer
         process_time = timeit.default_timer
 
+wall_timer = timeit.default_timer
+
 
 def get_maxrss():
     # Fallback function, in case we don't have one that works on the
@@ -487,7 +489,7 @@ def _load_vars(self):
         self.number = int(_get_first_attr(self._attr_sources, 'number', 0))
         self.sample_time = _get_first_attr(self._attr_sources, 'sample_time', 0.01)
         self.warmup_time = _get_first_attr(self._attr_sources, 'warmup_time', -1)
-        self.timer = _get_first_attr(self._attr_sources, 'timer', process_time)
+        self.timer = _get_first_attr(self._attr_sources, 'timer', wall_timer)
 
     def do_setup(self):
         result = Benchmark.do_setup(self)
@@ -547,7 +549,7 @@ def benchmark_timing(self, timer, min_repeat, max_repeat, max_time, warmup_time,
                          number, min_run_count):
 
         sample_time = self.sample_time
-        start_time = time.time()
+        start_time = wall_timer()
         run_count = 0
 
         samples = []
@@ -558,7 +560,7 @@ def too_slow(num_samples):
                 return False
             if num_samples < min_repeat:
                 return False
-            return time.time() > start_time + warmup_time + max_time
+            return wall_timer() > start_time + warmup_time + max_time
 
         if number == 0:
             # Select number & warmup.
@@ -569,14 +571,14 @@ def too_slow(num_samples):
             number = 1
             while True:
                 self._redo_setup_next = False
-                start = time.time()
+                start = wall_timer()
                 timing = timer.timeit(number)
-                wall_time = time.time() - start
+                wall_time = wall_timer() - start
                 actual_timing = max(wall_time, timing)
                 run_count += number
 
                 if actual_timing >= sample_time:
-                    if time.time() > start_time + warmup_time:
+                    if wall_timer() > start_time + warmup_time:
                         break
                 else:
                     try:
@@ -593,7 +595,7 @@ def too_slow(num_samples):
                 self._redo_setup_next = False
                 timing = timer.timeit(number)
                 run_count += number
-                if time.time() >= start_time + warmup_time:
+                if wall_timer() >= start_time + warmup_time:
                     break
 
             if too_slow(1):
@@ -1089,14 +1091,14 @@ def main_run_server(args):
 
             # Wait for results
             # (Poll in a loop is simplest --- also used by subprocess.py)
-            start_time = time.time()
+            start_time = wall_timer()
             is_timeout = False
             while True:
                 res, status = os.waitpid(pid, os.WNOHANG)
                 if res != 0:
                     break
 
-                if timeout is not None and time.time() > start_time + timeout:
+                if timeout is not None and wall_timer() > start_time + timeout:
                     # Timeout
                     if is_timeout:
                         os.kill(pid, signal.SIGKILL)
@@ -1142,7 +1144,7 @@ def main_timing(argv):
     parser.add_argument("--number", action="store", type=int, default=0)
     parser.add_argument("--repeat", action="store", type=int, default=0)
     parser.add_argument("--timer", action="store", choices=("process_time", "perf_counter"),
-                        default="process_time")
+                        default="perf_counter")
     parser.add_argument("--json", action="store_true")
     parser.add_argument("statement")
     args = parser.parse_args(argv)

docs/source/benchmarks.rst

@@ -138,20 +138,17 @@ Timing benchmarks
 - ``timer``: The timing function to use, which can be any source of
   monotonically increasing numbers, such as `time.clock`, `time.time`
   or ``time.process_time``.  If it's not provided, it defaults to
-  ``time.process_time`` (or a backported version of it for versions of
-  Python prior to 3.3), but other useful values are
-  `timeit.default_timer` to use the default ``timeit`` behavior on
-  your version of Python.
-
-  On Windows, `time.clock` has microsecond granularity, but
-  `time.time`'s granularity is 1/60th of a second. On Unix,
-  `time.clock` has 1/100th of a second granularity, and `time.time` is
-  much more precise. On either platform, `timeit.default_timer`
-  measures wall clock time, not the CPU time. This means that other
-  processes running on the same computer may interfere with the
-  timing.  That's why the default of ``time.process_time``, which only
-  measures the time used by the current process, is often the best
-  choice.
+  ``timeit.default_timer``, but other useful values are
+  ``process_time``, for which ``asv`` provides a backported version for
+  versions of Python prior to 3.3.
+
+  .. versionchanged:: 0.4
+
+     Previously, the default timer measured process time, which was chosen
+     to minimize noise from other processes. However, on Windows, this is
+     only available at a resolution of 15.6ms, which is greater than the
+     recommended benchmark runtime of 10ms. Therefore, we default to the
+     highest resolution clock on any platform.
 
 The ``sample_time``, ``number``, ``repeat``, and ``timer`` attributes
 can be adjusted in the ``setup()`` routine, which can be useful for

docs/source/writing_benchmarks.rst

@@ -266,11 +266,17 @@ stolen from IPython's `%timeit
 magic function.  This means that in most cases the benchmark function
 itself will be run many times to achieve accurate timing.
 
-The default timing function is `time.process_time` (POSIX
+The default timing function is `timeit.default_timer`, which uses the
+highest resolution clock available on a given platform to measure the
+elapsed wall time. This has the consequence of being more susceptible
+to noise from other processes, but the increase in resolution is more
+significant for shorter duration tests (particularly on Windows).
+
+Process timing is provided by the function `time.process_time` (POSIX
 ``CLOCK_PROCESS_CPUTIME``), which measures the CPU time used only by
 the current process.  You can change the timer by setting the
-benchmark's ``timer`` attribute, for example to `timeit.default_timer`
-to measure wall clock time.
+benchmark's ``timer`` attribute, for example to `time.process_time`
+to measure process time.
 
 .. note::