Merge pull request airspeed-velocity#637 from pv/ci-parallel

Speed up test suite

Run tests in parallel with pytest-xdist.
Fix timing issues that caused problems with pytest-xdist.
Use --bench= in tests to avoid running the whole slow benchmark suite.
Prefer close+join when using multiprocessing, to avoid some issues on python2 on windows.
Fix test_web:basic_html fixture to work with pytest-xdist.

Add --offline option for running tests without pip/conda downloading.
Pre-download necessary packages before running tests, which also mitigates some issues with conda's parallel execution safety.

.continuous-integration/download_reqs.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+"""
+download_reqs.py [OPTIONS]
+
+Pre-download required packages to cache for pip and/or conda.
+"""
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+
+import sys
+import argparse
+import subprocess
+import shlex
+import time
+
+
+PY_VERSIONS = [sys.version_info]
+if sys.version_info < (3,):
+    PY_VERSIONS.append((3, 6))
+else:
+    PY_VERSIONS.append((2, 7))
+
+
+def main():
+    p = argparse.ArgumentParser(usage=__doc__.strip())
+    p.add_argument('--pip', action='store', default=None,
+                   help="download files for offline pip cache")
+    p.add_argument('--conda', action='store_true',
+                   help=("download files to conda cache.\n"
+                         "NOTE: modifies current conda environment!"))
+    args = p.parse_args()
+
+    start_time = time.time()
+
+    if args.pip:
+        do_pip(args.pip)
+    if args.conda:
+        do_conda()
+
+    end_time = time.time()
+    print("Downloading took {} sec".format(end_time - start_time))
+
+
+def do_conda():
+    subs = dict(index_cache='', ver=sys.version_info)
+
+    call("""
+    conda create --download-only -n tmp --channel conda-forge -q --yes python={ver[0]}.{ver[1]}
+    """, subs)
+
+    for pyver in PY_VERSIONS:
+        subs['ver'] = pyver
+        call("""
+        conda create --download-only -n tmp -q --yes --use-index-cache python={ver[0]}.{ver[1]} wheel pip six=1.10 colorama=0.3.7
+        conda create --download-only -n tmp -q --yes --use-index-cache python={ver[0]}.{ver[1]} wheel pip six colorama=0.3.9
+        """, subs)
+
+
+def do_pip(cache_dir):
+    subs = dict(cache_dir=cache_dir)
+    for pyver in PY_VERSIONS:
+        if pyver == sys.version_info:
+            python = sys.executable
+        else:
+            python = 'python{ver[0]}.{ver[1]}'.format(ver=pyver)
+        if has_python(python):
+            subs['python'] = python
+            call("""
+            {python} -mpip download -d {cache_dir} six==1.10 colorama==0.3.7
+            {python} -mpip download -d {cache_dir} six colorama==0.3.9
+            """, subs)
+
+
+def has_python(cmd):
+    try:
+        ret = subprocess.call([cmd, '--version'])
+        return (ret == 0)
+    except OSError:
+        return False
+
+
+def call(cmds, subs=None):
+    if subs is None:
+        subs = {}
+    cmds = cmds.splitlines()
+    for line in cmds:
+        line = line.strip()
+        if not line:
+            continue
+        parts = [x.format(**subs) for x in shlex.split(line)]
+        parts = [x for x in parts if x]
+        print("$ {}".format(" ".join(parts)))
+        sys.stdout.flush()
+        subprocess.check_call(parts)
+
+
+if __name__ == "__main__":
+    main()

.gitignore

@@ -29,7 +29,9 @@ develop-eggs
 distribute-*.tar.gz
 
 # Other
+.asv
 .cache
+.pytest_cache
 .tox
 .*.swp
 *~

.travis.yml

@@ -67,15 +67,22 @@ install:
       echo -e '\ntravis_retry "$HOME/miniconda3/bin/conda.real" "$@"' >> $HOME/miniconda3/bin/conda;
       chmod +x $HOME/miniconda3/bin/conda;
       if $TRAVIS_PYTHON -c 'import virtualenv'; then echo "ERROR: virtualenv package is installed"; exit 1; fi;
+      $TRAVIS_PYTHON .continuous-integration/download_reqs.py --conda --pip=$HOME/download/pip-cache
     else
-      $TRAVIS_PIP install virtualenv;
+      if $TRAVIS_PYTHON -c 'import sys; sys.exit(0 if "__pypy__" in sys.modules else 1)'; then
+        $TRAVIS_PIP install virtualenv;
+      else
+        $TRAVIS_PIP install virtualenv numpy scipy;
+      fi
+      $TRAVIS_PYTHON .continuous-integration/download_reqs.py --pip=$HOME/download/pip-cache
     fi
-    $TRAVIS_PIP install selenium six pytest pytest-timeout feedparser python-hglib;
+    $TRAVIS_PIP install selenium six "pytest>=3.5" pytest-xdist feedparser python-hglib;
     if [[ "$COVERAGE" != '' ]]; then $TRAVIS_PIP install pytest-cov codecov; fi;
   - $TRAVIS_PYTHON setup.py build_ext -i
+  - export PIP_FIND_LINKS=file://$HOME/download/pip-cache
 
 script:
-  - $TRAVIS_PYTHON -m pytest --timeout=360 -l $COVERAGE test -vv --webdriver=PhantomJS
+  - $TRAVIS_PYTHON -m pytest -l $COVERAGE -vv --webdriver=PhantomJS -n 3 --offline test
 
 after_script:
   - if [[ "$COVERAGE" != '' ]]; then

appveyor.yml

@@ -10,9 +10,6 @@ environment:
 
   matrix:
 
-      - PYTHON_VERSION: "2.7"
-        platform: x64
-        PYTHON_ARCH: "64"
       - PYTHON_VERSION: "3.6"
         platform: x64
         PYTHON_ARCH: "64"
@@ -33,31 +30,29 @@ install:
 
     # Install the build and runtime dependencies of the project.
     - "conda update -q --yes conda"
+    - "conda install -q --yes python=%PYTHON_VERSION% conda six pip pytest pytest-xdist lockfile"
+
+    # Pre-download all necessary packages
+    - "python .continuous-integration/download_reqs.py --conda --pip=pip-cache"
+    - "set PIP_FIND_LINKS=file:///projects/asv/pip-cache"
 
     # Tell conda to not use hardlinks: on Windows it's not possible
     # to delete hard links to files in use, which causes problem when
     # trying to cleanup environments during the tests
     - "conda config --set always_copy True"
     - "conda config --set allow_softlinks False"
 
-    # Create a conda environment
-    - "conda create -q --yes -n test python=%PYTHON_VERSION%"
-    - "activate test"
-
     # Check that we have the expected version of Python
     - "python --version"
 
-    # Install specified version of dependencies
-    - "conda install -q --yes six pip pytest pytest-timeout numpy"
-
     # In-place build
     - "%CMD_IN_ENV% python setup.py build_ext -i"
 
 # Not a .NET project
 build: false
 
 test_script:
-    - "python -m pytest --timeout=360 -l --basetemp=%APPVEYOR_BUILD_FOLDER%\\tmp test -vv"
+    - "python -m pytest -l --basetemp=%APPVEYOR_BUILD_FOLDER%\\tmp -vv -n 3 --offline test"
 
 after_build:
     # Clear up pip cache

asv/benchmarks.py

@@ -408,18 +408,20 @@ def _disc_benchmarks(cls, conf, repo, environments, commit_hashes):
                 log.error(str(last_err))
                 raise util.UserError("Failed to build the project.")
 
-            result_file = tempfile.NamedTemporaryFile(delete=False)
+            result_dir = tempfile.mkdtemp()
             try:
-                result_file.close()
+                result_file = os.path.join(result_dir, 'result.json')
                 env.run(
                     [BENCHMARK_RUN_SCRIPT, 'discover',
-                     os.path.abspath(root), result_file.name],
+                     os.path.abspath(root),
+                     os.path.abspath(result_file)],
+                    cwd=result_dir,
                     dots=False)
 
-                with open(result_file.name, 'r') as fp:
+                with open(result_file, 'r') as fp:
                     benchmarks = json.load(fp)
             finally:
-                os.remove(result_file.name)
+                util.long_path_rmtree(result_dir)
 
         return benchmarks
 

asv/commands/publish.py

@@ -207,6 +207,8 @@ def run(cls, conf, range_spec=None, pull=True):
             pool = multiprocessing.Pool(n_processes)
             try:
                 graphs.detect_steps(pool, dots=log.dot)
+                pool.close()
+                pool.join()
             finally:
                 pool.terminate()
 

asv/commands/run.py

@@ -268,13 +268,16 @@ def run(cls, conf, range_spec=None, steps=None, bench=None, parallel=1,
                     with log.indent():
                         args = [(env, conf, repo, commit_hash) for env in subenv]
                         if parallel != 1:
-                            pool = multiprocessing.Pool(parallel)
                             try:
-                                successes = pool.map(_do_build_multiprocess, args)
+                                pool = multiprocessing.Pool(parallel)
+                                try:
+                                    successes = pool.map(_do_build_multiprocess, args)
+                                    pool.close()
+                                    pool.join()
+                                finally:
+                                    pool.terminate()
                             except util.ParallelFailure as exc:
                                 exc.reraise()
-                            finally:
-                                pool.close()
                         else:
                             successes = map(_do_build, args)
 

asv/commands/setup.py

@@ -67,12 +67,15 @@ def perform_setup(cls, environments, parallel=-1):
         log.info("Creating environments")
         with log.indent():
             if parallel != 1:
-                pool = multiprocessing.Pool(parallel)
                 try:
-                    pool.map(_create_parallel, environments)
+                    pool = multiprocessing.Pool(parallel)
+                    try:
+                        pool.map(_create_parallel, environments)
+                        pool.close()
+                        pool.join()
+                    finally:
+                        pool.terminate()
                 except util.ParallelFailure as exc:
                     exc.reraise()
-                finally:
-                    pool.close()
             else:
                 list(map(_create, environments))

asv/util.py

@@ -503,7 +503,7 @@ def watcher_run():
         is_timeout = was_timeout[0]
     else:
         try:
-            if posix:
+            if posix and is_main_thread():
                 # Forward signals related to Ctrl-Z handling; the child
                 # process is in a separate process group so it won't receive
                 # these automatically from the terminal
@@ -543,7 +543,7 @@ def sig_forward(signum, frame):
                         dots()
                     last_dot_time = time.time()
         finally:
-            if posix:
+            if posix and is_main_thread():
                 # Restore signal handlers
                 signal.signal(signal.SIGTSTP, signal.SIG_DFL)
                 signal.signal(signal.SIGCONT, signal.SIG_DFL)
@@ -618,6 +618,16 @@ def _killpg_safe(pgid, signo):
             raise
 
 
+def is_main_thread():
+    """
+    Return True if the current thread is the main thread.
+    """
+    if sys.version_info[0] >= 3:
+        return threading.current_thread() == threading.main_thread()
+    else:
+        return isinstance(threading.current_thread(), threading._MainThread)
+
+
 def write_json(path, data, api_version=None):
     """
     Writes JSON to the given path, including indentation and sorting.

test/benchmark/cache_examples.py

@@ -64,7 +64,7 @@ def track_fail(self):
 
 
 class ClassLevelCacheTimeoutSuccess:
-    timeout = 1.0
+    timeout = 2.0
 
     def setup_cache(self):
         time.sleep(2.0)

test/benchmark/time_secondary.py

@@ -25,13 +25,16 @@
 
 class TimeSecondary:
     sample_time = 0.05
+    _printed = False
 
     def time_factorial(self):
         x = 1
         for i in xrange(100):
             x *= i
-        # This is to generate invalid output
-        sys.stdout.write("X")
+        # This is to print things to stdout, but not spam too much
+        if not self._printed:
+            sys.stdout.write("X")
+            self._printed = True
 
     def time_exception(self):
         raise RuntimeError()

test/conftest.py

@@ -1,3 +1,5 @@
+import os
+
 
 def pytest_addoption(parser):
     parser.addoption("--webdriver", action="store", default="None",
@@ -6,3 +8,13 @@ def pytest_addoption(parser):
                            "FirefoxHeadless. Alternatively, it can be arbitrary Python code "
                            "with a return statement with selenium.webdriver object, for "
                            "example 'return Chrome()'"))
+
+    parser.addoption("--offline", action="store_true", default=False,
+                     help=("Do not download items from internet. Use if you have predownloaded "
+                           "packages and set PIP_FIND_LINKS."))
+
+
+def pytest_sessionstart(session):
+    if session.config.getoption('offline'):
+        os.environ['PIP_NO_INDEX'] = '1'
+        os.environ['CONDA_OFFLINE'] = 'True'

test/test_benchmarks.py

@@ -100,7 +100,7 @@ def test_find_benchmarks(tmpdir):
         'time_examples.TimeSuite.time_example_benchmark_1']['result'] != [None]
     assert isinstance(times['time_examples.TimeSuite.time_example_benchmark_1']['stats'][0]['std'], float)
     # The exact number of samples may vary if the calibration is not fully accurate
-    assert len(times['time_examples.TimeSuite.time_example_benchmark_1']['samples'][0]) in (8, 9, 10)
+    assert len(times['time_examples.TimeSuite.time_example_benchmark_1']['samples'][0]) >= 5
     # Benchmarks that raise exceptions should have a time of "None"
     assert times[
         'time_secondary.TimeSecondary.time_exception']['result'] == [None]

test/test_dev.py

@@ -68,8 +68,9 @@ def basic_conf_with_subdir(tmpdir):
 def test_dev(capsys, basic_conf):
     tmpdir, local, conf = basic_conf
 
-    # Test Dev runs
-    tools.run_asv_with_conf(conf, 'dev', _machine_file=join(tmpdir, 'asv-machine.json'))
+    # Test Dev runs (with full benchmark suite)
+    tools.run_asv_with_conf(conf, 'dev',
+                            _machine_file=join(tmpdir, 'asv-machine.json'))
     text, err = capsys.readouterr()
 
     # time_with_warnings failure case
@@ -88,7 +89,9 @@ def test_dev_with_repo_subdir(capsys, basic_conf_with_subdir):
     tmpdir, local, conf = basic_conf_with_subdir
 
     # Test Dev runs
-    tools.run_asv_with_conf(conf, 'dev', _machine_file=join(tmpdir, 'asv-machine.json'))
+    tools.run_asv_with_conf(conf, 'dev',
+                            '--bench=time_secondary.track_value',
+                            _machine_file=join(tmpdir, 'asv-machine.json'))
     text, err = capsys.readouterr()
 
     # Benchmarks were found and run
@@ -103,7 +106,10 @@ def test_run_python_same(capsys, basic_conf):
     tmpdir, local, conf = basic_conf
 
     # Test Run runs with python=same
-    tools.run_asv_with_conf(conf, 'run', '--python=same', _machine_file=join(tmpdir, 'asv-machine.json'))
+    tools.run_asv_with_conf(conf, 'run', '--python=same',
+                            '--bench=time_secondary.TimeSecondary.time_exception',
+                            '--bench=time_secondary.track_value',
+                            _machine_file=join(tmpdir, 'asv-machine.json'))
     text, err = capsys.readouterr()
 
     assert re.search("time_exception.*failed", text, re.S)