Multi-threaded unit testing by pytest (#483)

* implement x ** y gradients #468 #469

* enable pytest-xdist multi-threaded testing (#481)

* enable pytest-xdist multi-threaded testing (#481)

* _merged_ #479 by hand

* auto-compelete file name for #479

* test_verbose also support #479

* Skip non-essential builds on AppVeyor #485

* try fix appveyor.yml

* mistake in appveyor_filter.py

* really fix appveyor_filter.py

* try use `appveyor exit` & add msg in filter

* fix typo in appveyor_filter.py

* cache build in appveyor

* add test_files argument to test_cpp; better no test_cpp in test if argv > 2 specified

* fix typo in python/taichi/main.py

* add typo in main.py porpusely to figure out why travis fail because -n 2

* try again

* auto-detect pytest xdist

* test_files: None -> ()

* add pytest-xdist to ci_setup.py

* remove appveyor.yml modifications

Co-authored-by: Yuanming Hu <yuanming-hu@users.noreply.github.com>

ci_setup.py

@@ -167,7 +167,7 @@ def run(self):
 
     subprocess.run([get_python_executable(), "-m", "pip", "install", "--user",
                     "colorama", "numpy", "Pillow", "scipy", "pybind11",
-                    "GitPython", "yapf", "distro", "pytest", "autograd", "astor"])
+                    "GitPython", "yapf", "distro", "pytest", "autograd", "astor", "pytest-xdist"])
     print("importing numpy test:")
     ret = subprocess.run([get_python_executable(), "-c", "import numpy as np"])
     print("ret:", ret)

misc/empty_pytest.py

@@ -0,0 +1,2 @@
+def test_notest():
+  pass

python/taichi/main.py

@@ -6,7 +6,7 @@
 from taichi.tools.video import make_video, interpolate_frames, mp4_to_gif, scale_video, crop_video, accelerate_video
 
 
-def test_python(test_files=None, verbose=False):
+def test_python(test_files=(), verbose=False):
   print("\nRunning python tests...\n")
   import taichi as ti
   import pytest
@@ -17,26 +17,39 @@ def test_python(test_files=None, verbose=False):
     test_dir = ti.get_repo_directory()
   test_dir = os.path.join(test_dir, 'tests', 'python')
   args = []
-  if test_files:
+  if len(test_files):
     # run individual tests
     for f in test_files:
+      # auto-compelete
+      if not f.startswith('test_'):
+        f = 'test_' + f
+      if not f.endswith('.py'):
+        f = f + '.py'
       args.append(os.path.join(test_dir, f))
   else:
     # run all the tests
     args = [test_dir]
   if verbose:
     args += ['-s']
-
+  if len(test_files) == 0 or len(test_files) > 4:
+    if int(pytest.main(['misc/empty_pytest.py', '-n1'])) == 0: # if pytest has xdist
+      try:
+        from multiprocessing import cpu_count
+        cpu_count = cpu_count()
+      except:
+        cpu_count = 4
+      print(f'Starting {cpu_count} testing thread(s)...')
+      args += ['-n', str(cpu_count)]
   return int(pytest.main(args))
 
 
-def test_cpp():
+def test_cpp(test_files=()):
   import taichi as ti
   # Cpp tests use the legacy non LLVM backend
   ti.reset()
   print("Running C++ tests...")
   task = ti.Task('test')
-  return int(task.run(*sys.argv[2:]))
+  return int(task.run(*test_files))
 
 
 def main(debug=False):
@@ -102,13 +115,14 @@ def main(debug=False):
   elif mode == "test_python":
     return test_python(test_files=sys.argv[2:])
   elif mode == "test_cpp":
-    return test_cpp()
+    return test_cpp(test_files=sys.argv[2:])
   elif mode == "test":
-    if test_python() != 0:
+    if test_python(test_files=sys.argv[2:]) != 0:
       return -1
-    return test_cpp()
+    if len(sys.argv) <= 2:
+      return test_cpp()
   elif mode == "test_verbose":
-    if test_python(verbose=True) != 0:
+    if test_python(test_files=sys.argv[2:], verbose=True) != 0:
       return -1
     return test_cpp()
   elif mode == "build":