diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index c8fedc7ad329bd..c0bb051bd07832 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -207,10 +207,17 @@ def _create_parser(): group.add_argument('-m', '--match', metavar='PAT', dest='match_tests', action='append', help='match test cases and methods with glob pattern PAT') + group.add_argument('-i', '--ignore', metavar='PAT', + dest='ignore_tests', action='append', + help='ignore test cases and methods with glob pattern PAT') group.add_argument('--matchfile', metavar='FILENAME', dest='match_filename', help='similar to --match but get patterns from a ' 'text file, one pattern per line') + group.add_argument('--ignorefile', metavar='FILENAME', + dest='ignore_filename', + help='similar to --matchfile but it receives patterns ' + 'from text file to ignore') group.add_argument('-G', '--failfast', action='store_true', help='fail as soon as a test fails (only with -v or -W)') group.add_argument('-u', '--use', metavar='RES1,RES2,...', @@ -317,7 +324,8 @@ def _parse_args(args, **kwargs): findleaks=1, use_resources=None, trace=False, coverdir='coverage', runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None, pgo=False) + header=False, failfast=False, match_tests=None, ignore_tests=None, + pgo=False) for k, v in kwargs.items(): if not hasattr(ns, k): raise TypeError('%r is an invalid keyword argument ' @@ -395,6 +403,12 @@ def _parse_args(args, **kwargs): with open(ns.match_filename) as fp: for line in fp: ns.match_tests.append(line.strip()) + if ns.ignore_filename: + if ns.ignore_tests is None: + ns.ignore_tests = [] + with open(ns.ignore_filename) as fp: + for line in fp: + ns.ignore_tests.append(line.strip()) if ns.forever: # --forever implies --failfast ns.failfast = True diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 76ad3359f2d7ac..95b4856c8bed7d 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -287,7 +287,7 @@ def _list_cases(self, suite): def list_cases(self): support.verbose = False - support.set_match_tests(self.ns.match_tests) + support.set_match_tests(self.ns.match_tests, self.ns.ignore_tests) for test_name in self.selected: abstest = get_abs_module(self.ns, test_name) @@ -394,7 +394,10 @@ def run_tests_sequential(self): save_modules = sys.modules.keys() - self.log("Run tests sequentially") + msg = "Run tests sequentially" + if self.ns.timeout: + msg += " (timeout: %s)" % format_duration(self.ns.timeout) + self.log(msg) previous_test = None for test_index, test_name in enumerate(self.tests, 1): diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index eeb108bb447394..558f2099c66f5e 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -123,7 +123,7 @@ def _runtest(ns, test_name): start_time = time.perf_counter() try: - support.set_match_tests(ns.match_tests) + support.set_match_tests(ns.match_tests, ns.ignore_tests) support.junit_xml_list = xml_list = [] if ns.xmlpath else None if ns.failfast: support.failfast = True diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 71db8d69668599..7a18e45434bb46 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -3,6 +3,7 @@ import json import os import queue +import signal import subprocess import sys import threading @@ -31,6 +32,8 @@ # Time to wait until a worker completes: should be immediate JOIN_TIMEOUT = 30.0 # seconds +USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) + def must_stop(result, ns): if result.result == INTERRUPTED: @@ -59,12 +62,16 @@ def run_test_in_subprocess(testname, ns): # Running the child from the same working directory as regrtest's original # invocation ensures that TEMPDIR for the child is the same when # sysconfig.is_python_build() is true. See issue 15300. + kw = {} + if USE_PROCESS_GROUP: + kw['start_new_session'] = True return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, close_fds=(os.name != 'nt'), - cwd=support.SAVEDCWD) + cwd=support.SAVEDCWD, + **kw) def run_tests_worker(ns, test_name): @@ -149,11 +156,24 @@ def _kill(self): return self._killed = True - print(f"Kill {self}", file=sys.stderr, flush=True) + if USE_PROCESS_GROUP: + what = f"{self} process group" + else: + what = f"{self}" + + print(f"Kill {what}", file=sys.stderr, flush=True) try: - popen.kill() + if USE_PROCESS_GROUP: + os.killpg(popen.pid, signal.SIGKILL) + else: + popen.kill() + except ProcessLookupError: + # popen.kill(): the process completed, the TestWorkerProcess thread + # read its exit status, but Popen.send_signal() read the returncode + # just before Popen.wait() set returncode. + pass except OSError as exc: - print_warning(f"Failed to kill {self}: {exc!r}") + print_warning(f"Failed to kill {what}: {exc!r}") def stop(self): # Method called from a different thread to stop this thread @@ -332,7 +352,11 @@ def __init__(self, regrtest): self.output = queue.Queue() self.pending = MultiprocessIterator(self.regrtest.tests) if self.ns.timeout is not None: - self.worker_timeout = self.ns.timeout * 1.5 + # Rely on faulthandler to kill a worker process. This timouet is + # when faulthandler fails to kill a worker process. Give a maximum + # of 5 minutes to faulthandler to kill the worker. + self.worker_timeout = min(self.ns.timeout * 1.5, + self.ns.timeout + 5 * 60) else: self.worker_timeout = None self.workers = None @@ -340,8 +364,12 @@ def __init__(self, regrtest): def start_workers(self): self.workers = [TestWorkerProcess(index, self) for index in range(1, self.ns.use_mp + 1)] - self.log("Run tests in parallel using %s child processes" - % len(self.workers)) + msg = f"Run tests in parallel using {len(self.workers)} child processes" + if self.ns.timeout: + msg += (" (timeout: %s, worker timeout: %s)" + % (format_duration(self.ns.timeout), + format_duration(self.worker_timeout))) + self.log(msg) for worker in self.workers: worker.start() diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index d6a7819cb01398..a42c6f771b3d30 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2047,7 +2047,9 @@ def _run_suite(suite): # By default, don't filter tests _match_test_func = None -_match_test_patterns = None + +_accept_test_patterns = None +_ignore_test_patterns = None def match_test(test): @@ -2063,18 +2065,45 @@ def _is_full_match_test(pattern): # as a full test identifier. # Example: 'test.test_os.FileTests.test_access'. # - # Reject patterns which contain fnmatch patterns: '*', '?', '[...]' - # or '[!...]'. For example, reject 'test_access*'. + # ignore patterns which contain fnmatch patterns: '*', '?', '[...]' + # or '[!...]'. For example, ignore 'test_access*'. return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) -def set_match_tests(patterns): - global _match_test_func, _match_test_patterns +def set_match_tests(accept_patterns=None, ignore_patterns=None): + global _match_test_func, _accept_test_patterns, _ignore_test_patterns - if patterns == _match_test_patterns: - # No change: no need to recompile patterns. - return + if accept_patterns is None: + accept_patterns = () + if ignore_patterns is None: + ignore_patterns = () + + accept_func = ignore_func = None + + if accept_patterns != _accept_test_patterns: + accept_patterns, accept_func = _compile_match_function(accept_patterns) + if ignore_patterns != _ignore_test_patterns: + ignore_patterns, ignore_func = _compile_match_function(ignore_patterns) + + # Create a copy since patterns can be mutable and so modified later + _accept_test_patterns = tuple(accept_patterns) + _ignore_test_patterns = tuple(ignore_patterns) + + if accept_func is not None or ignore_func is not None: + def match_function(test_id): + accept = True + ignore = False + if accept_func: + accept = accept_func(test_id) + if ignore_func: + ignore = ignore_func(test_id) + return accept and not ignore + + _match_test_func = match_function + + +def _compile_match_function(patterns): if not patterns: func = None # set_match_tests(None) behaves as set_match_tests(()) @@ -2102,10 +2131,7 @@ def match_test_regex(test_id): func = match_test_regex - # Create a copy since patterns can be mutable and so modified later - _match_test_patterns = tuple(patterns) - _match_test_func = func - + return patterns, func def run_unittest(*classes): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index d2221b3448fa32..e2fa1977ba6c7f 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -157,6 +157,24 @@ def test_single(self): self.assertTrue(ns.single) self.checkError([opt, '-f', 'foo'], "don't go together") + def test_ignore(self): + for opt in '-i', '--ignore': + with self.subTest(opt=opt): + ns = libregrtest._parse_args([opt, 'pattern']) + self.assertEqual(ns.ignore_tests, ['pattern']) + self.checkError([opt], 'expected one argument') + + self.addCleanup(support.unlink, support.TESTFN) + with open(support.TESTFN, "w") as fp: + print('matchfile1', file=fp) + print('matchfile2', file=fp) + + filename = os.path.abspath(support.TESTFN) + ns = libregrtest._parse_args(['-m', 'match', + '--ignorefile', filename]) + self.assertEqual(ns.ignore_tests, + ['matchfile1', 'matchfile2']) + def test_match(self): for opt in '-m', '--match': with self.subTest(opt=opt): @@ -960,6 +978,42 @@ def parse_methods(self, output): regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE) return [match.group(1) for match in regex.finditer(output)] + def test_ignorefile(self): + code = textwrap.dedent(""" + import unittest + + class Tests(unittest.TestCase): + def test_method1(self): + pass + def test_method2(self): + pass + def test_method3(self): + pass + def test_method4(self): + pass + """) + all_methods = ['test_method1', 'test_method2', + 'test_method3', 'test_method4'] + testname = self.create_test(code=code) + + # only run a subset + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + + subset = [ + # only ignore the method name + 'test_method1', + # ignore the full identifier + '%s.Tests.test_method3' % testname] + with open(filename, "w") as fp: + for name in subset: + print(name, file=fp) + + output = self.run_tests("-v", "--ignorefile", filename, testname) + methods = self.parse_methods(output) + subset = ['test_method2', 'test_method4'] + self.assertEqual(methods, subset) + def test_matchfile(self): code = textwrap.dedent(""" import unittest diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 8f0746aed8299a..e3ce670b8437f9 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -527,6 +527,7 @@ def id(self): test_access = Test('test.test_os.FileTests.test_access') test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') + # Test acceptance with support.swap_attr(support, '_match_test_func', None): # match all support.set_match_tests([]) @@ -534,45 +535,92 @@ def id(self): self.assertTrue(support.match_test(test_chdir)) # match all using None - support.set_match_tests(None) + support.set_match_tests(None, None) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # match the full test identifier - support.set_match_tests([test_access.id()]) + support.set_match_tests([test_access.id()], None) self.assertTrue(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) # match the module name - support.set_match_tests(['test_os']) + support.set_match_tests(['test_os'], None) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # Test '*' pattern - support.set_match_tests(['test_*']) + support.set_match_tests(['test_*'], None) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # Test case sensitivity - support.set_match_tests(['filetests']) + support.set_match_tests(['filetests'], None) self.assertFalse(support.match_test(test_access)) - support.set_match_tests(['FileTests']) + support.set_match_tests(['FileTests'], None) self.assertTrue(support.match_test(test_access)) # Test pattern containing '.' and a '*' metacharacter - support.set_match_tests(['*test_os.*.test_*']) + support.set_match_tests(['*test_os.*.test_*'], None) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # Multiple patterns - support.set_match_tests([test_access.id(), test_chdir.id()]) + support.set_match_tests([test_access.id(), test_chdir.id()], None) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) - support.set_match_tests(['test_access', 'DONTMATCH']) + support.set_match_tests(['test_access', 'DONTMATCH'], None) self.assertTrue(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) + # Test rejection + with support.swap_attr(support, '_match_test_func', None): + # match all + support.set_match_tests(ignore_patterns=[]) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # match all using None + support.set_match_tests(None, None) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # match the full test identifier + support.set_match_tests(None, [test_access.id()]) + self.assertFalse(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # match the module name + support.set_match_tests(None, ['test_os']) + self.assertFalse(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + # Test '*' pattern + support.set_match_tests(None, ['test_*']) + self.assertFalse(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + # Test case sensitivity + support.set_match_tests(None, ['filetests']) + self.assertTrue(support.match_test(test_access)) + support.set_match_tests(None, ['FileTests']) + self.assertFalse(support.match_test(test_access)) + + # Test pattern containing '.' and a '*' metacharacter + support.set_match_tests(None, ['*test_os.*.test_*']) + self.assertFalse(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + # Multiple patterns + support.set_match_tests(None, [test_access.id(), test_chdir.id()]) + self.assertFalse(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + support.set_match_tests(None, ['test_access', 'DONTMATCH']) + self.assertFalse(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + def test_fd_count(self): # We cannot test the absolute value of fd_count(): on old Linux # kernel or glibc versions, os.urandom() keeps a FD open on diff --git a/Misc/NEWS.d/next/Tests/2019-10-17-00-49-38.bpo-38502.vUEic7.rst b/Misc/NEWS.d/next/Tests/2019-10-17-00-49-38.bpo-38502.vUEic7.rst new file mode 100644 index 00000000000000..1df523e90c3881 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2019-10-17-00-49-38.bpo-38502.vUEic7.rst @@ -0,0 +1,3 @@ +test.regrtest now uses process groups in the multiprocessing mode (-jN command +line option) if process groups are available: if :func:`os.setsid` and +:func:`os.killpg` functions are available. diff --git a/Misc/NEWS.d/next/Tests/2019-10-30-00-01-43.bpo-37957.X1r78F.rst b/Misc/NEWS.d/next/Tests/2019-10-30-00-01-43.bpo-37957.X1r78F.rst new file mode 100644 index 00000000000000..75e186ef33e071 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2019-10-30-00-01-43.bpo-37957.X1r78F.rst @@ -0,0 +1,3 @@ +test.regrtest now can receive a list of test patterns to ignore (using the +-i/--ignore argument) or a file with a list of patterns to ignore (using the +--ignore-file argument). Patch by Pablo Galindo.