diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py index 3427b51b60af86..5e8dba5dbde71a 100644 --- a/Lib/test/libregrtest/__init__.py +++ b/Lib/test/libregrtest/__init__.py @@ -1,5 +1,2 @@ -# We import importlib *ASAP* in order to test #15386 -import importlib - from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES, ALL_RESOURCES from test.libregrtest.main import main diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 9f1bf6800824a6..6e24a8681b27e7 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -207,10 +207,17 @@ def _create_parser(): group.add_argument('-m', '--match', metavar='PAT', dest='match_tests', action='append', help='match test cases and methods with glob pattern PAT') + group.add_argument('-i', '--ignore', metavar='PAT', + dest='ignore_tests', action='append', + help='ignore test cases and methods with glob pattern PAT') group.add_argument('--matchfile', metavar='FILENAME', dest='match_filename', help='similar to --match but get patterns from a ' 'text file, one pattern per line') + group.add_argument('--ignorefile', metavar='FILENAME', + dest='ignore_filename', + help='similar to --matchfile but it receives patterns ' + 'from text file to ignore') group.add_argument('-G', '--failfast', action='store_true', help='fail as soon as a test fails (only with -v or -W)') group.add_argument('-u', '--use', metavar='RES1,RES2,...', @@ -315,7 +322,8 @@ def _parse_args(args, **kwargs): findleaks=1, use_resources=None, trace=False, coverdir='coverage', runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None, pgo=False) + header=False, failfast=False, match_tests=None, ignore_tests=None, + pgo=False) for k, v in kwargs.items(): if not hasattr(ns, k): raise TypeError('%r is an invalid keyword argument ' @@ -391,6 +399,12 @@ def _parse_args(args, **kwargs): with open(ns.match_filename) as fp: for line in fp: ns.match_tests.append(line.strip()) + if ns.ignore_filename: + if ns.ignore_tests is None: + ns.ignore_tests = [] + with open(ns.ignore_filename) as fp: + for line in fp: + ns.ignore_tests.append(line.strip()) if ns.forever: # --forever implies --failfast ns.failfast = True diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 05878a8af4c325..c521d530ce1611 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -282,7 +282,7 @@ def _list_cases(self, suite): def list_cases(self): support.verbose = False - support.set_match_tests(self.ns.match_tests) + support.set_match_tests(self.ns.match_tests, self.ns.ignore_tests) for test_name in self.selected: abstest = get_abs_module(self.ns, test_name) @@ -389,7 +389,10 @@ def run_tests_sequential(self): save_modules = sys.modules.keys() - self.log("Run tests sequentially") + msg = "Run tests sequentially" + if self.ns.timeout: + msg += " (timeout: %s)" % format_duration(self.ns.timeout) + self.log(msg) previous_test = None for test_index, test_name in enumerate(self.tests, 1): diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index e7dce180cb3753..558f2099c66f5e 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -123,7 +123,7 @@ def _runtest(ns, test_name): start_time = time.perf_counter() try: - support.set_match_tests(ns.match_tests) + support.set_match_tests(ns.match_tests, ns.ignore_tests) support.junit_xml_list = xml_list = [] if ns.xmlpath else None if ns.failfast: support.failfast = True @@ -313,9 +313,7 @@ def cleanup_test_droppings(test_name, verbose): # since if a test leaves a file open, it cannot be deleted by name (while # there's nothing we can do about that here either, we can display the # name of the offending test, which is a real help). - for name in (support.TESTFN, - "db_home", - ): + for name in (support.TESTFN,): if not os.path.exists(name): continue diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 2770cf93bd7d8d..7a18e45434bb46 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -3,6 +3,7 @@ import json import os import queue +import signal import subprocess import sys import threading @@ -31,6 +32,8 @@ # Time to wait until a worker completes: should be immediate JOIN_TIMEOUT = 30.0 # seconds +USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) + def must_stop(result, ns): if result.result == INTERRUPTED: @@ -59,12 +62,16 @@ def run_test_in_subprocess(testname, ns): # Running the child from the same working directory as regrtest's original # invocation ensures that TEMPDIR for the child is the same when # sysconfig.is_python_build() is true. See issue 15300. + kw = {} + if USE_PROCESS_GROUP: + kw['start_new_session'] = True return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, close_fds=(os.name != 'nt'), - cwd=support.SAVEDCWD) + cwd=support.SAVEDCWD, + **kw) def run_tests_worker(ns, test_name): @@ -127,32 +134,46 @@ def __init__(self, worker_id, runner): def __repr__(self): info = [f'TestWorkerProcess #{self.worker_id}'] if self.is_alive(): - dt = time.monotonic() - self.start_time - info.append("running for %s" % format_duration(dt)) + info.append("running") else: info.append('stopped') test = self.current_test_name if test: info.append(f'test={test}') popen = self._popen - if popen: - info.append(f'pid={popen.pid}') + if popen is not None: + dt = time.monotonic() - self.start_time + info.extend((f'pid={self._popen.pid}', + f'time={format_duration(dt)}')) return '<%s>' % ' '.join(info) def _kill(self): + popen = self._popen + if popen is None: + return + if self._killed: return self._killed = True - popen = self._popen - if popen is None: - return + if USE_PROCESS_GROUP: + what = f"{self} process group" + else: + what = f"{self}" - print(f"Kill {self}", file=sys.stderr, flush=True) + print(f"Kill {what}", file=sys.stderr, flush=True) try: - popen.kill() + if USE_PROCESS_GROUP: + os.killpg(popen.pid, signal.SIGKILL) + else: + popen.kill() + except ProcessLookupError: + # popen.kill(): the process completed, the TestWorkerProcess thread + # read its exit status, but Popen.send_signal() read the returncode + # just before Popen.wait() set returncode. + pass except OSError as exc: - print_warning(f"Failed to kill {self}: {exc!r}") + print_warning(f"Failed to kill {what}: {exc!r}") def stop(self): # Method called from a different thread to stop this thread @@ -170,9 +191,10 @@ def _run_process(self, test_name): self.current_test_name = test_name try: + popen = run_test_in_subprocess(test_name, self.ns) + self._killed = False - self._popen = run_test_in_subprocess(test_name, self.ns) - popen = self._popen + self._popen = popen except: self.current_test_name = None raise @@ -330,7 +352,11 @@ def __init__(self, regrtest): self.output = queue.Queue() self.pending = MultiprocessIterator(self.regrtest.tests) if self.ns.timeout is not None: - self.worker_timeout = self.ns.timeout * 1.5 + # Rely on faulthandler to kill a worker process. This timouet is + # when faulthandler fails to kill a worker process. Give a maximum + # of 5 minutes to faulthandler to kill the worker. + self.worker_timeout = min(self.ns.timeout * 1.5, + self.ns.timeout + 5 * 60) else: self.worker_timeout = None self.workers = None @@ -338,8 +364,12 @@ def __init__(self, regrtest): def start_workers(self): self.workers = [TestWorkerProcess(index, self) for index in range(1, self.ns.use_mp + 1)] - self.log("Run tests in parallel using %s child processes" - % len(self.workers)) + msg = f"Run tests in parallel using {len(self.workers)} child processes" + if self.ns.timeout: + msg += (" (timeout: %s, worker timeout: %s)" + % (format_duration(self.ns.timeout), + format_duration(self.worker_timeout))) + self.log(msg) for worker in self.workers: worker.start() diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 4362e92fbda90a..ea7f2c2f185c31 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -67,29 +67,34 @@ def setup_tests(ns): if ns.threshold is not None: gc.set_threshold(ns.threshold) + suppress_msvcrt_asserts(ns.verbose and ns.verbose >= 2) + + support.use_resources = ns.use_resources + + +def suppress_msvcrt_asserts(verbose): try: import msvcrt except ImportError: - pass - else: - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass + return + + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + return + + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + if verbose: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - if ns.verbose and ns.verbose >= 2: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - else: - msvcrt.CrtSetReportMode(m, 0) + msvcrt.CrtSetReportMode(m, 0) - support.use_resources = ns.use_resources def replace_stdout(): diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index d04d19985f7a05..b198c2ca763911 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1936,7 +1936,9 @@ def _run_suite(suite): # By default, don't filter tests _match_test_func = None -_match_test_patterns = None + +_accept_test_patterns = None +_ignore_test_patterns = None def match_test(test): @@ -1952,18 +1954,45 @@ def _is_full_match_test(pattern): # as a full test identifier. # Example: 'test.test_os.FileTests.test_access'. # - # Reject patterns which contain fnmatch patterns: '*', '?', '[...]' - # or '[!...]'. For example, reject 'test_access*'. + # ignore patterns which contain fnmatch patterns: '*', '?', '[...]' + # or '[!...]'. For example, ignore 'test_access*'. return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) -def set_match_tests(patterns): - global _match_test_func, _match_test_patterns +def set_match_tests(accept_patterns=None, ignore_patterns=None): + global _match_test_func, _accept_test_patterns, _ignore_test_patterns - if patterns == _match_test_patterns: - # No change: no need to recompile patterns. - return + if accept_patterns is None: + accept_patterns = () + if ignore_patterns is None: + ignore_patterns = () + + accept_func = ignore_func = None + + if accept_patterns != _accept_test_patterns: + accept_patterns, accept_func = _compile_match_function(accept_patterns) + if ignore_patterns != _ignore_test_patterns: + ignore_patterns, ignore_func = _compile_match_function(ignore_patterns) + + # Create a copy since patterns can be mutable and so modified later + _accept_test_patterns = tuple(accept_patterns) + _ignore_test_patterns = tuple(ignore_patterns) + + if accept_func is not None or ignore_func is not None: + def match_function(test_id): + accept = True + ignore = False + if accept_func: + accept = accept_func(test_id) + if ignore_func: + ignore = ignore_func(test_id) + return accept and not ignore + + _match_test_func = match_function + + +def _compile_match_function(patterns): if not patterns: func = None # set_match_tests(None) behaves as set_match_tests(()) @@ -1991,10 +2020,7 @@ def match_test_regex(test_id): func = match_test_regex - # Create a copy since patterns can be mutable and so modified later - _match_test_patterns = tuple(patterns) - _match_test_func = func - + return patterns, func def run_unittest(*classes): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 26c9b16293c2be..a5ac8102008c13 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -157,6 +157,24 @@ def test_single(self): self.assertTrue(ns.single) self.checkError([opt, '-f', 'foo'], "don't go together") + def test_ignore(self): + for opt in '-i', '--ignore': + with self.subTest(opt=opt): + ns = libregrtest._parse_args([opt, 'pattern']) + self.assertEqual(ns.ignore_tests, ['pattern']) + self.checkError([opt], 'expected one argument') + + self.addCleanup(support.unlink, support.TESTFN) + with open(support.TESTFN, "w") as fp: + print('matchfile1', file=fp) + print('matchfile2', file=fp) + + filename = os.path.abspath(support.TESTFN) + ns = libregrtest._parse_args(['-m', 'match', + '--ignorefile', filename]) + self.assertEqual(ns.ignore_tests, + ['matchfile1', 'matchfile2']) + def test_match(self): for opt in '-m', '--match': with self.subTest(opt=opt): @@ -931,6 +949,42 @@ def parse_methods(self, output): regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE) return [match.group(1) for match in regex.finditer(output)] + def test_ignorefile(self): + code = textwrap.dedent(""" + import unittest + + class Tests(unittest.TestCase): + def test_method1(self): + pass + def test_method2(self): + pass + def test_method3(self): + pass + def test_method4(self): + pass + """) + all_methods = ['test_method1', 'test_method2', + 'test_method3', 'test_method4'] + testname = self.create_test(code=code) + + # only run a subset + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + + subset = [ + # only ignore the method name + 'test_method1', + # ignore the full identifier + '%s.Tests.test_method3' % testname] + with open(filename, "w") as fp: + for name in subset: + print(name, file=fp) + + output = self.run_tests("-v", "--ignorefile", filename, testname) + methods = self.parse_methods(output) + subset = ['test_method2', 'test_method4'] + self.assertEqual(methods, subset) + def test_matchfile(self): code = textwrap.dedent(""" import unittest diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index e29846efe55406..7c1ddb87f19222 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -529,6 +529,7 @@ def id(self): test_access = Test('test.test_os.FileTests.test_access') test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') + # Test acceptance with support.swap_attr(support, '_match_test_func', None): # match all support.set_match_tests([]) @@ -536,45 +537,92 @@ def id(self): self.assertTrue(support.match_test(test_chdir)) # match all using None - support.set_match_tests(None) + support.set_match_tests(None, None) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # match the full test identifier - support.set_match_tests([test_access.id()]) + support.set_match_tests([test_access.id()], None) self.assertTrue(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) # match the module name - support.set_match_tests(['test_os']) + support.set_match_tests(['test_os'], None) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # Test '*' pattern - support.set_match_tests(['test_*']) + support.set_match_tests(['test_*'], None) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # Test case sensitivity - support.set_match_tests(['filetests']) + support.set_match_tests(['filetests'], None) self.assertFalse(support.match_test(test_access)) - support.set_match_tests(['FileTests']) + support.set_match_tests(['FileTests'], None) self.assertTrue(support.match_test(test_access)) # Test pattern containing '.' and a '*' metacharacter - support.set_match_tests(['*test_os.*.test_*']) + support.set_match_tests(['*test_os.*.test_*'], None) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # Multiple patterns - support.set_match_tests([test_access.id(), test_chdir.id()]) + support.set_match_tests([test_access.id(), test_chdir.id()], None) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) - support.set_match_tests(['test_access', 'DONTMATCH']) + support.set_match_tests(['test_access', 'DONTMATCH'], None) self.assertTrue(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) + # Test rejection + with support.swap_attr(support, '_match_test_func', None): + # match all + support.set_match_tests(ignore_patterns=[]) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # match all using None + support.set_match_tests(None, None) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # match the full test identifier + support.set_match_tests(None, [test_access.id()]) + self.assertFalse(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # match the module name + support.set_match_tests(None, ['test_os']) + self.assertFalse(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + # Test '*' pattern + support.set_match_tests(None, ['test_*']) + self.assertFalse(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + # Test case sensitivity + support.set_match_tests(None, ['filetests']) + self.assertTrue(support.match_test(test_access)) + support.set_match_tests(None, ['FileTests']) + self.assertFalse(support.match_test(test_access)) + + # Test pattern containing '.' and a '*' metacharacter + support.set_match_tests(None, ['*test_os.*.test_*']) + self.assertFalse(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + # Multiple patterns + support.set_match_tests(None, [test_access.id(), test_chdir.id()]) + self.assertFalse(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + support.set_match_tests(None, ['test_access', 'DONTMATCH']) + self.assertFalse(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + def test_fd_count(self): # We cannot test the absolute value of fd_count(): on old Linux # kernel or glibc versions, os.urandom() keeps a FD open on diff --git a/Misc/NEWS.d/next/Tests/2019-10-17-00-49-38.bpo-38502.vUEic7.rst b/Misc/NEWS.d/next/Tests/2019-10-17-00-49-38.bpo-38502.vUEic7.rst new file mode 100644 index 00000000000000..1df523e90c3881 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2019-10-17-00-49-38.bpo-38502.vUEic7.rst @@ -0,0 +1,3 @@ +test.regrtest now uses process groups in the multiprocessing mode (-jN command +line option) if process groups are available: if :func:`os.setsid` and +:func:`os.killpg` functions are available. diff --git a/Misc/NEWS.d/next/Tests/2019-10-30-00-01-43.bpo-37957.X1r78F.rst b/Misc/NEWS.d/next/Tests/2019-10-30-00-01-43.bpo-37957.X1r78F.rst new file mode 100644 index 00000000000000..75e186ef33e071 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2019-10-30-00-01-43.bpo-37957.X1r78F.rst @@ -0,0 +1,3 @@ +test.regrtest now can receive a list of test patterns to ignore (using the +-i/--ignore argument) or a file with a list of patterns to ignore (using the +--ignore-file argument). Patch by Pablo Galindo.