Skip to content

Commit

Permalink
bpo-37957: Allow regrtest to receive a file with test (and subtests) …
Browse files Browse the repository at this point in the history
…to ignore (GH-16989)

When building Python in some uncommon platforms there are some known tests that will fail. Right now, the test suite has the ability to ignore entire tests using the -x option and to receive a filter file using the --matchfile filter. The problem with the --matchfile option is that it receives a file with patterns to accept and when you want to ignore a couple of tests and subtests, is too cumbersome to lists ALL tests that are not the ones that you want to accept and he problem with -x is that is not easy to ignore just a subtests that fail and the whole test needs to be ignored.

For these reasons, add a new option to allow to ignore a list of test and subtests for these situations.
  • Loading branch information
pablogsal authored Nov 19, 2019
1 parent ef5aa9a commit e0cd8aa
Show file tree
Hide file tree
Showing 7 changed files with 169 additions and 24 deletions.
16 changes: 15 additions & 1 deletion Lib/test/libregrtest/cmdline.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,10 +207,17 @@ def _create_parser():
group.add_argument('-m', '--match', metavar='PAT',
dest='match_tests', action='append',
help='match test cases and methods with glob pattern PAT')
group.add_argument('-i', '--ignore', metavar='PAT',
dest='ignore_tests', action='append',
help='ignore test cases and methods with glob pattern PAT')
group.add_argument('--matchfile', metavar='FILENAME',
dest='match_filename',
help='similar to --match but get patterns from a '
'text file, one pattern per line')
group.add_argument('--ignorefile', metavar='FILENAME',
dest='ignore_filename',
help='similar to --matchfile but it receives patterns '
'from text file to ignore')
group.add_argument('-G', '--failfast', action='store_true',
help='fail as soon as a test fails (only with -v or -W)')
group.add_argument('-u', '--use', metavar='RES1,RES2,...',
Expand Down Expand Up @@ -317,7 +324,8 @@ def _parse_args(args, **kwargs):
findleaks=1, use_resources=None, trace=False, coverdir='coverage',
runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
random_seed=None, use_mp=None, verbose3=False, forever=False,
header=False, failfast=False, match_tests=None, pgo=False)
header=False, failfast=False, match_tests=None, ignore_tests=None,
pgo=False)
for k, v in kwargs.items():
if not hasattr(ns, k):
raise TypeError('%r is an invalid keyword argument '
Expand Down Expand Up @@ -395,6 +403,12 @@ def _parse_args(args, **kwargs):
with open(ns.match_filename) as fp:
for line in fp:
ns.match_tests.append(line.strip())
if ns.ignore_filename:
if ns.ignore_tests is None:
ns.ignore_tests = []
with open(ns.ignore_filename) as fp:
for line in fp:
ns.ignore_tests.append(line.strip())
if ns.forever:
# --forever implies --failfast
ns.failfast = True
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/libregrtest/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ def _list_cases(self, suite):

def list_cases(self):
support.verbose = False
support.set_match_tests(self.ns.match_tests)
support.set_match_tests(self.ns.match_tests, self.ns.ignore_tests)

for test_name in self.selected:
abstest = get_abs_module(self.ns, test_name)
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/libregrtest/runtest.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ def _runtest(ns, test_name):

start_time = time.perf_counter()
try:
support.set_match_tests(ns.match_tests)
support.set_match_tests(ns.match_tests, ns.ignore_tests)
support.junit_xml_list = xml_list = [] if ns.xmlpath else None
if ns.failfast:
support.failfast = True
Expand Down
50 changes: 38 additions & 12 deletions Lib/test/support/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2096,7 +2096,9 @@ def _run_suite(suite):

# By default, don't filter tests
_match_test_func = None
_match_test_patterns = None

_accept_test_patterns = None
_ignore_test_patterns = None


def match_test(test):
Expand All @@ -2112,18 +2114,45 @@ def _is_full_match_test(pattern):
# as a full test identifier.
# Example: 'test.test_os.FileTests.test_access'.
#
# Reject patterns which contain fnmatch patterns: '*', '?', '[...]'
# or '[!...]'. For example, reject 'test_access*'.
# ignore patterns which contain fnmatch patterns: '*', '?', '[...]'
# or '[!...]'. For example, ignore 'test_access*'.
return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern))


def set_match_tests(patterns):
global _match_test_func, _match_test_patterns
def set_match_tests(accept_patterns=None, ignore_patterns=None):
global _match_test_func, _accept_test_patterns, _ignore_test_patterns

if patterns == _match_test_patterns:
# No change: no need to recompile patterns.
return

if accept_patterns is None:
accept_patterns = ()
if ignore_patterns is None:
ignore_patterns = ()

accept_func = ignore_func = None

if accept_patterns != _accept_test_patterns:
accept_patterns, accept_func = _compile_match_function(accept_patterns)
if ignore_patterns != _ignore_test_patterns:
ignore_patterns, ignore_func = _compile_match_function(ignore_patterns)

# Create a copy since patterns can be mutable and so modified later
_accept_test_patterns = tuple(accept_patterns)
_ignore_test_patterns = tuple(ignore_patterns)

if accept_func is not None or ignore_func is not None:
def match_function(test_id):
accept = True
ignore = False
if accept_func:
accept = accept_func(test_id)
if ignore_func:
ignore = ignore_func(test_id)
return accept and not ignore

_match_test_func = match_function


def _compile_match_function(patterns):
if not patterns:
func = None
# set_match_tests(None) behaves as set_match_tests(())
Expand Down Expand Up @@ -2151,10 +2180,7 @@ def match_test_regex(test_id):

func = match_test_regex

# Create a copy since patterns can be mutable and so modified later
_match_test_patterns = tuple(patterns)
_match_test_func = func

return patterns, func


def run_unittest(*classes):
Expand Down
54 changes: 54 additions & 0 deletions Lib/test/test_regrtest.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,24 @@ def test_single(self):
self.assertTrue(ns.single)
self.checkError([opt, '-f', 'foo'], "don't go together")

def test_ignore(self):
for opt in '-i', '--ignore':
with self.subTest(opt=opt):
ns = libregrtest._parse_args([opt, 'pattern'])
self.assertEqual(ns.ignore_tests, ['pattern'])
self.checkError([opt], 'expected one argument')

self.addCleanup(support.unlink, support.TESTFN)
with open(support.TESTFN, "w") as fp:
print('matchfile1', file=fp)
print('matchfile2', file=fp)

filename = os.path.abspath(support.TESTFN)
ns = libregrtest._parse_args(['-m', 'match',
'--ignorefile', filename])
self.assertEqual(ns.ignore_tests,
['matchfile1', 'matchfile2'])

def test_match(self):
for opt in '-m', '--match':
with self.subTest(opt=opt):
Expand Down Expand Up @@ -961,6 +979,42 @@ def parse_methods(self, output):
regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE)
return [match.group(1) for match in regex.finditer(output)]

def test_ignorefile(self):
code = textwrap.dedent("""
import unittest
class Tests(unittest.TestCase):
def test_method1(self):
pass
def test_method2(self):
pass
def test_method3(self):
pass
def test_method4(self):
pass
""")
all_methods = ['test_method1', 'test_method2',
'test_method3', 'test_method4']
testname = self.create_test(code=code)

# only run a subset
filename = support.TESTFN
self.addCleanup(support.unlink, filename)

subset = [
# only ignore the method name
'test_method1',
# ignore the full identifier
'%s.Tests.test_method3' % testname]
with open(filename, "w") as fp:
for name in subset:
print(name, file=fp)

output = self.run_tests("-v", "--ignorefile", filename, testname)
methods = self.parse_methods(output)
subset = ['test_method2', 'test_method4']
self.assertEqual(methods, subset)

def test_matchfile(self):
code = textwrap.dedent("""
import unittest
Expand Down
66 changes: 57 additions & 9 deletions Lib/test/test_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -527,52 +527,100 @@ def id(self):
test_access = Test('test.test_os.FileTests.test_access')
test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir')

# Test acceptance
with support.swap_attr(support, '_match_test_func', None):
# match all
support.set_match_tests([])
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))

# match all using None
support.set_match_tests(None)
support.set_match_tests(None, None)
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))

# match the full test identifier
support.set_match_tests([test_access.id()])
support.set_match_tests([test_access.id()], None)
self.assertTrue(support.match_test(test_access))
self.assertFalse(support.match_test(test_chdir))

# match the module name
support.set_match_tests(['test_os'])
support.set_match_tests(['test_os'], None)
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))

# Test '*' pattern
support.set_match_tests(['test_*'])
support.set_match_tests(['test_*'], None)
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))

# Test case sensitivity
support.set_match_tests(['filetests'])
support.set_match_tests(['filetests'], None)
self.assertFalse(support.match_test(test_access))
support.set_match_tests(['FileTests'])
support.set_match_tests(['FileTests'], None)
self.assertTrue(support.match_test(test_access))

# Test pattern containing '.' and a '*' metacharacter
support.set_match_tests(['*test_os.*.test_*'])
support.set_match_tests(['*test_os.*.test_*'], None)
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))

# Multiple patterns
support.set_match_tests([test_access.id(), test_chdir.id()])
support.set_match_tests([test_access.id(), test_chdir.id()], None)
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))

support.set_match_tests(['test_access', 'DONTMATCH'])
support.set_match_tests(['test_access', 'DONTMATCH'], None)
self.assertTrue(support.match_test(test_access))
self.assertFalse(support.match_test(test_chdir))

# Test rejection
with support.swap_attr(support, '_match_test_func', None):
# match all
support.set_match_tests(ignore_patterns=[])
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))

# match all using None
support.set_match_tests(None, None)
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))

# match the full test identifier
support.set_match_tests(None, [test_access.id()])
self.assertFalse(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))

# match the module name
support.set_match_tests(None, ['test_os'])
self.assertFalse(support.match_test(test_access))
self.assertFalse(support.match_test(test_chdir))

# Test '*' pattern
support.set_match_tests(None, ['test_*'])
self.assertFalse(support.match_test(test_access))
self.assertFalse(support.match_test(test_chdir))

# Test case sensitivity
support.set_match_tests(None, ['filetests'])
self.assertTrue(support.match_test(test_access))
support.set_match_tests(None, ['FileTests'])
self.assertFalse(support.match_test(test_access))

# Test pattern containing '.' and a '*' metacharacter
support.set_match_tests(None, ['*test_os.*.test_*'])
self.assertFalse(support.match_test(test_access))
self.assertFalse(support.match_test(test_chdir))

# Multiple patterns
support.set_match_tests(None, [test_access.id(), test_chdir.id()])
self.assertFalse(support.match_test(test_access))
self.assertFalse(support.match_test(test_chdir))

support.set_match_tests(None, ['test_access', 'DONTMATCH'])
self.assertFalse(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))

def test_fd_count(self):
# We cannot test the absolute value of fd_count(): on old Linux
# kernel or glibc versions, os.urandom() keeps a FD open on
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
test.regrtest now can receive a list of test patterns to ignore (using the
-i/--ignore argument) or a file with a list of patterns to ignore (using the
--ignore-file argument). Patch by Pablo Galindo.

0 comments on commit e0cd8aa

Please sign in to comment.