From bf2c10c8105d770354d0abc451de4ffd9b986562 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 17 Dec 2017 11:26:51 +0100 Subject: [PATCH 01/10] parameterset: refactor marking empty parametersets --- _pytest/mark.py | 15 ++++++++++----- _pytest/python.py | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/_pytest/mark.py b/_pytest/mark.py index 45182bdcd33..35cb3224af4 100644 --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -73,7 +73,7 @@ def extract_from(cls, parameterset, legacy_force_tuple=False): return cls(argval, marks=newmarks, id=None) @classmethod - def _for_parameterize(cls, argnames, argvalues, function): + def _for_parameterize(cls, argnames, argvalues, function, config): if not isinstance(argnames, (tuple, list)): argnames = [x.strip() for x in argnames.split(",") if x.strip()] force_tuple = len(argnames) == 1 @@ -85,10 +85,7 @@ def _for_parameterize(cls, argnames, argvalues, function): del argvalues if not parameters: - fs, lineno = getfslineno(function) - reason = "got empty parameter set %r, function %s at %s:%d" % ( - argnames, function.__name__, fs, lineno) - mark = MARK_GEN.skip(reason=reason) + mark = get_empty_parameterset_mark(config, argnames, function) parameters.append(ParameterSet( values=(NOTSET,) * len(argnames), marks=[mark], @@ -97,6 +94,14 @@ def _for_parameterize(cls, argnames, argvalues, function): return argnames, parameters +def get_empty_parameterset_mark(config, argnames, function): + + fs, lineno = getfslineno(function) + reason = "got empty parameter set %r, function %s at %s:%d" % ( + argnames, function.__name__, fs, lineno) + return MARK_GEN.skip(reason=reason) + + class MarkerError(Exception): """Error in use of a pytest marker/attribute.""" diff --git a/_pytest/python.py b/_pytest/python.py index 735489e484a..2a84677ec31 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -786,7 +786,7 @@ def parametrize(self, argnames, argvalues, indirect=False, ids=None, from _pytest.mark import ParameterSet from py.io import saferepr argnames, parameters = ParameterSet._for_parameterize( - argnames, argvalues, self.function) + argnames, argvalues, self.function, self.config) del argvalues if scope is None: From 37b41de779957b0e0e826016b7dee9ee2f4aec89 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 17 Dec 2017 12:49:12 +0100 Subject: [PATCH 02/10] fix #2527 - introduce a option to pic the empty parameterset action --- _pytest/mark.py | 21 +++++++++++++++++++-- changelog/2527.feature | 1 + testing/test_mark.py | 23 +++++++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 changelog/2527.feature diff --git a/_pytest/mark.py b/_pytest/mark.py index 35cb3224af4..24bb55838c8 100644 --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -95,11 +95,17 @@ def _for_parameterize(cls, argnames, argvalues, function, config): def get_empty_parameterset_mark(config, argnames, function): - + requested_mark = config.getini('empty_parameterset') + if requested_mark in ('', None, 'skip'): + mark = MARK_GEN.skip + elif requested_mark == 'xfail': + mark = MARK_GEN.xfail(run=False) + else: + raise LookupError(requested_mark) fs, lineno = getfslineno(function) reason = "got empty parameter set %r, function %s at %s:%d" % ( argnames, function.__name__, fs, lineno) - return MARK_GEN.skip(reason=reason) + return mark(reason=reason) class MarkerError(Exception): @@ -141,6 +147,9 @@ def pytest_addoption(parser): ) parser.addini("markers", "markers for test functions", 'linelist') + parser.addini( + "empty_parameterset", + "default marker for empty parametersets") def pytest_cmdline_main(config): @@ -284,6 +293,14 @@ def pytest_configure(config): if config.option.strict: MARK_GEN._config = config + empty_parameterset = config.getini("empty_parameterset") + + if empty_parameterset not in ('skip', 'xfail', None, ''): + from pytest import UsageError + raise UsageError( + "empty_parameterset must be one of skip and xfail," + " but it is {!r}".format(empty_parameterset)) + def pytest_unconfigure(config): MARK_GEN._config = getattr(config, '_old_mark_config', None) diff --git a/changelog/2527.feature b/changelog/2527.feature new file mode 100644 index 00000000000..97e2a63fb41 --- /dev/null +++ b/changelog/2527.feature @@ -0,0 +1 @@ +introduce a pytest ini option to pick the mark for empty parametersets and allow to use xfail(run=False) \ No newline at end of file diff --git a/testing/test_mark.py b/testing/test_mark.py index 45e88ae8f4b..1e601fc4426 100644 --- a/testing/test_mark.py +++ b/testing/test_mark.py @@ -891,3 +891,26 @@ class TestMarkDecorator(object): ]) def test__eq__(self, lhs, rhs, expected): assert (lhs == rhs) == expected + + +@pytest.mark.parametrize('mark', [None, 'skip', 'xfail']) +def test_parameterset_for_parametrize_marks(testdir, mark): + if mark is not None: + testdir.makeini("[pytest]\nempty_parameterset=" + mark) + + config = testdir.parseconfig() + from _pytest.mark import pytest_configure, get_empty_parameterset_mark + pytest_configure(config) + result_mark = get_empty_parameterset_mark(config, ['a'], all) + if mark is None: + # normalize to the requested name + mark = 'skip' + assert result_mark.name == mark + + if mark == 'xfail': + assert result_mark.kwargs.get('run') is False + + +def test_parameterset_for_parametrize_bad_markname(testdir): + with pytest.raises(pytest.UsageError): + test_parameterset_for_parametrize_marks(testdir, 'bad') From 7f83605c81205398976b658b868137b41156578a Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 17 Dec 2017 15:25:56 +0100 Subject: [PATCH 03/10] fix empty parameterset tests by mocking a config object --- testing/python/metafunc.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/testing/python/metafunc.py b/testing/python/metafunc.py index 06979681a03..99f99f6cda2 100644 --- a/testing/python/metafunc.py +++ b/testing/python/metafunc.py @@ -14,7 +14,7 @@ class TestMetafunc(object): - def Metafunc(self, func): + def Metafunc(self, func, config=None): # the unit tests of this class check if things work correctly # on the funcarg level, so we don't need a full blown # initiliazation @@ -26,7 +26,7 @@ def __init__(self, names): names = fixtures.getfuncargnames(func) fixtureinfo = FixtureInfo(names) - return python.Metafunc(func, fixtureinfo, None) + return python.Metafunc(func, fixtureinfo, config) def test_no_funcargs(self, testdir): def function(): @@ -156,7 +156,19 @@ def func(x, y): def test_parametrize_empty_list(self): def func(y): pass - metafunc = self.Metafunc(func) + + class MockConfig(object): + def getini(self, name): + return '' + + @property + def hook(self): + return self + + def pytest_make_parametrize_id(self, **kw): + pass + + metafunc = self.Metafunc(func, MockConfig()) metafunc.parametrize("y", []) assert 'skip' == metafunc._calls[0].marks[0].name From d4c11e58aa58fbb6d2e560bc62111a7efb955ca3 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 26 Jan 2018 11:18:50 +0100 Subject: [PATCH 04/10] exted empty parameterset check with reason test --- testing/test_mark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/test_mark.py b/testing/test_mark.py index 1e601fc4426..c85a30b41c0 100644 --- a/testing/test_mark.py +++ b/testing/test_mark.py @@ -906,7 +906,7 @@ def test_parameterset_for_parametrize_marks(testdir, mark): # normalize to the requested name mark = 'skip' assert result_mark.name == mark - + assert result_mark.kwargs['reason'].startswith("got empty parameter set ") if mark == 'xfail': assert result_mark.kwargs.get('run') is False From 8979b2a9d78161739fe870ef71be800067cff33a Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 26 Jan 2018 11:26:48 +0100 Subject: [PATCH 05/10] document empty_parameterset in customize.rst --- doc/en/customize.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/en/customize.rst b/doc/en/customize.rst index 6edeedf98fb..bbff9d18995 100644 --- a/doc/en/customize.rst +++ b/doc/en/customize.rst @@ -346,3 +346,21 @@ passed multiple times. The expected format is ``name=value``. For example:: # content of pytest.ini [pytest] console_output_style = classic + + +.. confval:: empty_parameterset + + .. versionadded:: 3.4 + + allows to pick the action for empty parametersets in parameterization + + * ``skip`` skips tests with a empty parameterset + * ``xfail`` marks tests with a empty parameterset as xfail(run=False) + + The default is ``skip``, it will be shifted to xfail in future. + + .. code-block:: ini + + # content of pytest.ini + [pytest] + empty_parameterset = xfail \ No newline at end of file From d550c33cd02512fd78a9a6b0e205dbbaf4f5e9d1 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 26 Jan 2018 11:56:24 +0100 Subject: [PATCH 06/10] s/empty_parameterset/empty_parameter_set_mark --- _pytest/mark.py | 4 ++-- doc/en/customize.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/_pytest/mark.py b/_pytest/mark.py index 24bb55838c8..c3352b387fe 100644 --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -95,7 +95,7 @@ def _for_parameterize(cls, argnames, argvalues, function, config): def get_empty_parameterset_mark(config, argnames, function): - requested_mark = config.getini('empty_parameterset') + requested_mark = config.getini('empty_parameter_set_mark') if requested_mark in ('', None, 'skip'): mark = MARK_GEN.skip elif requested_mark == 'xfail': @@ -148,7 +148,7 @@ def pytest_addoption(parser): parser.addini("markers", "markers for test functions", 'linelist') parser.addini( - "empty_parameterset", + "empty_parameter_set_mark", "default marker for empty parametersets") diff --git a/doc/en/customize.rst b/doc/en/customize.rst index bbff9d18995..eb79ba6682f 100644 --- a/doc/en/customize.rst +++ b/doc/en/customize.rst @@ -348,7 +348,7 @@ passed multiple times. The expected format is ``name=value``. For example:: console_output_style = classic -.. confval:: empty_parameterset +.. confval:: empty_parameter_set_mark .. versionadded:: 3.4 @@ -363,4 +363,4 @@ passed multiple times. The expected format is ``name=value``. For example:: # content of pytest.ini [pytest] - empty_parameterset = xfail \ No newline at end of file + empty_parameter_set_mark = xfail \ No newline at end of file From 77de45cce3bac71119c99581b848b5e20660276d Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 26 Jan 2018 12:01:27 +0100 Subject: [PATCH 07/10] enhance docs for empty_parameter_set_mark according to review comments --- changelog/2527.feature | 2 +- doc/en/customize.rst | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/changelog/2527.feature b/changelog/2527.feature index 97e2a63fb41..ed00398d900 100644 --- a/changelog/2527.feature +++ b/changelog/2527.feature @@ -1 +1 @@ -introduce a pytest ini option to pick the mark for empty parametersets and allow to use xfail(run=False) \ No newline at end of file +Introduce ``empty_parameter_set_mark`` ini option to select which mark to apply when ``@pytest.mark.parametrize`` is given an empty set of parameters. Valid options are ``skip`` (default) and ``xfail``. Note that it is planned to change the default to ``xfail`` in future releases as this is considered less error prone. \ No newline at end of file diff --git a/doc/en/customize.rst b/doc/en/customize.rst index eb79ba6682f..80efe6b8353 100644 --- a/doc/en/customize.rst +++ b/doc/en/customize.rst @@ -350,17 +350,23 @@ passed multiple times. The expected format is ``name=value``. For example:: .. confval:: empty_parameter_set_mark - .. versionadded:: 3.4 + .. versionadded:: 3.4 - allows to pick the action for empty parametersets in parameterization + Allows to pick the action for empty parametersets in parameterization - * ``skip`` skips tests with a empty parameterset - * ``xfail`` marks tests with a empty parameterset as xfail(run=False) + * ``skip`` skips tests with a empty parameterset (default) + * ``xfail`` marks tests with a empty parameterset as xfail(run=False) - The default is ``skip``, it will be shifted to xfail in future. + .. note:: - .. code-block:: ini + it is planned to change the default to ``xfail`` in future releases + as this is considered less error prone. see `#3155`_ + + .. code-block:: ini # content of pytest.ini [pytest] - empty_parameter_set_mark = xfail \ No newline at end of file + empty_parameter_set_mark = xfail + + +.. _`#3155`: https://github.com/pytest-dev/pytest/issues/3155 \ No newline at end of file From a54cd4c2fdf63634560920ec7c4a582b8ca32e0e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 26 Jan 2018 12:05:52 +0100 Subject: [PATCH 08/10] correct testing and usage of the empty_parameter_set_mark config option --- _pytest/mark.py | 2 +- testing/test_mark.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/_pytest/mark.py b/_pytest/mark.py index c3352b387fe..e5d956252f2 100644 --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -293,7 +293,7 @@ def pytest_configure(config): if config.option.strict: MARK_GEN._config = config - empty_parameterset = config.getini("empty_parameterset") + empty_parameterset = config.getini("empty_parameter_set_mark") if empty_parameterset not in ('skip', 'xfail', None, ''): from pytest import UsageError diff --git a/testing/test_mark.py b/testing/test_mark.py index c85a30b41c0..c3a80083cb6 100644 --- a/testing/test_mark.py +++ b/testing/test_mark.py @@ -893,16 +893,16 @@ def test__eq__(self, lhs, rhs, expected): assert (lhs == rhs) == expected -@pytest.mark.parametrize('mark', [None, 'skip', 'xfail']) +@pytest.mark.parametrize('mark', [None, '', 'skip', 'xfail']) def test_parameterset_for_parametrize_marks(testdir, mark): if mark is not None: - testdir.makeini("[pytest]\nempty_parameterset=" + mark) + testdir.makeini("[pytest]\nempty_parameter_set_mark=" + mark) config = testdir.parseconfig() from _pytest.mark import pytest_configure, get_empty_parameterset_mark pytest_configure(config) result_mark = get_empty_parameterset_mark(config, ['a'], all) - if mark is None: + if mark in (None, ''): # normalize to the requested name mark = 'skip' assert result_mark.name == mark From 17a1ed5edf73e12d14665a2e6ed2399ae6415579 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 26 Jan 2018 12:12:26 +0100 Subject: [PATCH 09/10] use a constant to sort out repeated use of the EMPTY_PARAMETERSET_OPTION --- _pytest/mark.py | 13 +++++++------ testing/test_mark.py | 8 ++++++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/_pytest/mark.py b/_pytest/mark.py index e5d956252f2..b49f0fc644e 100644 --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -13,6 +13,8 @@ from .deprecated import MARK_PARAMETERSET_UNPACKING from .compat import NOTSET, getfslineno +EMPTY_PARAMETERSET_OPTION = "empty_parameter_set_mark" + def alias(name, warning=None): getter = attrgetter(name) @@ -95,7 +97,7 @@ def _for_parameterize(cls, argnames, argvalues, function, config): def get_empty_parameterset_mark(config, argnames, function): - requested_mark = config.getini('empty_parameter_set_mark') + requested_mark = config.getini(EMPTY_PARAMETERSET_OPTION) if requested_mark in ('', None, 'skip'): mark = MARK_GEN.skip elif requested_mark == 'xfail': @@ -148,7 +150,7 @@ def pytest_addoption(parser): parser.addini("markers", "markers for test functions", 'linelist') parser.addini( - "empty_parameter_set_mark", + EMPTY_PARAMETERSET_OPTION, "default marker for empty parametersets") @@ -293,13 +295,12 @@ def pytest_configure(config): if config.option.strict: MARK_GEN._config = config - empty_parameterset = config.getini("empty_parameter_set_mark") + empty_parameterset = config.getini(EMPTY_PARAMETERSET_OPTION) if empty_parameterset not in ('skip', 'xfail', None, ''): - from pytest import UsageError raise UsageError( - "empty_parameterset must be one of skip and xfail," - " but it is {!r}".format(empty_parameterset)) + "{!s} must be one of skip and xfail," + " but it is {!r}".format(EMPTY_PARAMETERSET_OPTION, empty_parameterset)) def pytest_unconfigure(config): diff --git a/testing/test_mark.py b/testing/test_mark.py index c3a80083cb6..b4dd656340e 100644 --- a/testing/test_mark.py +++ b/testing/test_mark.py @@ -3,7 +3,10 @@ import sys import pytest -from _pytest.mark import MarkGenerator as Mark, ParameterSet, transfer_markers +from _pytest.mark import ( + MarkGenerator as Mark, ParameterSet, transfer_markers, + EMPTY_PARAMETERSET_OPTION, +) class TestMark(object): @@ -896,7 +899,8 @@ def test__eq__(self, lhs, rhs, expected): @pytest.mark.parametrize('mark', [None, '', 'skip', 'xfail']) def test_parameterset_for_parametrize_marks(testdir, mark): if mark is not None: - testdir.makeini("[pytest]\nempty_parameter_set_mark=" + mark) + testdir.makeini( + "[pytest]\n{}={}".format(EMPTY_PARAMETERSET_OPTION, mark)) config = testdir.parseconfig() from _pytest.mark import pytest_configure, get_empty_parameterset_mark From 169635e8899e60de8155ad151ff99f10c3652fa0 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 27 Jan 2018 11:02:32 -0200 Subject: [PATCH 10/10] Move example of empty_parameter_set_mark closer to the options --- doc/en/customize.rst | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/doc/en/customize.rst b/doc/en/customize.rst index 80efe6b8353..9fe094ba19f 100644 --- a/doc/en/customize.rst +++ b/doc/en/customize.rst @@ -357,16 +357,17 @@ passed multiple times. The expected format is ``name=value``. For example:: * ``skip`` skips tests with a empty parameterset (default) * ``xfail`` marks tests with a empty parameterset as xfail(run=False) - .. note:: + .. code-block:: ini - it is planned to change the default to ``xfail`` in future releases - as this is considered less error prone. see `#3155`_ + # content of pytest.ini + [pytest] + empty_parameter_set_mark = xfail - .. code-block:: ini + .. note:: + + The default value of this option is planned to change to ``xfail`` in future releases + as this is considered less error prone, see `#3155`_ for more details. - # content of pytest.ini - [pytest] - empty_parameter_set_mark = xfail -.. _`#3155`: https://github.com/pytest-dev/pytest/issues/3155 \ No newline at end of file +.. _`#3155`: https://github.com/pytest-dev/pytest/issues/3155