empty parameterset - enable opt to xfail

_pytest/mark.py

@@ -13,6 +13,8 @@
 from .deprecated import MARK_PARAMETERSET_UNPACKING
 from .compat import NOTSET, getfslineno
 
+EMPTY_PARAMETERSET_OPTION = "empty_parameter_set_mark"
+
 
 def alias(name, warning=None):
     getter = attrgetter(name)
@@ -73,7 +75,7 @@ def extract_from(cls, parameterset, legacy_force_tuple=False):
         return cls(argval, marks=newmarks, id=None)
 
     @classmethod
-    def _for_parameterize(cls, argnames, argvalues, function):
+    def _for_parameterize(cls, argnames, argvalues, function, config):
         if not isinstance(argnames, (tuple, list)):
             argnames = [x.strip() for x in argnames.split(",") if x.strip()]
             force_tuple = len(argnames) == 1
@@ -85,10 +87,7 @@ def _for_parameterize(cls, argnames, argvalues, function):
         del argvalues
 
         if not parameters:
-            fs, lineno = getfslineno(function)
-            reason = "got empty parameter set %r, function %s at %s:%d" % (
-                argnames, function.__name__, fs, lineno)
-            mark = MARK_GEN.skip(reason=reason)
+            mark = get_empty_parameterset_mark(config, argnames, function)
             parameters.append(ParameterSet(
                 values=(NOTSET,) * len(argnames),
                 marks=[mark],
@@ -97,6 +96,20 @@ def _for_parameterize(cls, argnames, argvalues, function):
         return argnames, parameters
 
 
+def get_empty_parameterset_mark(config, argnames, function):
+    requested_mark = config.getini(EMPTY_PARAMETERSET_OPTION)
+    if requested_mark in ('', None, 'skip'):
+        mark = MARK_GEN.skip
+    elif requested_mark == 'xfail':
+        mark = MARK_GEN.xfail(run=False)
+    else:
+        raise LookupError(requested_mark)
+    fs, lineno = getfslineno(function)
+    reason = "got empty parameter set %r, function %s at %s:%d" % (
+        argnames, function.__name__, fs, lineno)
+    return mark(reason=reason)
+
+
 class MarkerError(Exception):
 
     """Error in use of a pytest marker/attribute."""
@@ -136,6 +149,9 @@ def pytest_addoption(parser):
     )
 
     parser.addini("markers", "markers for test functions", 'linelist')
+    parser.addini(
+        EMPTY_PARAMETERSET_OPTION,
+        "default marker for empty parametersets")
 
 
 def pytest_cmdline_main(config):
@@ -279,6 +295,13 @@ def pytest_configure(config):
     if config.option.strict:
         MARK_GEN._config = config
 
+    empty_parameterset = config.getini(EMPTY_PARAMETERSET_OPTION)
+
+    if empty_parameterset not in ('skip', 'xfail', None, ''):
+        raise UsageError(
+            "{!s} must be one of skip and xfail,"
+            " but it is {!r}".format(EMPTY_PARAMETERSET_OPTION, empty_parameterset))
+
 
 def pytest_unconfigure(config):
     MARK_GEN._config = getattr(config, '_old_mark_config', None)

_pytest/python.py

@@ -786,7 +786,7 @@ def parametrize(self, argnames, argvalues, indirect=False, ids=None,
         from _pytest.mark import ParameterSet
         from py.io import saferepr
         argnames, parameters = ParameterSet._for_parameterize(
-            argnames, argvalues, self.function)
+            argnames, argvalues, self.function, self.config)
         del argvalues
 
         if scope is None:

changelog/2527.feature

@@ -0,0 +1 @@
+Introduce ``empty_parameter_set_mark`` ini option to select which mark to apply when ``@pytest.mark.parametrize`` is given an empty set of parameters. Valid options are ``skip`` (default) and ``xfail``. Note that it is planned to change the default to ``xfail`` in future releases as this is considered less error prone.

doc/en/customize.rst

@@ -346,3 +346,28 @@ passed multiple times. The expected format is ``name=value``. For example::
         # content of pytest.ini
         [pytest]
         console_output_style = classic
+
+
+.. confval:: empty_parameter_set_mark
+
+    .. versionadded:: 3.4
+
+    Allows to pick the action for empty parametersets in parameterization
+
+    * ``skip`` skips tests with a empty parameterset (default)
+    * ``xfail`` marks tests with a empty parameterset as xfail(run=False)
+
+    .. code-block:: ini
+
+      # content of pytest.ini
+      [pytest]
+      empty_parameter_set_mark = xfail
+
+    .. note::
+
+      The default value of this option is planned to change to ``xfail`` in future releases
+      as this is considered less error prone, see `#3155`_ for more details.
+
+
+
+.. _`#3155`: https://github.com/pytest-dev/pytest/issues/3155

testing/python/metafunc.py

@@ -14,7 +14,7 @@
 
 
 class TestMetafunc(object):
-    def Metafunc(self, func):
+    def Metafunc(self, func, config=None):
         # the unit tests of this class check if things work correctly
         # on the funcarg level, so we don't need a full blown
         # initiliazation
@@ -26,7 +26,7 @@ def __init__(self, names):
 
         names = fixtures.getfuncargnames(func)
         fixtureinfo = FixtureInfo(names)
-        return python.Metafunc(func, fixtureinfo, None)
+        return python.Metafunc(func, fixtureinfo, config)
 
     def test_no_funcargs(self, testdir):
         def function():
@@ -156,7 +156,19 @@ def func(x, y):
     def test_parametrize_empty_list(self):
         def func(y):
             pass
-        metafunc = self.Metafunc(func)
+
+        class MockConfig(object):
+            def getini(self, name):
+                return ''
+
+            @property
+            def hook(self):
+                return self
+
+            def pytest_make_parametrize_id(self, **kw):
+                pass
+
+        metafunc = self.Metafunc(func, MockConfig())
         metafunc.parametrize("y", [])
         assert 'skip' == metafunc._calls[0].marks[0].name
 

testing/test_mark.py

@@ -3,7 +3,10 @@
 import sys
 
 import pytest
-from _pytest.mark import MarkGenerator as Mark, ParameterSet, transfer_markers
+from _pytest.mark import (
+    MarkGenerator as Mark, ParameterSet, transfer_markers,
+    EMPTY_PARAMETERSET_OPTION,
+)
 
 
 class TestMark(object):
@@ -891,3 +894,27 @@ class TestMarkDecorator(object):
     ])
     def test__eq__(self, lhs, rhs, expected):
         assert (lhs == rhs) == expected
+
+
+@pytest.mark.parametrize('mark', [None, '', 'skip', 'xfail'])
+def test_parameterset_for_parametrize_marks(testdir, mark):
+    if mark is not None:
+        testdir.makeini(
+            "[pytest]\n{}={}".format(EMPTY_PARAMETERSET_OPTION, mark))
+
+    config = testdir.parseconfig()
+    from _pytest.mark import pytest_configure, get_empty_parameterset_mark
+    pytest_configure(config)
+    result_mark = get_empty_parameterset_mark(config, ['a'], all)
+    if mark in (None, ''):
+        # normalize to the requested name
+        mark = 'skip'
+    assert result_mark.name == mark
+    assert result_mark.kwargs['reason'].startswith("got empty parameter set ")
+    if mark == 'xfail':
+        assert result_mark.kwargs.get('run') is False
+
+
+def test_parameterset_for_parametrize_bad_markname(testdir):
+    with pytest.raises(pytest.UsageError):
+        test_parameterset_for_parametrize_marks(testdir, 'bad')