Skip to content

Commit

Permalink
Fix teardown error reporting when --maxfail=1 (pytest-dev#11721)
Browse files Browse the repository at this point in the history
Co-authored-by: Ran Benita <ran@unusedvar.com>
  • Loading branch information
bbrown1867 and bluetech authored Jan 3, 2024
1 parent f017df4 commit 12b9bd5
Show file tree
Hide file tree
Showing 6 changed files with 156 additions and 2 deletions.
1 change: 1 addition & 0 deletions AUTHORS
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ Aviral Verma
Aviv Palivoda
Babak Keyvani
Barney Gale
Ben Brown
Ben Gartner
Ben Webb
Benjamin Peterson
Expand Down
1 change: 1 addition & 0 deletions changelog/11706.bugfix.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix reporting of teardown errors in higher-scoped fixtures when using `--maxfail` or `--stepwise`.
42 changes: 40 additions & 2 deletions src/_pytest/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import importlib
import os
import sys
import warnings
from pathlib import Path
from typing import AbstractSet
from typing import Callable
Expand Down Expand Up @@ -44,6 +45,7 @@
from _pytest.reports import TestReport
from _pytest.runner import collect_one_node
from _pytest.runner import SetupState
from _pytest.warning_types import PytestWarning


def pytest_addoption(parser: Parser) -> None:
Expand Down Expand Up @@ -548,8 +550,8 @@ def __init__(self, config: Config) -> None:
)
self.testsfailed = 0
self.testscollected = 0
self.shouldstop: Union[bool, str] = False
self.shouldfail: Union[bool, str] = False
self._shouldstop: Union[bool, str] = False
self._shouldfail: Union[bool, str] = False
self.trace = config.trace.root.get("collection")
self._initialpaths: FrozenSet[Path] = frozenset()
self._initialpaths_with_parents: FrozenSet[Path] = frozenset()
Expand All @@ -576,6 +578,42 @@ def __repr__(self) -> str:
self.testscollected,
)

@property
def shouldstop(self) -> Union[bool, str]:
return self._shouldstop

@shouldstop.setter
def shouldstop(self, value: Union[bool, str]) -> None:
# The runner checks shouldfail and assumes that if it is set we are
# definitely stopping, so prevent unsetting it.
if value is False and self._shouldstop:
warnings.warn(
PytestWarning(
"session.shouldstop cannot be unset after it has been set; ignoring."
),
stacklevel=2,
)
return
self._shouldstop = value

@property
def shouldfail(self) -> Union[bool, str]:
return self._shouldfail

@shouldfail.setter
def shouldfail(self, value: Union[bool, str]) -> None:
# The runner checks shouldfail and assumes that if it is set we are
# definitely stopping, so prevent unsetting it.
if value is False and self._shouldfail:
warnings.warn(
PytestWarning(
"session.shouldfail cannot be unset after it has been set; ignoring."
),
stacklevel=2,
)
return
self._shouldfail = value

@property
def startpath(self) -> Path:
"""The path from which pytest was invoked.
Expand Down
4 changes: 4 additions & 0 deletions src/_pytest/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,10 @@ def runtestprotocol(
show_test_item(item)
if not item.config.getoption("setuponly", False):
reports.append(call_and_report(item, "call", log))
# If the session is about to fail or stop, teardown everything - this is
# necessary to correctly report fixture teardown errors (see #11706)
if item.session.shouldfail or item.session.shouldstop:
nextitem = None
reports.append(call_and_report(item, "teardown", log, nextitem=nextitem))
# After all teardown hooks have been called
# want funcargs and request info to go away.
Expand Down
50 changes: 50 additions & 0 deletions testing/test_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -1087,3 +1087,53 @@ def func() -> None:
with pytest.raises(TypeError) as excinfo:
OutcomeException(func) # type: ignore
assert str(excinfo.value) == expected


def test_teardown_session_failed(pytester: Pytester) -> None:
"""Test that higher-scoped fixture teardowns run in the context of the last
item after the test session bails early due to --maxfail.
Regression test for #11706.
"""
pytester.makepyfile(
"""
import pytest
@pytest.fixture(scope="module")
def baz():
yield
pytest.fail("This is a failing teardown")
def test_foo(baz):
pytest.fail("This is a failing test")
def test_bar(): pass
"""
)
result = pytester.runpytest("--maxfail=1")
result.assert_outcomes(failed=1, errors=1)


def test_teardown_session_stopped(pytester: Pytester) -> None:
"""Test that higher-scoped fixture teardowns run in the context of the last
item after the test session bails early due to --stepwise.
Regression test for #11706.
"""
pytester.makepyfile(
"""
import pytest
@pytest.fixture(scope="module")
def baz():
yield
pytest.fail("This is a failing teardown")
def test_foo(baz):
pytest.fail("This is a failing test")
def test_bar(): pass
"""
)
result = pytester.runpytest("--stepwise")
result.assert_outcomes(failed=1, errors=1)
60 changes: 60 additions & 0 deletions testing/test_session.py
Original file line number Diff line number Diff line change
Expand Up @@ -418,3 +418,63 @@ def test_rootdir_wrong_option_arg(pytester: Pytester) -> None:
result.stderr.fnmatch_lines(
["*Directory *wrong_dir* not found. Check your '--rootdir' option.*"]
)


def test_shouldfail_is_sticky(pytester: Pytester) -> None:
"""Test that session.shouldfail cannot be reset to False after being set.
Issue #11706.
"""
pytester.makeconftest(
"""
def pytest_sessionfinish(session):
assert session.shouldfail
session.shouldfail = False
assert session.shouldfail
"""
)
pytester.makepyfile(
"""
import pytest
def test_foo():
pytest.fail("This is a failing test")
def test_bar(): pass
"""
)

result = pytester.runpytest("--maxfail=1", "-Wall")

result.assert_outcomes(failed=1, warnings=1)
result.stdout.fnmatch_lines("*session.shouldfail cannot be unset*")


def test_shouldstop_is_sticky(pytester: Pytester) -> None:
"""Test that session.shouldstop cannot be reset to False after being set.
Issue #11706.
"""
pytester.makeconftest(
"""
def pytest_sessionfinish(session):
assert session.shouldstop
session.shouldstop = False
assert session.shouldstop
"""
)
pytester.makepyfile(
"""
import pytest
def test_foo():
pytest.fail("This is a failing test")
def test_bar(): pass
"""
)

result = pytester.runpytest("--stepwise", "-Wall")

result.assert_outcomes(failed=1, warnings=1)
result.stdout.fnmatch_lines("*session.shouldstop cannot be unset*")

0 comments on commit 12b9bd5

Please sign in to comment.