Treat more exceptions as fails

hypothesis-python/RELEASE.rst

@@ -0,0 +1,9 @@
+RELEASE_TYPE: minor
+
+This release follows :pypi:`pytest` in considering :class:`SystemExit` and
+:class:`GeneratorExit` exceptions to be test failures, meaning that we will
+shink to minimal examples and check for flakiness even though they subclass
+:class:`BaseException` directly (:issue:`2223`).
+
+:class:`KeyboardInterrupt` continues to interrupt everything, and will be
+re-raised immediately.

hypothesis-python/src/hypothesis/core.py

@@ -493,7 +493,11 @@ def failure_exceptions_to_catch():
     This is intended to cover most common test runners; if you would
     like another to be added please open an issue or pull request.
     """
-    exceptions = [Exception]
+    # While SystemExit and GeneratorExit are instances of BaseException, we also
+    # expect them to be deterministic - unlike KeyboardInterrupt - and so we treat
+    # them as standard exceptions, check for flakiness, etc.
+    # See https://github.com/HypothesisWorks/hypothesis/issues/2223 for details.
+    exceptions = [Exception, SystemExit, GeneratorExit]
     if "_pytest" in sys.modules:  # pragma: no branch
         exceptions.append(sys.modules["_pytest"].outcomes.Failed)
     return tuple(exceptions)

hypothesis-python/tests/cover/test_baseexception.py → hypothesis-python/tests/nocover/test_baseexception.py

@@ -48,9 +48,7 @@ def test_do_nothing(x):
         test_do_nothing()
 
 
-@pytest.mark.parametrize(
-    "e", [KeyboardInterrupt, SystemExit, GeneratorExit, ValueError]
-)
+@pytest.mark.parametrize("e", [KeyboardInterrupt, ValueError])
 def test_baseexception_no_rerun_no_flaky(e):
     runs = [0]
     interrupt = 3
@@ -100,3 +98,31 @@ def test_do_nothing(x):
         # Now SystemExit and GeneratorExit are caught like other exceptions
         with pytest.raises(Flaky):
             test_do_nothing()
+
+
+TEMPLATE = """
+from hypothesis import given, note, strategies as st
+
+@st.composite
+def things(draw):
+    raise {exception}
+
+
+@given(st.data(), st.integers())
+def test(data, x):
+    if x > 100:
+        data.draw({strategy})
+        raise {exception}
+"""
+
+
+@pytest.mark.parametrize("exc_name", ["SystemExit", "GeneratorExit"])
+@pytest.mark.parametrize("use_composite", [True, False])
+def test_explanations(testdir, exc_name, use_composite):
+    code = TEMPLATE.format(
+        exception=exc_name, strategy="things()" if use_composite else "st.none()"
+    )
+    test_file = str(testdir.makepyfile(code))
+    pytest_stdout = str(testdir.runpytest_inprocess(test_file, "--tb=native").stdout)
+    assert "x=101" in pytest_stdout
+    assert exc_name in pytest_stdout