Merge pull request #4022 from tybug/post-test-case-hook-fix

Adjust `post_test_case_hook` scope

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+This patch fixes an issue when realizing symbolics with our experimental :obj:`~hypothesis.settings.backend` setting.

hypothesis-python/src/hypothesis/internal/conjecture/data.py

@@ -1211,7 +1211,7 @@ class PrimitiveProvider(abc.ABC):
     def __init__(self, conjecturedata: Optional["ConjectureData"], /) -> None:
         self._cd = conjecturedata
 
-    def post_test_case_hook(self, value):
+    def post_test_case_hook(self, value: IRType) -> IRType:
         # hook for providers to modify values returned by draw_* after a full
         # test case concludes. Originally exposed for crosshair to reify its
         # symbolic values into actual values.
@@ -1966,7 +1966,9 @@ def __init__(
         self.max_depth = 0
         self.has_discards = False
 
-        self.provider = provider(self) if isinstance(provider, type) else provider
+        self.provider: PrimitiveProvider = (
+            provider(self) if isinstance(provider, type) else provider
+        )
         assert isinstance(self.provider, PrimitiveProvider)
 
         self.__result: "Optional[ConjectureResult]" = None

hypothesis-python/src/hypothesis/internal/conjecture/engine.py

@@ -38,7 +38,7 @@
 from hypothesis import HealthCheck, Phase, Verbosity, settings as Settings
 from hypothesis._settings import local_settings
 from hypothesis.database import ExampleDatabase
-from hypothesis.errors import InvalidArgument, StopTest
+from hypothesis.errors import Flaky, HypothesisException, InvalidArgument, StopTest
 from hypothesis.internal.cache import LRUReusedCache
 from hypothesis.internal.compat import (
     NotRequired,
@@ -453,6 +453,24 @@ def test_function(self, data: ConjectureData) -> None:
                     ),
                 }
                 self.stats_per_test_case.append(call_stats)
+                if self.settings.backend != "hypothesis":
+                    for node in data.examples.ir_tree_nodes:
+                        value = data.provider.post_test_case_hook(node.value)
+                        expected_type = {
+                            "string": str,
+                            "float": float,
+                            "integer": int,
+                            "boolean": bool,
+                            "bytes": bytes,
+                        }[node.ir_type]
+                        if type(value) is not expected_type:
+                            raise HypothesisException(
+                                f"expected {expected_type} from "
+                                f"{data.provider.post_test_case_hook.__qualname__}, "
+                                f"got {type(value)} ({value!r})"
+                            )
+                        node.value = value
+
                 self._cache(data)
                 if data.invalid_at is not None:  # pragma: no branch # coverage bug?
                     self.misaligned_count += 1
@@ -496,18 +514,23 @@ def test_function(self, data: ConjectureData) -> None:
 
         if data.status == Status.INTERESTING:
             if self.settings.backend != "hypothesis":
-                for node in data.examples.ir_tree_nodes:
-                    value = data.provider.post_test_case_hook(node.value)
-                    # require providers to return something valid here.
-                    assert (
-                        value is not None
-                    ), "providers must return a non-null value from post_test_case_hook"
-                    node.value = value
-
                 # drive the ir tree through the test function to convert it
                 # to a buffer
+                initial_origin = data.interesting_origin
                 data = ConjectureData.for_ir_tree(data.examples.ir_tree_nodes)
                 self.__stoppable_test_function(data)
+                data.freeze()
+                # we'd like to use expected_failure machinery here from
+                # StateForActualGivenExecution for better diagnostic reports of eg
+                # flaky deadlines, but we're too low down in the engine for that.
+                # for now a worse generic flaky error will have to do.
+                if data.status != Status.INTERESTING:
+                    raise Flaky(
+                        f"Inconsistent results from replaying a failing test case!\n"
+                        f"  last: {Status.INTERESTING.name} from {initial_origin}\n"
+                        f"  this: {data.status.name}"
+                    )
+
                 self._cache(data)
 
             key = data.interesting_origin

hypothesis-python/tests/conjecture/test_alt_backend.py

@@ -18,7 +18,7 @@
 
 from hypothesis import given, settings, strategies as st
 from hypothesis.database import InMemoryExampleDatabase
-from hypothesis.errors import InvalidArgument
+from hypothesis.errors import Flaky, HypothesisException, InvalidArgument
 from hypothesis.internal.compat import int_to_bytes
 from hypothesis.internal.conjecture.data import (
     AVAILABLE_PROVIDERS,
@@ -354,3 +354,39 @@ def test_function(n):
             <= test_case_lifetime_init_count
             <= test_function_count + 10
         )
+
+
+def test_flaky_with_backend():
+    with temp_register_backend("trivial", TrivialProvider):
+
+        calls = 0
+
+        @given(st.integers())
+        @settings(backend="trivial", database=None)
+        def test_function(n):
+            nonlocal calls
+            calls += 1
+            assert n != calls % 2
+
+        with pytest.raises(Flaky):
+            test_function()
+
+
+class BadPostTestCaseHookProvider(TrivialProvider):
+    def post_test_case_hook(self, value):
+        return None
+
+
+def test_bad_post_test_case_hook():
+    with temp_register_backend("bad_hook", BadPostTestCaseHookProvider):
+
+        @given(st.integers())
+        @settings(backend="bad_hook")
+        def test_function(n):
+            pass
+
+        with pytest.raises(
+            HypothesisException,
+            match="expected .* from BadPostTestCaseHookProvider.post_test_case_hook",
+        ):
+            test_function()