Merge pull request #4173 from tybug/node-template

Implement `NodeTemplate` + small fixes

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+Internal refactorings in preparation for upcoming changes.

hypothesis-python/src/hypothesis/control.py

@@ -149,10 +149,10 @@ def prep_args_kwargs_from_strategies(self, kwarg_strategies):
         arg_labels = {}
         kwargs = {}
         for k, s in kwarg_strategies.items():
-            start_idx = self.data.index_ir
+            start_idx = len(self.data.ir_nodes)
             with deprecate_random_in_strategy("from {}={!r}", k, s) as check:
                 obj = check(self.data.draw(s, observe_as=f"generate:{k}"))
-            end_idx = self.data.index_ir
+            end_idx = len(self.data.ir_nodes)
             kwargs[k] = obj
 
             # This high up the stack, we can't see or really do much with the conjecture

hypothesis-python/src/hypothesis/internal/conjecture/data.py

@@ -58,6 +58,7 @@
     sign_aware_lte,
 )
 from hypothesis.internal.intervalsets import IntervalSet
+from hypothesis.reporting import debug_report
 
 if TYPE_CHECKING:
     from typing import TypeAlias
@@ -969,6 +970,11 @@ def trivial(self):
             min_value = self.kwargs["min_value"]
             max_value = self.kwargs["max_value"]
 
+            # shrink_towards is not respected for unbounded integers. (though
+            # probably it should be?)
+            if min_value is None and max_value is None:
+                return self.value == 0
+
             if min_value is not None:
                 shrink_towards = max(min_value, shrink_towards)
             if max_value is not None:
@@ -1000,6 +1006,9 @@ def trivial(self):
             # also not incorrect to be conservative here.
             return False
         if self.ir_type == "boolean":
+            p = self.kwargs["p"]
+            if p == 1.0:
+                return True
             return self.value is False
         if self.ir_type == "string":
             # smallest size and contains only the smallest-in-shrink-order character.
@@ -1038,6 +1047,15 @@ def __repr__(self) -> str:
         return f"{self.ir_type} {self.value!r}{forced_marker} {self.kwargs!r}"
 
 
+@attr.s(slots=True)
+class NodeTemplate:
+    type: Literal["simplest"] = attr.ib()
+    size: int = attr.ib()
+
+    def __attrs_post_init__(self) -> None:
+        assert self.size > 0
+
+
 def ir_value_permitted(value, ir_type, kwargs):
     if ir_type == "integer":
         min_value = kwargs["min_value"]
@@ -1085,8 +1103,11 @@ def ir_size(ir: Iterable[IRType]) -> int:
     return len(ir_to_bytes(ir))
 
 
-def ir_size_nodes(nodes: Iterable[IRNode]) -> int:
-    return ir_size([n.value for n in nodes])
+def ir_size_nodes(nodes: Iterable[Union[IRNode, NodeTemplate]]) -> int:
+    size = 0
+    for node in nodes:
+        size += node.size if isinstance(node, NodeTemplate) else ir_size([node.value])
+    return size
 
 
 def ir_value_key(ir_type, v):
@@ -1967,7 +1988,7 @@ def for_buffer(
     @classmethod
     def for_ir_tree(
         cls,
-        ir_tree_prefix: Sequence[IRNode],
+        ir_tree_prefix: Sequence[Union[IRNode, NodeTemplate]],
         *,
         observer: Optional[DataObserver] = None,
         provider: Union[type, PrimitiveProvider] = HypothesisProvider,
@@ -1996,7 +2017,7 @@ def __init__(
         random: Optional[Random],
         observer: Optional[DataObserver] = None,
         provider: Union[type, PrimitiveProvider] = HypothesisProvider,
-        ir_tree_prefix: Optional[Sequence[IRNode]] = None,
+        ir_tree_prefix: Optional[Sequence[Union[IRNode, NodeTemplate]]] = None,
         max_length_ir: Optional[int] = None,
     ) -> None:
         from hypothesis.internal.conjecture.engine import BUFFER_SIZE_IR
@@ -2116,6 +2137,7 @@ def _draw(self, ir_type, kwargs, *, observe, forced, fake_forced):
         # end of the function, but avoids trying to use a null self.random when
         # drawing past the node of a ConjectureData.for_ir_tree data.
         if self.length_ir == self.max_length_ir:
+            debug_report(f"overrun because hit {self.max_length_ir=}")
             self.mark_overrun()
 
         if self.ir_prefix is not None and observe:
@@ -2127,6 +2149,7 @@ def _draw(self, ir_type, kwargs, *, observe, forced, fake_forced):
                         ir_type, kwargs, forced=forced, random=self.__random
                     )
                 except StopTest:
+                    debug_report("overrun because ir_to_buffer overran")
                     self.mark_overrun()
 
             if forced is None:
@@ -2143,7 +2166,10 @@ def _draw(self, ir_type, kwargs, *, observe, forced, fake_forced):
                 value, kwargs=kwargs, was_forced=was_forced
             )
             size = ir_size([value])
-            if size + self.length_ir > self.max_length_ir:
+            if self.length_ir + size > self.max_length_ir:
+                debug_report(
+                    f"overrun because {self.length_ir=} + {size=} > {self.max_length_ir=}"
+                )
                 self.mark_overrun()
 
             node = IRNode(
@@ -2155,7 +2181,6 @@ def _draw(self, ir_type, kwargs, *, observe, forced, fake_forced):
             )
             self.__example_record.record_ir_draw()
             self.ir_nodes += (node,)
-            self.index_ir += 1
             self.length_ir += size
 
         return value
@@ -2341,6 +2366,25 @@ def _pop_ir_tree_node(
         assert self.index_ir < len(self.ir_prefix)
 
         node = self.ir_prefix[self.index_ir]
+        if isinstance(node, NodeTemplate):
+            assert node.size >= 0
+            # node templates have to be at the end for now, since it's not immediately
+            # apparent how to handle overruning a node template while generating a single
+            # node if the alternative is not "the entire data is an overrun".
+            assert self.index_ir == len(self.ir_prefix) - 1
+            if node.type == "simplest":
+                try:
+                    value = buffer_to_ir(ir_type, kwargs, buffer=bytes(BUFFER_SIZE))
+                except StopTest:
+                    self.mark_overrun()
+            else:
+                raise NotImplementedError
+
+            node.size -= ir_size([value])
+            if node.size < 0:
+                self.mark_overrun()
+            return value
+
         value = node.value
         # If we're trying to:
         # * draw a different ir type at the same location
@@ -2381,6 +2425,7 @@ def _pop_ir_tree_node(
                 #   buffer_to_ir(ir_type, kwargs, buffer=bytes(BUFFER_SIZE))
                 self.mark_overrun()
 
+        self.index_ir += 1
         return value
 
     def as_result(self) -> Union[ConjectureResult, _Overrun]:

hypothesis-python/tests/conjecture/common.py

@@ -9,6 +9,7 @@
 # obtain one at https://mozilla.org/MPL/2.0/.
 
 import math
+import sys
 from contextlib import contextmanager
 
 from hypothesis import HealthCheck, Phase, assume, settings, strategies as st
@@ -19,12 +20,14 @@
     COLLECTION_DEFAULT_MAX_SIZE,
     ConjectureData,
     IRNode,
+    IRType,
     Status,
 )
 from hypothesis.internal.conjecture.engine import BUFFER_SIZE, ConjectureRunner
 from hypothesis.internal.conjecture.utils import calc_label_from_name
 from hypothesis.internal.entropy import deterministic_PRNG
 from hypothesis.internal.floats import SMALLEST_SUBNORMAL, sign_aware_lte
+from hypothesis.internal.intervalsets import IntervalSet
 from hypothesis.strategies._internal.strings import OneCharStringStrategy, TextStrategy
 
 from tests.common.strategies import intervals
@@ -50,6 +53,10 @@ def run_to_buffer(f):
     return bytes(run_to_data(f).buffer)
 
 
+def run_to_nodes(f):
+    return run_to_data(f).ir_nodes
+
+
 @contextmanager
 def buffer_size_limit(n):
     original = engine_module.BUFFER_SIZE
@@ -364,3 +371,64 @@ def ir_nodes(draw, *, was_forced=None, ir_type=None):
     was_forced = draw(st.booleans()) if was_forced is None else was_forced
 
     return IRNode(ir_type=ir_type, value=value, kwargs=kwargs, was_forced=was_forced)
+
+
+def ir(*values: list[IRType]) -> list[IRNode]:
+    """
+    For inline-creating an ir node or list of ir nodes, where you don't care about the
+    kwargs. This uses maximally-permissable kwargs and infers the ir_type you meant
+    based on the type of the value.
+
+    You can optionally pass (value, kwargs) to as an element in order to override
+    the default kwargs for that element.
+    """
+    mapping = {
+        float: (
+            "float",
+            {
+                "min_value": -math.inf,
+                "max_value": math.inf,
+                "allow_nan": True,
+                "smallest_nonzero_magnitude": SMALLEST_SUBNORMAL,
+            },
+        ),
+        int: (
+            "integer",
+            {
+                "min_value": None,
+                "max_value": None,
+                "weights": None,
+                "shrink_towards": 0,
+            },
+        ),
+        str: (
+            "string",
+            {
+                "intervals": IntervalSet(((0, sys.maxunicode),)),
+                "min_size": 0,
+                "max_size": COLLECTION_DEFAULT_MAX_SIZE,
+            },
+        ),
+        bytes: ("bytes", {"min_size": 0, "max_size": COLLECTION_DEFAULT_MAX_SIZE}),
+        bool: ("boolean", {"p": 0.5}),
+    }
+    nodes = []
+    for value in values:
+        override_kwargs = {}
+        if isinstance(value, tuple):
+            (value, override_kwargs) = value
+            if override_kwargs is None:
+                override_kwargs = {}
+
+        (ir_type, kwargs) = mapping[type(value)]
+
+        nodes.append(
+            IRNode(
+                ir_type=ir_type,
+                value=value,
+                kwargs=kwargs | override_kwargs,
+                was_forced=False,
+            )
+        )
+
+    return tuple(nodes)