fix(plugins/forks): Make fork_covariant_parametrize into a marker (#…

…1057)

* fix(plugins/forks): Make `fork_covariant_parametrize` into a marker

* fix: changelog

* Update docs/writing_tests/test_markers.md

Co-authored-by: danceratopz <danceratopz@gmail.com>

* refactor(plugins/forks): rename to `parametrize_by_fork` and argument names

---------

Co-authored-by: danceratopz <danceratopz@gmail.com>

docs/CHANGELOG.md

@@ -64,7 +64,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 - ✨ Generate Transaction Test type ([#933](https://github.com/ethereum/execution-spec-tests/pull/933)).
 - ✨ Add a default location for evm logs (`--evm-dump-dir`) when filling tests ([#999](https://github.com/ethereum/execution-spec-tests/pull/999)).
 - ✨ Slow tests now have greater timeout when making a request to the T8N server ([#1037](https://github.com/ethereum/execution-spec-tests/pull/1037)).
-- ✨ Introduce [`fork_covariant_parametrize`](https://ethereum.github.io/execution-spec-tests/main/writing_tests/test_markers/#custom-fork-covariant-markers) helper function ([#1019](https://github.com/ethereum/execution-spec-tests/pull/1019)).
+- ✨ Introduce [`pytest.mark.parametrize_by_fork`](https://ethereum.github.io/execution-spec-tests/main/writing_tests/test_markers/#pytestmarkfork_parametrize) helper marker ([#1019](https://github.com/ethereum/execution-spec-tests/pull/1019), [#1057](https://github.com/ethereum/execution-spec-tests/pull/1057)).
 - 🔀 Update EIP-7251 according to [spec updates](https://github.com/ethereum/EIPs/pull/9127) ([#1024](https://github.com/ethereum/execution-spec-tests/pull/1024)).
 - 🔀 Update EIP-7002 according to [spec updates](https://github.com/ethereum/EIPs/pull/9119) ([#1024](https://github.com/ethereum/execution-spec-tests/pull/1024)).
 - 🐞 fix(consume): allow absolute paths with `--evm-bin` ([#1052](https://github.com/ethereum/execution-spec-tests/pull/1052)).

docs/writing_tests/test_markers.md

@@ -271,27 +271,32 @@ def test_something_with_all_tx_types_but_skip_type_1(state_test_only, tx_type):
 
 In this example, the test will be skipped if `tx_type` is equal to 1 by returning a `pytest.mark.skip` marker, and return `None` otherwise.
 
-## Custom Fork Covariant Markers
+## `@pytest.mark.parametrize_by_fork`
 
-Custom fork covariant markers can be created by using the `fork_covariant_parametrize` decorator.
+A test can be dynamically parametrized based on the fork using the `parametrize_by_fork` marker.
 
-This decorator takes three arguments:
+This marker takes two positional arguments:
 
-- `parameter_names`: A list of parameter names that will be parametrized using the custom function.
-- `fn`: A function that takes the fork as parameter and returns a list of values that will be used to parametrize the test.
-- `marks`: A marker, list of markers, or a lambda function that can be used to add additional markers to the test.
+- `argnames`: A list of parameter names that will be parametrized using the custom function.
+- `fn`: A function that takes the fork as parameter and returns a list of values that will be used to parametrize the test at that specific fork.
+
+And one keyword argument:
+
+- `marks` (optional): A marker, list of markers, or a lambda function that can be used to add additional markers to the generated tests.
+
+The marked test function will be parametrized by the values returned by the `fn` function for each fork.
+
+If the parameters that are being parametrized is only a single parameter, the return value of `fn` should be a list of values for that parameter.
+
+If the parameters that are being parametrized are multiple, the return value of `fn` should be a list of tuples/lists, where each tuple contains the values for each parameter.
 
 ```python
 import pytest
 
-from pytest_plugins import fork_covariant_parametrize
-
 def covariant_function(fork):
     return [[1, 2], [3, 4]] if fork.name() == "Paris" else [[4, 5], [5, 6], [6, 7]]
 
-@fork_covariant_parametrize(parameter_names=[
-    "test_parameter", "test_parameter_2"
-], fn=covariant_function)
+@pytest.mark.parametrize_by_fork("test_parameter,test_parameter_2", covariant_function)
 @pytest.mark.valid_from("Paris")
 @pytest.mark.valid_until("Shanghai")
 def test_case(state_test_only, test_parameter, test_parameter_2):

src/pytest_plugins/__init__.py

@@ -1,5 +1 @@
 """Package containing pytest plugins related to test filling."""
-
-from .forks import fork_covariant_parametrize
-
-__all__ = ["fork_covariant_parametrize"]

src/pytest_plugins/forks/__init__.py

@@ -3,7 +3,3 @@
 tests based on the user-provided fork range the tests' specified validity
 markers.
 """
-
-from .forks import fork_covariant_parametrize
-
-__all__ = ["fork_covariant_parametrize"]

src/pytest_plugins/forks/forks.py

@@ -105,15 +105,15 @@ def __init__(
         self.fork = fork
 
     @property
-    def parameter_names(self) -> List[str]:
+    def argnames(self) -> List[str]:
         """Return the parameter names for the test case."""
-        parameter_names = []
+        argnames = []
         for p in self.fork_covariant_parameters:
-            parameter_names.extend(p.names)
-        return parameter_names
+            argnames.extend(p.names)
+        return argnames
 
     @property
-    def parameter_values(self) -> List[ParameterSet]:
+    def argvalues(self) -> List[ParameterSet]:
         """Return the parameter values for the test case."""
         parameter_set_combinations = itertools.product(
             # Add the values for each parameter, all of them are lists of at least one element.
@@ -146,7 +146,7 @@ class CovariantDescriptor:
     the parametrized values change depending on the fork.
     """
 
-    parameter_names: List[str] = []
+    argnames: List[str] = []
     fn: Callable[[Fork], List[Any] | Iterable[Any]] | None = None
 
     selector: FunctionType | None = None
@@ -156,8 +156,9 @@ class CovariantDescriptor:
 
     def __init__(
         self,
-        parameter_names: List[str] | str,
+        argnames: List[str] | str,
         fn: Callable[[Fork], List[Any] | Iterable[Any]] | None = None,
+        *,
         selector: FunctionType | None = None,
         marks: None
         | pytest.Mark
@@ -168,17 +169,18 @@ def __init__(
         Initialize a new covariant descriptor.
 
         Args:
-            parameter_names: The names of the parameters that are covariant with the fork.
+            argnames: The names of the parameters that are covariant with the fork.
             fn: A function that takes the fork as the single parameter and returns the values for
                 the parameter for each fork.
             selector: A function that filters the values for the parameter.
             marks: A list of pytest marks to apply to the test cases parametrized by the parameter.
 
         """
-        if isinstance(parameter_names, str):
-            self.parameter_names = parameter_names.split(",")
-        else:
-            self.parameter_names = parameter_names
+        self.argnames = (
+            [argname.strip() for argname in argnames.split(",")]
+            if isinstance(argnames, str)
+            else argnames
+        )
         self.fn = fn
         self.selector = selector
         self.marks = marks
@@ -195,7 +197,7 @@ def process_value(
         if isinstance(parameters_values, ParameterSet):
             return parameters_values
 
-        if len(self.parameter_names) == 1:
+        if len(self.argnames) == 1:
             # Wrap values that are meant for a single parameter in a list
             parameters_values = [parameters_values]
         marks = self.marks
@@ -237,7 +239,7 @@ def add_values(self, fork_parametrizer: ForkParametrizer) -> None:
         values = self.process_values(values)
         assert len(values) > 0
         fork_parametrizer.fork_covariant_parameters.append(
-            ForkCovariantParameter(names=self.parameter_names, values=values)
+            ForkCovariantParameter(names=self.argnames, values=values)
         )
 
 
@@ -300,7 +302,7 @@ def fn(fork: Fork) -> List[Any]:
             return getattr(fork, self.fork_attribute_name)(block_number=0, timestamp=0)
 
         super().__init__(
-            parameter_names=self.marker_parameter_names,
+            argnames=self.marker_parameter_names,
             fn=fn,
             selector=selector,
             marks=marks,
@@ -312,7 +314,7 @@ def covariant_decorator(
     marker_name: str,
     description: str,
     fork_attribute_name: str,
-    parameter_names: List[str],
+    argnames: List[str],
 ) -> Type[CovariantDecorator]:
     """Generate a new covariant decorator subclass."""
     return type(
@@ -322,7 +324,7 @@ def covariant_decorator(
             "marker_name": marker_name,
             "description": description,
             "fork_attribute_name": fork_attribute_name,
-            "marker_parameter_names": parameter_names,
+            "marker_parameter_names": argnames,
         },
     )
 
@@ -333,103 +335,53 @@ def covariant_decorator(
         description="marks a test to be parametrized for all tx types at parameter named tx_type"
         " of type int",
         fork_attribute_name="tx_types",
-        parameter_names=["tx_type"],
+        argnames=["tx_type"],
     ),
     covariant_decorator(
         marker_name="with_all_contract_creating_tx_types",
         description="marks a test to be parametrized for all tx types that can create a contract"
         " at parameter named tx_type of type int",
         fork_attribute_name="contract_creating_tx_types",
-        parameter_names=["tx_type"],
+        argnames=["tx_type"],
     ),
     covariant_decorator(
         marker_name="with_all_precompiles",
         description="marks a test to be parametrized for all precompiles at parameter named"
         " precompile of type int",
         fork_attribute_name="precompiles",
-        parameter_names=["precompile"],
+        argnames=["precompile"],
     ),
     covariant_decorator(
         marker_name="with_all_evm_code_types",
         description="marks a test to be parametrized for all EVM code types at parameter named"
         " `evm_code_type` of type `EVMCodeType`, such as `LEGACY` and `EOF_V1`",
         fork_attribute_name="evm_code_types",
-        parameter_names=["evm_code_type"],
+        argnames=["evm_code_type"],
     ),
     covariant_decorator(
         marker_name="with_all_call_opcodes",
         description="marks a test to be parametrized for all *CALL opcodes at parameter named"
         " call_opcode, and also the appropriate EVM code type at parameter named evm_code_type",
         fork_attribute_name="call_opcodes",
-        parameter_names=["call_opcode", "evm_code_type"],
+        argnames=["call_opcode", "evm_code_type"],
     ),
     covariant_decorator(
         marker_name="with_all_create_opcodes",
         description="marks a test to be parametrized for all *CREATE* opcodes at parameter named"
         " create_opcode, and also the appropriate EVM code type at parameter named evm_code_type",
         fork_attribute_name="create_opcodes",
-        parameter_names=["create_opcode", "evm_code_type"],
+        argnames=["create_opcode", "evm_code_type"],
     ),
     covariant_decorator(
         marker_name="with_all_system_contracts",
         description="marks a test to be parametrized for all system contracts at parameter named"
         " system_contract of type int",
         fork_attribute_name="system_contracts",
-        parameter_names=["system_contract"],
+        argnames=["system_contract"],
     ),
 ]
 
 
-FORK_COVARIANT_PARAMETRIZE_ATTRIBUTE = "fork_covariant_parametrize"
-
-
-def fork_covariant_parametrize(
-    *,
-    parameter_names: List[str] | str,
-    fn: Callable[[Fork], List[Any] | Iterable[Any]],
-    marks: None
-    | pytest.Mark
-    | pytest.MarkDecorator
-    | List[pytest.Mark | pytest.MarkDecorator] = None,
-):
-    """
-    Decorate a test function by fork-covariant parameters.
-
-    The decorated function will be parametrized by the values returned by the `fn` function
-    for each fork.
-
-    If the parameters that are being parametrized is only a single parameter, the return value
-    of `fn` should be a list of values for that parameter.
-
-    If the parameters that are being parametrized are multiple, the return value of `fn` should
-    be a list of tuples/lists, where each tuple contains the values for each parameter.
-
-    Args:
-        parameter_names: Names of the parameters to be parametrized and that are covariant with
-            the fork.
-        fn: Function that takes the fork as the single parameter and returns the values for
-            the parameter for each fork.
-        marks: List of pytest marks to apply to all test cases parametrized.
-
-    """
-
-    def decorator(decorated_function: FunctionType) -> FunctionType:
-        """Decorate a test function by covariant parameters."""
-        covariant_descriptor = CovariantDescriptor(
-            parameter_names=parameter_names,
-            fn=fn,
-            marks=marks,
-        )
-        covariant_descriptors: List[CovariantDescriptor] = getattr(
-            decorated_function, FORK_COVARIANT_PARAMETRIZE_ATTRIBUTE, []
-        )
-        covariant_descriptors.append(covariant_descriptor)
-        setattr(decorated_function, FORK_COVARIANT_PARAMETRIZE_ATTRIBUTE, covariant_descriptors)
-        return decorated_function
-
-    return decorator
-
-
 @pytest.hookimpl(tryfirst=True)
 def pytest_configure(config: pytest.Config):
     """
@@ -749,11 +701,12 @@ def add_fork_covariant_parameters(
         for fork_parametrizer in fork_parametrizers:
             covariant_descriptor(metafunc=metafunc).add_values(fork_parametrizer=fork_parametrizer)
 
-    if hasattr(metafunc.function, FORK_COVARIANT_PARAMETRIZE_ATTRIBUTE):
-        covariant_descriptors: List[CovariantDescriptor] = getattr(
-            metafunc.function, FORK_COVARIANT_PARAMETRIZE_ATTRIBUTE
-        )
-        for descriptor in covariant_descriptors:
+    for marker in metafunc.definition.iter_markers():
+        if marker.name == "parametrize_by_fork":
+            descriptor = CovariantDescriptor(
+                *marker.args,
+                **marker.kwargs,
+            )
             for fork_parametrizer in fork_parametrizers:
                 descriptor.add_values(fork_parametrizer=fork_parametrizer)
 
@@ -767,10 +720,10 @@ def parameters_from_fork_parametrizer_list(
 
     for fork_parametrizer in fork_parametrizers:
         if not param_names:
-            param_names = fork_parametrizer.parameter_names
+            param_names = fork_parametrizer.argnames
         else:
-            assert param_names == fork_parametrizer.parameter_names
-        param_values.extend(fork_parametrizer.parameter_values)
+            assert param_names == fork_parametrizer.argnames
+        param_values.extend(fork_parametrizer.argvalues)
 
     # Remove duplicate parameters
     param_1 = 0

src/pytest_plugins/forks/tests/test_covariant_markers.py

@@ -288,12 +288,13 @@ def test_case(state_test_only, tx_type):
             """
             import pytest
 
-            from pytest_plugins import fork_covariant_parametrize
-
             def covariant_function(fork):
                 return [1, 2] if fork.name() == "Paris" else [3, 4, 5]
 
-            @fork_covariant_parametrize(parameter_names=["test_parameter"], fn=covariant_function)
+            @pytest.mark.parametrize_by_fork(
+                argnames=["test_parameter"],
+                fn=covariant_function,
+            )
             @pytest.mark.valid_from("Paris")
             @pytest.mark.valid_until("Shanghai")
             def test_case(state_test_only, test_parameter):
@@ -307,14 +308,10 @@ def test_case(state_test_only, test_parameter):
             """
             import pytest
 
-            from pytest_plugins import fork_covariant_parametrize
-
             def covariant_function(fork):
                 return [[1, 2], [3, 4]] if fork.name() == "Paris" else [[4, 5], [5, 6], [6, 7]]
 
-            @fork_covariant_parametrize(parameter_names=[
-                "test_parameter", "test_parameter_2"
-            ], fn=covariant_function)
+            @pytest.mark.parametrize_by_fork("test_parameter,test_parameter_2", covariant_function)
             @pytest.mark.valid_from("Paris")
             @pytest.mark.valid_until("Shanghai")
             def test_case(state_test_only, test_parameter, test_parameter_2):
@@ -328,8 +325,6 @@ def test_case(state_test_only, test_parameter, test_parameter_2):
             """
             import pytest
 
-            from pytest_plugins import fork_covariant_parametrize
-
             def covariant_function(fork):
                 return [
                     pytest.param(1, id="first_value"),
@@ -340,7 +335,7 @@ def covariant_function(fork):
                     5,
                 ]
 
-            @fork_covariant_parametrize(parameter_names=["test_parameter"], fn=covariant_function)
+            @pytest.mark.parametrize_by_fork("test_parameter",covariant_function)
             @pytest.mark.valid_from("Paris")
             @pytest.mark.valid_until("Shanghai")
             def test_case(state_test_only, test_parameter):
@@ -354,8 +349,6 @@ def test_case(state_test_only, test_parameter):
             """
             import pytest
 
-            from pytest_plugins import fork_covariant_parametrize
-
             def covariant_function(fork):
                 return [
                     pytest.param(1, 2, id="first_test"),
@@ -366,7 +359,7 @@ def covariant_function(fork):
                     pytest.param(6, 7, id="sixth_test"),
                 ]
 
-            @fork_covariant_parametrize(parameter_names=[
+            @pytest.mark.parametrize_by_fork(argnames=[
                 "test_parameter", "test_parameter_2"
             ], fn=covariant_function)
             @pytest.mark.valid_from("Paris")

src/pytest_plugins/forks/tests/test_fork_parametrizer_types.py

@@ -189,8 +189,8 @@ def test_fork_parametrizer(
     expected_parameter_sets: List[ParameterSet],
 ):
     """Test that the fork parametrizer correctly parametrizes tests based on the fork name."""
-    parameter_names, values = parameters_from_fork_parametrizer_list(fork_parametrizers)
-    assert parameter_names == expected_names
+    argnames, values = parameters_from_fork_parametrizer_list(fork_parametrizers)
+    assert argnames == expected_names
     assert len(values) == len(expected_parameter_sets)
     for i in range(len(values)):
         assert len(values[i].values) == len(expected_parameter_sets[i].values)