From 518fbee2771433cc6efe3f2d0a85f72a6788e72b Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Thu, 9 Jan 2025 10:38:59 -0600 Subject: [PATCH] fix(plugins/forks): Make `fork_covariant_parametrize` into a marker (#1057) * fix(plugins/forks): Make `fork_covariant_parametrize` into a marker * fix: changelog * Update docs/writing_tests/test_markers.md Co-authored-by: danceratopz * refactor(plugins/forks): rename to `parametrize_by_fork` and argument names --------- Co-authored-by: danceratopz --- docs/CHANGELOG.md | 2 +- docs/writing_tests/test_markers.md | 27 ++-- src/pytest_plugins/__init__.py | 4 - src/pytest_plugins/forks/__init__.py | 4 - src/pytest_plugins/forks/forks.py | 117 ++++++------------ .../forks/tests/test_covariant_markers.py | 21 ++-- .../tests/test_fork_parametrizer_types.py | 4 +- 7 files changed, 61 insertions(+), 118 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 4c5af439b9c..4e703fe1a37 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -64,7 +64,7 @@ Test fixtures for use by clients are available for each release on the [Github r - ✨ Generate Transaction Test type ([#933](https://github.com/ethereum/execution-spec-tests/pull/933)). - ✨ Add a default location for evm logs (`--evm-dump-dir`) when filling tests ([#999](https://github.com/ethereum/execution-spec-tests/pull/999)). - ✨ Slow tests now have greater timeout when making a request to the T8N server ([#1037](https://github.com/ethereum/execution-spec-tests/pull/1037)). -- ✨ Introduce [`fork_covariant_parametrize`](https://ethereum.github.io/execution-spec-tests/main/writing_tests/test_markers/#custom-fork-covariant-markers) helper function ([#1019](https://github.com/ethereum/execution-spec-tests/pull/1019)). +- ✨ Introduce [`pytest.mark.parametrize_by_fork`](https://ethereum.github.io/execution-spec-tests/main/writing_tests/test_markers/#pytestmarkfork_parametrize) helper marker ([#1019](https://github.com/ethereum/execution-spec-tests/pull/1019), [#1057](https://github.com/ethereum/execution-spec-tests/pull/1057)). - 🔀 Update EIP-7251 according to [spec updates](https://github.com/ethereum/EIPs/pull/9127) ([#1024](https://github.com/ethereum/execution-spec-tests/pull/1024)). - 🔀 Update EIP-7002 according to [spec updates](https://github.com/ethereum/EIPs/pull/9119) ([#1024](https://github.com/ethereum/execution-spec-tests/pull/1024)). - 🐞 fix(consume): allow absolute paths with `--evm-bin` ([#1052](https://github.com/ethereum/execution-spec-tests/pull/1052)). diff --git a/docs/writing_tests/test_markers.md b/docs/writing_tests/test_markers.md index 3fcbadf8659..ca5847283ac 100644 --- a/docs/writing_tests/test_markers.md +++ b/docs/writing_tests/test_markers.md @@ -271,27 +271,32 @@ def test_something_with_all_tx_types_but_skip_type_1(state_test_only, tx_type): In this example, the test will be skipped if `tx_type` is equal to 1 by returning a `pytest.mark.skip` marker, and return `None` otherwise. -## Custom Fork Covariant Markers +## `@pytest.mark.parametrize_by_fork` -Custom fork covariant markers can be created by using the `fork_covariant_parametrize` decorator. +A test can be dynamically parametrized based on the fork using the `parametrize_by_fork` marker. -This decorator takes three arguments: +This marker takes two positional arguments: -- `parameter_names`: A list of parameter names that will be parametrized using the custom function. -- `fn`: A function that takes the fork as parameter and returns a list of values that will be used to parametrize the test. -- `marks`: A marker, list of markers, or a lambda function that can be used to add additional markers to the test. +- `argnames`: A list of parameter names that will be parametrized using the custom function. +- `fn`: A function that takes the fork as parameter and returns a list of values that will be used to parametrize the test at that specific fork. + +And one keyword argument: + +- `marks` (optional): A marker, list of markers, or a lambda function that can be used to add additional markers to the generated tests. + +The marked test function will be parametrized by the values returned by the `fn` function for each fork. + +If the parameters that are being parametrized is only a single parameter, the return value of `fn` should be a list of values for that parameter. + +If the parameters that are being parametrized are multiple, the return value of `fn` should be a list of tuples/lists, where each tuple contains the values for each parameter. ```python import pytest -from pytest_plugins import fork_covariant_parametrize - def covariant_function(fork): return [[1, 2], [3, 4]] if fork.name() == "Paris" else [[4, 5], [5, 6], [6, 7]] -@fork_covariant_parametrize(parameter_names=[ - "test_parameter", "test_parameter_2" -], fn=covariant_function) +@pytest.mark.parametrize_by_fork("test_parameter,test_parameter_2", covariant_function) @pytest.mark.valid_from("Paris") @pytest.mark.valid_until("Shanghai") def test_case(state_test_only, test_parameter, test_parameter_2): diff --git a/src/pytest_plugins/__init__.py b/src/pytest_plugins/__init__.py index 2b3dc4ba47b..f3c4ed7eb6c 100644 --- a/src/pytest_plugins/__init__.py +++ b/src/pytest_plugins/__init__.py @@ -1,5 +1 @@ """Package containing pytest plugins related to test filling.""" - -from .forks import fork_covariant_parametrize - -__all__ = ["fork_covariant_parametrize"] diff --git a/src/pytest_plugins/forks/__init__.py b/src/pytest_plugins/forks/__init__.py index eb404537ff8..7ce63ea6273 100644 --- a/src/pytest_plugins/forks/__init__.py +++ b/src/pytest_plugins/forks/__init__.py @@ -3,7 +3,3 @@ tests based on the user-provided fork range the tests' specified validity markers. """ - -from .forks import fork_covariant_parametrize - -__all__ = ["fork_covariant_parametrize"] diff --git a/src/pytest_plugins/forks/forks.py b/src/pytest_plugins/forks/forks.py index 14bbffcbffd..d319c65f398 100644 --- a/src/pytest_plugins/forks/forks.py +++ b/src/pytest_plugins/forks/forks.py @@ -105,15 +105,15 @@ def __init__( self.fork = fork @property - def parameter_names(self) -> List[str]: + def argnames(self) -> List[str]: """Return the parameter names for the test case.""" - parameter_names = [] + argnames = [] for p in self.fork_covariant_parameters: - parameter_names.extend(p.names) - return parameter_names + argnames.extend(p.names) + return argnames @property - def parameter_values(self) -> List[ParameterSet]: + def argvalues(self) -> List[ParameterSet]: """Return the parameter values for the test case.""" parameter_set_combinations = itertools.product( # Add the values for each parameter, all of them are lists of at least one element. @@ -146,7 +146,7 @@ class CovariantDescriptor: the parametrized values change depending on the fork. """ - parameter_names: List[str] = [] + argnames: List[str] = [] fn: Callable[[Fork], List[Any] | Iterable[Any]] | None = None selector: FunctionType | None = None @@ -156,8 +156,9 @@ class CovariantDescriptor: def __init__( self, - parameter_names: List[str] | str, + argnames: List[str] | str, fn: Callable[[Fork], List[Any] | Iterable[Any]] | None = None, + *, selector: FunctionType | None = None, marks: None | pytest.Mark @@ -168,17 +169,18 @@ def __init__( Initialize a new covariant descriptor. Args: - parameter_names: The names of the parameters that are covariant with the fork. + argnames: The names of the parameters that are covariant with the fork. fn: A function that takes the fork as the single parameter and returns the values for the parameter for each fork. selector: A function that filters the values for the parameter. marks: A list of pytest marks to apply to the test cases parametrized by the parameter. """ - if isinstance(parameter_names, str): - self.parameter_names = parameter_names.split(",") - else: - self.parameter_names = parameter_names + self.argnames = ( + [argname.strip() for argname in argnames.split(",")] + if isinstance(argnames, str) + else argnames + ) self.fn = fn self.selector = selector self.marks = marks @@ -195,7 +197,7 @@ def process_value( if isinstance(parameters_values, ParameterSet): return parameters_values - if len(self.parameter_names) == 1: + if len(self.argnames) == 1: # Wrap values that are meant for a single parameter in a list parameters_values = [parameters_values] marks = self.marks @@ -237,7 +239,7 @@ def add_values(self, fork_parametrizer: ForkParametrizer) -> None: values = self.process_values(values) assert len(values) > 0 fork_parametrizer.fork_covariant_parameters.append( - ForkCovariantParameter(names=self.parameter_names, values=values) + ForkCovariantParameter(names=self.argnames, values=values) ) @@ -300,7 +302,7 @@ def fn(fork: Fork) -> List[Any]: return getattr(fork, self.fork_attribute_name)(block_number=0, timestamp=0) super().__init__( - parameter_names=self.marker_parameter_names, + argnames=self.marker_parameter_names, fn=fn, selector=selector, marks=marks, @@ -312,7 +314,7 @@ def covariant_decorator( marker_name: str, description: str, fork_attribute_name: str, - parameter_names: List[str], + argnames: List[str], ) -> Type[CovariantDecorator]: """Generate a new covariant decorator subclass.""" return type( @@ -322,7 +324,7 @@ def covariant_decorator( "marker_name": marker_name, "description": description, "fork_attribute_name": fork_attribute_name, - "marker_parameter_names": parameter_names, + "marker_parameter_names": argnames, }, ) @@ -333,103 +335,53 @@ def covariant_decorator( description="marks a test to be parametrized for all tx types at parameter named tx_type" " of type int", fork_attribute_name="tx_types", - parameter_names=["tx_type"], + argnames=["tx_type"], ), covariant_decorator( marker_name="with_all_contract_creating_tx_types", description="marks a test to be parametrized for all tx types that can create a contract" " at parameter named tx_type of type int", fork_attribute_name="contract_creating_tx_types", - parameter_names=["tx_type"], + argnames=["tx_type"], ), covariant_decorator( marker_name="with_all_precompiles", description="marks a test to be parametrized for all precompiles at parameter named" " precompile of type int", fork_attribute_name="precompiles", - parameter_names=["precompile"], + argnames=["precompile"], ), covariant_decorator( marker_name="with_all_evm_code_types", description="marks a test to be parametrized for all EVM code types at parameter named" " `evm_code_type` of type `EVMCodeType`, such as `LEGACY` and `EOF_V1`", fork_attribute_name="evm_code_types", - parameter_names=["evm_code_type"], + argnames=["evm_code_type"], ), covariant_decorator( marker_name="with_all_call_opcodes", description="marks a test to be parametrized for all *CALL opcodes at parameter named" " call_opcode, and also the appropriate EVM code type at parameter named evm_code_type", fork_attribute_name="call_opcodes", - parameter_names=["call_opcode", "evm_code_type"], + argnames=["call_opcode", "evm_code_type"], ), covariant_decorator( marker_name="with_all_create_opcodes", description="marks a test to be parametrized for all *CREATE* opcodes at parameter named" " create_opcode, and also the appropriate EVM code type at parameter named evm_code_type", fork_attribute_name="create_opcodes", - parameter_names=["create_opcode", "evm_code_type"], + argnames=["create_opcode", "evm_code_type"], ), covariant_decorator( marker_name="with_all_system_contracts", description="marks a test to be parametrized for all system contracts at parameter named" " system_contract of type int", fork_attribute_name="system_contracts", - parameter_names=["system_contract"], + argnames=["system_contract"], ), ] -FORK_COVARIANT_PARAMETRIZE_ATTRIBUTE = "fork_covariant_parametrize" - - -def fork_covariant_parametrize( - *, - parameter_names: List[str] | str, - fn: Callable[[Fork], List[Any] | Iterable[Any]], - marks: None - | pytest.Mark - | pytest.MarkDecorator - | List[pytest.Mark | pytest.MarkDecorator] = None, -): - """ - Decorate a test function by fork-covariant parameters. - - The decorated function will be parametrized by the values returned by the `fn` function - for each fork. - - If the parameters that are being parametrized is only a single parameter, the return value - of `fn` should be a list of values for that parameter. - - If the parameters that are being parametrized are multiple, the return value of `fn` should - be a list of tuples/lists, where each tuple contains the values for each parameter. - - Args: - parameter_names: Names of the parameters to be parametrized and that are covariant with - the fork. - fn: Function that takes the fork as the single parameter and returns the values for - the parameter for each fork. - marks: List of pytest marks to apply to all test cases parametrized. - - """ - - def decorator(decorated_function: FunctionType) -> FunctionType: - """Decorate a test function by covariant parameters.""" - covariant_descriptor = CovariantDescriptor( - parameter_names=parameter_names, - fn=fn, - marks=marks, - ) - covariant_descriptors: List[CovariantDescriptor] = getattr( - decorated_function, FORK_COVARIANT_PARAMETRIZE_ATTRIBUTE, [] - ) - covariant_descriptors.append(covariant_descriptor) - setattr(decorated_function, FORK_COVARIANT_PARAMETRIZE_ATTRIBUTE, covariant_descriptors) - return decorated_function - - return decorator - - @pytest.hookimpl(tryfirst=True) def pytest_configure(config: pytest.Config): """ @@ -749,11 +701,12 @@ def add_fork_covariant_parameters( for fork_parametrizer in fork_parametrizers: covariant_descriptor(metafunc=metafunc).add_values(fork_parametrizer=fork_parametrizer) - if hasattr(metafunc.function, FORK_COVARIANT_PARAMETRIZE_ATTRIBUTE): - covariant_descriptors: List[CovariantDescriptor] = getattr( - metafunc.function, FORK_COVARIANT_PARAMETRIZE_ATTRIBUTE - ) - for descriptor in covariant_descriptors: + for marker in metafunc.definition.iter_markers(): + if marker.name == "parametrize_by_fork": + descriptor = CovariantDescriptor( + *marker.args, + **marker.kwargs, + ) for fork_parametrizer in fork_parametrizers: descriptor.add_values(fork_parametrizer=fork_parametrizer) @@ -767,10 +720,10 @@ def parameters_from_fork_parametrizer_list( for fork_parametrizer in fork_parametrizers: if not param_names: - param_names = fork_parametrizer.parameter_names + param_names = fork_parametrizer.argnames else: - assert param_names == fork_parametrizer.parameter_names - param_values.extend(fork_parametrizer.parameter_values) + assert param_names == fork_parametrizer.argnames + param_values.extend(fork_parametrizer.argvalues) # Remove duplicate parameters param_1 = 0 diff --git a/src/pytest_plugins/forks/tests/test_covariant_markers.py b/src/pytest_plugins/forks/tests/test_covariant_markers.py index 44448f12016..0230abd3d44 100644 --- a/src/pytest_plugins/forks/tests/test_covariant_markers.py +++ b/src/pytest_plugins/forks/tests/test_covariant_markers.py @@ -288,12 +288,13 @@ def test_case(state_test_only, tx_type): """ import pytest - from pytest_plugins import fork_covariant_parametrize - def covariant_function(fork): return [1, 2] if fork.name() == "Paris" else [3, 4, 5] - @fork_covariant_parametrize(parameter_names=["test_parameter"], fn=covariant_function) + @pytest.mark.parametrize_by_fork( + argnames=["test_parameter"], + fn=covariant_function, + ) @pytest.mark.valid_from("Paris") @pytest.mark.valid_until("Shanghai") def test_case(state_test_only, test_parameter): @@ -307,14 +308,10 @@ def test_case(state_test_only, test_parameter): """ import pytest - from pytest_plugins import fork_covariant_parametrize - def covariant_function(fork): return [[1, 2], [3, 4]] if fork.name() == "Paris" else [[4, 5], [5, 6], [6, 7]] - @fork_covariant_parametrize(parameter_names=[ - "test_parameter", "test_parameter_2" - ], fn=covariant_function) + @pytest.mark.parametrize_by_fork("test_parameter,test_parameter_2", covariant_function) @pytest.mark.valid_from("Paris") @pytest.mark.valid_until("Shanghai") def test_case(state_test_only, test_parameter, test_parameter_2): @@ -328,8 +325,6 @@ def test_case(state_test_only, test_parameter, test_parameter_2): """ import pytest - from pytest_plugins import fork_covariant_parametrize - def covariant_function(fork): return [ pytest.param(1, id="first_value"), @@ -340,7 +335,7 @@ def covariant_function(fork): 5, ] - @fork_covariant_parametrize(parameter_names=["test_parameter"], fn=covariant_function) + @pytest.mark.parametrize_by_fork("test_parameter",covariant_function) @pytest.mark.valid_from("Paris") @pytest.mark.valid_until("Shanghai") def test_case(state_test_only, test_parameter): @@ -354,8 +349,6 @@ def test_case(state_test_only, test_parameter): """ import pytest - from pytest_plugins import fork_covariant_parametrize - def covariant_function(fork): return [ pytest.param(1, 2, id="first_test"), @@ -366,7 +359,7 @@ def covariant_function(fork): pytest.param(6, 7, id="sixth_test"), ] - @fork_covariant_parametrize(parameter_names=[ + @pytest.mark.parametrize_by_fork(argnames=[ "test_parameter", "test_parameter_2" ], fn=covariant_function) @pytest.mark.valid_from("Paris") diff --git a/src/pytest_plugins/forks/tests/test_fork_parametrizer_types.py b/src/pytest_plugins/forks/tests/test_fork_parametrizer_types.py index 77292c53ee0..e655b46605a 100644 --- a/src/pytest_plugins/forks/tests/test_fork_parametrizer_types.py +++ b/src/pytest_plugins/forks/tests/test_fork_parametrizer_types.py @@ -189,8 +189,8 @@ def test_fork_parametrizer( expected_parameter_sets: List[ParameterSet], ): """Test that the fork parametrizer correctly parametrizes tests based on the fork name.""" - parameter_names, values = parameters_from_fork_parametrizer_list(fork_parametrizers) - assert parameter_names == expected_names + argnames, values = parameters_from_fork_parametrizer_list(fork_parametrizers) + assert argnames == expected_names assert len(values) == len(expected_parameter_sets) for i in range(len(values)): assert len(values[i].values) == len(expected_parameter_sets[i].values)