fix(tests): EOF - Remove duplicate container tests, automatically che…

…ck for duplicates (#800)

* feat(specs): Check EOF test duplicates

* fix(specs): Skip dupplicates on EOFStateTest automatically

* fix(tests): EOF - remove duplicates

* fix(specs): tests

src/ethereum_test_specs/base.py

@@ -9,6 +9,7 @@
 from pathlib import Path
 from typing import Callable, ClassVar, Dict, Generator, Iterator, List, Optional
 
+import pytest
 from pydantic import BaseModel, Field
 
 from ethereum_test_base_types import to_hex
@@ -78,6 +79,7 @@ class BaseTest(BaseModel):
     def generate(
         self,
         *,
+        request: pytest.FixtureRequest,
         t8n: TransitionTool,
         fork: Fork,
         fixture_format: FixtureFormats,

src/ethereum_test_specs/blockchain.py

@@ -5,6 +5,7 @@
 from pprint import pprint
 from typing import Any, Callable, ClassVar, Dict, Generator, List, Optional, Tuple, Type
 
+import pytest
 from pydantic import ConfigDict, Field, field_validator
 
 from ethereum_test_base_types import (
@@ -714,6 +715,7 @@ def make_hive_fixture(
 
     def generate(
         self,
+        request: pytest.FixtureRequest,
         t8n: TransitionTool,
         fork: Fork,
         fixture_format: FixtureFormats,

src/ethereum_test_specs/eof.py

@@ -7,8 +7,9 @@
 from pathlib import Path
 from shutil import which
 from subprocess import CompletedProcess
-from typing import Any, Callable, ClassVar, Generator, List, Optional, Type
+from typing import Any, Callable, ClassVar, Dict, Generator, List, Optional, Type
 
+import pytest
 from pydantic import Field, model_validator
 
 from ethereum_test_base_types import Account, Bytes
@@ -24,6 +25,8 @@
 from .base import BaseTest
 from .state import StateTest
 
+existing_tests: Dict[Bytes, str] = {}
+
 
 class EOFBaseException(Exception):
     """
@@ -186,12 +189,18 @@ def pytest_parameter_name(cls) -> str:
     def make_eof_test_fixture(
         self,
         *,
+        request: pytest.FixtureRequest,
         fork: Fork,
         eips: Optional[List[int]],
     ) -> Fixture:
         """
         Generate the EOF test fixture.
         """
+        if self.data in existing_tests:
+            pytest.fail(
+                f"Duplicate EOF test: {self.data}, existing test: {existing_tests[self.data]}"
+            )
+        existing_tests[self.data] = request.node.nodeid
         vectors = [
             Vector(
                 code=self.data,
@@ -259,6 +268,7 @@ def verify_result(self, result: CompletedProcess, expected_result: Result, code:
     def generate(
         self,
         *,
+        request: pytest.FixtureRequest,
         t8n: TransitionTool,
         fork: Fork,
         eips: Optional[List[int]] = None,
@@ -269,7 +279,7 @@ def generate(
         Generate the BlockchainTest fixture.
         """
         if fixture_format == FixtureFormats.EOF_TEST:
-            return self.make_eof_test_fixture(fork=fork, eips=eips)
+            return self.make_eof_test_fixture(request=request, fork=fork, eips=eips)
 
         raise Exception(f"Unknown fixture format: {fixture_format}")
 
@@ -358,6 +368,7 @@ def generate_state_test(self) -> StateTest:
     def generate(
         self,
         *,
+        request: pytest.FixtureRequest,
         t8n: TransitionTool,
         fork: Fork,
         eips: Optional[List[int]] = None,
@@ -368,14 +379,18 @@ def generate(
         Generate the BlockchainTest fixture.
         """
         if fixture_format == FixtureFormats.EOF_TEST:
-            return self.make_eof_test_fixture(fork=fork, eips=eips)
+            if self.data in existing_tests:
+                # Gracefully skip duplicate tests because one EOFStateTest can generate multiple
+                # state fixtures with the same data.
+                pytest.skip(f"Duplicate EOF container on EOFStateTest: {request.node.nodeid}")
+            return self.make_eof_test_fixture(request=request, fork=fork, eips=eips)
         elif fixture_format in (
             FixtureFormats.STATE_TEST,
             FixtureFormats.BLOCKCHAIN_TEST,
             FixtureFormats.BLOCKCHAIN_TEST_ENGINE,
         ):
             return self.generate_state_test().generate(
-                t8n=t8n, fork=fork, fixture_format=fixture_format, eips=eips
+                request=request, t8n=t8n, fork=fork, fixture_format=fixture_format, eips=eips
             )
 
         raise Exception(f"Unknown fixture format: {fixture_format}")

src/ethereum_test_specs/state.py

@@ -4,6 +4,8 @@
 
 from typing import Any, Callable, ClassVar, Dict, Generator, List, Optional, Type
 
+import pytest
+
 from ethereum_test_exceptions import EngineAPIError
 from ethereum_test_fixtures import BaseFixture, FixtureFormats
 from ethereum_test_fixtures.state import (
@@ -161,6 +163,7 @@ def make_state_test_fixture(
 
     def generate(
         self,
+        request: pytest.FixtureRequest,
         t8n: TransitionTool,
         fork: Fork,
         fixture_format: FixtureFormats,
@@ -171,7 +174,7 @@ def generate(
         """
         if fixture_format in BlockchainTest.supported_fixture_formats:
             return self.generate_blockchain_test().generate(
-                t8n=t8n, fork=fork, fixture_format=fixture_format, eips=eips
+                request=request, t8n=t8n, fork=fork, fixture_format=fixture_format, eips=eips
             )
         elif fixture_format == FixtureFormats.STATE_TEST:
             return self.make_state_test_fixture(t8n, fork, eips)

src/ethereum_test_specs/tests/test_expect.py

@@ -115,7 +115,9 @@ def test_post_storage_value_mismatch(
     Test post state `Account.storage` exceptions during state test fixture generation.
     """
     with pytest.raises(Storage.KeyValueMismatch) as e_info:
-        state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
+        state_test.generate(
+            request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
+        )
     assert e_info.value == expected_exception
 
 
@@ -141,10 +143,14 @@ def test_post_nonce_value_mismatch(pre: Alloc, post: Alloc, state_test, t8n, for
     pre_nonce = pre_account.nonce
     post_nonce = post_account.nonce
     if "nonce" not in post_account.model_fields_set:  # no exception
-        state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
+        state_test.generate(
+            request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
+        )
         return
     with pytest.raises(Account.NonceMismatch) as e_info:
-        state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
+        state_test.generate(
+            request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
+        )
     assert e_info.value == Account.NonceMismatch(
         address=ADDRESS_UNDER_TEST, want=post_nonce, got=pre_nonce
     )
@@ -172,10 +178,14 @@ def test_post_code_value_mismatch(pre: Alloc, post: Alloc, state_test, t8n, fork
     pre_code = pre_account.code
     post_code = post_account.code
     if "code" not in post_account.model_fields_set:  # no exception
-        state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
+        state_test.generate(
+            request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
+        )
         return
     with pytest.raises(Account.CodeMismatch) as e_info:
-        state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
+        state_test.generate(
+            request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
+        )
     assert e_info.value == Account.CodeMismatch(
         address=ADDRESS_UNDER_TEST, want=post_code, got=pre_code
     )
@@ -203,10 +213,14 @@ def test_post_balance_value_mismatch(pre: Alloc, post: Alloc, state_test, t8n, f
     pre_balance = pre_account.balance
     post_balance = post_account.balance
     if "balance" not in post_account.model_fields_set:  # no exception
-        state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
+        state_test.generate(
+            request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
+        )
         return
     with pytest.raises(Account.BalanceMismatch) as e_info:
-        state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
+        state_test.generate(
+            request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
+        )
     assert e_info.value == Account.BalanceMismatch(
         address=ADDRESS_UNDER_TEST, want=post_balance, got=pre_balance
     )
@@ -245,7 +259,11 @@ def test_post_account_mismatch(state_test, t8n, fork, exception_type: Type[Excep
     fixture generation.
     """
     if exception_type is None:
-        state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
+        state_test.generate(
+            request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
+        )
         return
     with pytest.raises(exception_type) as _:
-        state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
+        state_test.generate(
+            request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
+        )

src/ethereum_test_specs/tests/test_fixtures.py

@@ -91,7 +91,12 @@ def test_make_genesis(fork: Fork, hash: bytes):  # noqa: D103
         post={},
         blocks=[],
         tag="some_state_test",
-    ).generate(t8n, fork, fixture_format=FixtureFormats.BLOCKCHAIN_TEST)
+    ).generate(
+        request=None,  # type: ignore
+        t8n=t8n,
+        fork=fork,
+        fixture_format=FixtureFormats.BLOCKCHAIN_TEST,
+    )
     assert isinstance(fixture, BlockchainFixture)
     assert fixture.genesis is not None
 
@@ -154,6 +159,7 @@ def test_fill_state_test(
         tx=tx,
         tag="my_chain_id_test",
     ).generate(
+        request=None,  # type: ignore
         t8n=t8n,
         fork=fork,
         fixture_format=fixture_format,
@@ -486,6 +492,7 @@ def blockchain_test_fixture(  # noqa: D102
             genesis_environment=genesis_environment,
             tag="my_blockchain_test_valid_txs",
         ).generate(
+            request=None,  # type: ignore
             t8n=t8n,
             fork=fork,
             fixture_format=fixture_format,
@@ -868,6 +875,7 @@ def test_fill_blockchain_invalid_txs(fork: Fork, check_hive: bool, expected_json
         blocks=blocks,
         genesis_environment=genesis_environment,
     ).generate(
+        request=None,  # type: ignore
         t8n=t8n,
         fork=fork,
         fixture_format=fixture_format,

src/ethereum_test_tools/tests/test_code.py

@@ -636,6 +636,7 @@ def test_switch(tx_data: bytes, switch_bytecode: bytes, expected_storage: Mappin
         post=post,
     )
     state_test.generate(
+        request=None,  # type: ignore
         t8n=GethTransitionTool(),
         fork=Cancun,
         fixture_format=FixtureFormats.BLOCKCHAIN_TEST,

src/pytest_plugins/filler/filler.py

@@ -823,6 +823,7 @@ def __init__(self, *args, **kwargs):
                     kwargs["pre"] = pre
                 super(BaseTestWrapper, self).__init__(*args, **kwargs)
                 fixture = self.generate(
+                    request=request,
                     t8n=t8n,
                     fork=fork,
                     fixture_format=fixture_format,

tests/prague/eip7692_eof_v1/eip3540_eof_v1/test_container_validation.py

@@ -35,13 +35,6 @@
 @pytest.mark.parametrize(
     "container",
     [
-        Container(
-            name="single_code_section_with_data_section",
-            sections=[
-                Section.Code(code=Op.STOP),
-                Section.Data(data="0x00"),
-            ],
-        ),
         Container(
             name="single_code_section_max_stack_size",
             sections=[
@@ -181,37 +174,30 @@ def test_valid_containers(
         ),
         Container(
             name="incomplete_magic",
-            raw_bytes=bytes([0xEF]),
+            raw_bytes="ef",
             validity_error=EOFException.INVALID_MAGIC,
         ),
         Container(
             name="no_version",
-            raw_bytes=bytes([0xEF, 0x00]),
+            raw_bytes="ef00",
             validity_error=[EOFException.INVALID_VERSION, EOFException.INVALID_MAGIC],
         ),
         Container(
             name="no_type_header",
-            raw_bytes=bytes([0xEF, 0x00, 0x01]),
-            # TODO the exception must be about missing section types
+            raw_bytes="ef00 01",
             validity_error=EOFException.MISSING_HEADERS_TERMINATOR,
         ),
         Container(
             name="no_type_section_size",
-            raw_bytes=bytes(
-                [0xEF, 0x00, 0x01, 0x01],
-            ),
-            # TODO the exception must be about incomplete section in the header
+            raw_bytes="ef00 01 01",
             validity_error=[
                 EOFException.MISSING_HEADERS_TERMINATOR,
                 EOFException.INVALID_TYPE_SECTION_SIZE,
             ],
         ),
         Container(
             name="incomplete_type_section_size",
-            raw_bytes=bytes(
-                [0xEF, 0x00, 0x01, 0x01, 0x00],
-            ),
-            # TODO the exception must be about incomplete section in the header
+            raw_bytes="ef00010100",
             validity_error=[
                 EOFException.INCOMPLETE_SECTION_SIZE,
                 EOFException.INVALID_TYPE_SECTION_SIZE,
@@ -242,7 +228,7 @@ def test_valid_containers(
         ),
         Container(
             name="code_section_count_incomplete",
-            raw_bytes=bytes([0xEF, 0x00, 0x01, 0x01, 0x00, 0x04, 0x02, 0x00]),
+            raw_bytes="ef00 01 01 0004 02 00",
             validity_error=EOFException.INCOMPLETE_SECTION_NUMBER,
         ),
         Container(
@@ -255,7 +241,7 @@ def test_valid_containers(
         ),
         Container(
             name="code_section_size_incomplete",
-            raw_bytes=bytes([0xEF, 0x00, 0x01, 0x01, 0x00, 0x04, 0x02, 0x00, 0x01, 0x00]),
+            raw_bytes="ef00 01 01 0004 02 0001 00",
             validity_error=[EOFException.INCOMPLETE_SECTION_SIZE, EOFException.ZERO_SECTION_SIZE],
         ),
         Container(
@@ -1059,21 +1045,6 @@ def test_valid_containers(
             ],
             validity_error=EOFException.INPUTS_OUTPUTS_NUM_ABOVE_LIMIT,
         ),
-        Container(
-            name="code_section_output_too_large_2",
-            sections=[
-                Section.Code(
-                    code=Op.JUMPF[1],
-                ),
-                Section.Code(
-                    code=(Op.PUSH0 * (MAX_CODE_OUTPUTS + 1)) + Op.RETF,
-                    code_inputs=0,
-                    code_outputs=(MAX_CODE_OUTPUTS + 1),
-                    max_stack_height=(MAX_CODE_OUTPUTS + 1),
-                ),
-            ],
-            validity_error=EOFException.INVALID_NON_RETURNING_FLAG,
-        ),
         Container(
             name="single_code_section_max_stack_size_too_large",
             sections=[
@@ -1113,6 +1084,8 @@ def test_magic_validation(
     """
     Verify EOF container 2-byte magic
     """
+    if magic_0 == 0xEF and magic_1 == 0:
+        pytest.skip("Valid magic")
     code = bytearray(bytes(VALID_CONTAINER))
     code[0] = magic_0
     code[1] = magic_1
@@ -1130,6 +1103,8 @@ def test_version_validation(
     """
     Verify EOF container version
     """
+    if version == 1:
+        pytest.skip("Valid version")
     code = bytearray(bytes(VALID_CONTAINER))
     code[2] = version
     eof_test(