From 31273b7bac8ad77b03a6d3fa42146c84659a4cf4 Mon Sep 17 00:00:00 2001 From: Dimitry Kh Date: Fri, 22 Nov 2024 13:58:14 +0100 Subject: [PATCH] json schema verification: verify block headers in generated blockchain tests --- src/ethereum_test_fixtures/file.py | 5 +- .../schemas/blockchain/genesis.py | 102 +++++++++++++ .../schemas/blockchain/test.py | 69 +++++++++ .../schemas/common/types.py | 139 ++++++++++++++++++ src/ethereum_test_fixtures/verify_format.py | 30 ++++ whitelist.txt | 6 +- 6 files changed, 349 insertions(+), 2 deletions(-) create mode 100644 src/ethereum_test_fixtures/schemas/blockchain/genesis.py create mode 100644 src/ethereum_test_fixtures/schemas/blockchain/test.py create mode 100644 src/ethereum_test_fixtures/schemas/common/types.py create mode 100644 src/ethereum_test_fixtures/verify_format.py diff --git a/src/ethereum_test_fixtures/file.py b/src/ethereum_test_fixtures/file.py index 7115334948..a11ba9b403 100644 --- a/src/ethereum_test_fixtures/file.py +++ b/src/ethereum_test_fixtures/file.py @@ -13,6 +13,7 @@ from .blockchain import Fixture as BlockchainFixture from .eof import Fixture as EOFFixture from .state import Fixture as StateFixture +from .verify_format import VerifyFixtureJson FixtureModel = BlockchainFixture | BlockchainEngineFixture | StateFixture | EOFFixture @@ -64,7 +65,9 @@ def collect_into_file(self, file_path: Path): """ json_fixtures: Dict[str, Dict[str, Any]] = {} for name, fixture in self.items(): - json_fixtures[name] = fixture.json_dict_with_info() + fixture_json = fixture.json_dict_with_info() + VerifyFixtureJson(name, fixture_json) + json_fixtures[name] = fixture_json with open(file_path, "w") as f: json.dump(json_fixtures, f, indent=4) diff --git a/src/ethereum_test_fixtures/schemas/blockchain/genesis.py b/src/ethereum_test_fixtures/schemas/blockchain/genesis.py new file mode 100644 index 0000000000..d0838cc77a --- /dev/null +++ b/src/ethereum_test_fixtures/schemas/blockchain/genesis.py @@ -0,0 +1,102 @@ +""" +Define genesisHeader schema for filled .json tests +""" + +from pydantic import BaseModel, model_validator + +from ..common.types import ( + DataBytes, + FixedHash8, + FixedHash20, + FixedHash32, + FixedHash256, + PrefixedEvenHex, +) + + +class BlockRecord(BaseModel): + """Block record in blockchain tests""" + + blockHeader: dict # noqa: N815 + rlp: str + transactions: list # noqa: N815 + uncleHeaders: list # noqa: N815 + + +class FrontierHeader(BaseModel): + """Frontier block header in test json""" + + bloom: FixedHash256 + coinbase: FixedHash20 + difficulty: PrefixedEvenHex + extraData: DataBytes # noqa: N815" + gasLimit: PrefixedEvenHex # noqa: N815" + gasUsed: PrefixedEvenHex # noqa: N815" + hash: FixedHash32 + mixHash: FixedHash32 # noqa: N815" + nonce: FixedHash8 + number: PrefixedEvenHex + parentHash: FixedHash32 # noqa: N815" + receiptTrie: FixedHash32 # noqa: N815" + stateRoot: FixedHash32 # noqa: N815" + timestamp: PrefixedEvenHex + transactionsTrie: FixedHash32 # noqa: N815" + uncleHash: FixedHash32 # noqa: N815" + + class Config: + """Forbids any extra fields that are not declared in the model""" + + extra = "forbid" + + +class HomesteadHeader(FrontierHeader): + """Homestead block header in test json""" + + +class ByzantiumHeader(HomesteadHeader): + """Byzantium block header in test json""" + + +class ConstantinopleHeader(ByzantiumHeader): + """Constantinople block header in test json""" + + +class IstanbulHeader(ConstantinopleHeader): + """Istanbul block header in test json""" + + +class BerlinHeader(IstanbulHeader): + """Berlin block header in test json""" + + +class LondonHeader(BerlinHeader): + """London block header in test json""" + + baseFeePerGas: PrefixedEvenHex # noqa: N815 + + +class ParisHeader(LondonHeader): + """Paris block header in test json""" + + @model_validator(mode="after") + def check_block_header(self): + """ + Validate Paris block header rules + """ + + if self.difficulty != "0x00": + raise ValueError("Starting from Paris, block difficulty must be 0x00") + + +class ShanghaiHeader(ParisHeader): + """Shanghai block header in test json""" + + withdrawalsRoot: FixedHash32 # noqa: N815 + + +class CancunHeader(ShanghaiHeader): + """Cancun block header in test json""" + + blobGasUsed: PrefixedEvenHex # noqa: N815 + excessBlobGas: PrefixedEvenHex # noqa: N815 + parentBeaconBlockRoot: FixedHash32 # noqa: N815 diff --git a/src/ethereum_test_fixtures/schemas/blockchain/test.py b/src/ethereum_test_fixtures/schemas/blockchain/test.py new file mode 100644 index 0000000000..6056d107fb --- /dev/null +++ b/src/ethereum_test_fixtures/schemas/blockchain/test.py @@ -0,0 +1,69 @@ +""" +Schema for filled Blockchain Test +""" + +from pydantic import BaseModel, Field, model_validator + +from .genesis import ( + BerlinHeader, + BlockRecord, + ByzantiumHeader, + CancunHeader, + ConstantinopleHeader, + FrontierHeader, + HomesteadHeader, + IstanbulHeader, + LondonHeader, + ParisHeader, + ShanghaiHeader, +) + + +class BlockchainTestFixtureModel(BaseModel): + """ + Blockchain test file + """ + + info: dict = Field(alias="_info") + network: str + genesisBlockHeader: dict # noqa: N815 + pre: dict + postState: dict # noqa: N815 + lastblockhash: str + genesisRLP: str # noqa: N815 + blocks: list[BlockRecord] + sealEngine: str # noqa: N815 + + class Config: + """Forbids any extra fields that are not declared in the model""" + + extra = "forbid" + + @model_validator(mode="after") + def check_block_headers(self): + """ + Validate genesis header fields based by fork + """ + # TODO str to Fork class comparison + allowed_networks = { + "Frontier": FrontierHeader, + "Homestead": HomesteadHeader, + "EIP150": HomesteadHeader, + "EIP158": HomesteadHeader, + "Byzantium": ByzantiumHeader, + "Constantinople": ConstantinopleHeader, + "ConstantinopleFix": ConstantinopleHeader, + "Istanbul": IstanbulHeader, + "Berlin": BerlinHeader, + "London": LondonHeader, + "Paris": ParisHeader, + "Shanghai": ShanghaiHeader, + "Cancun": CancunHeader, + } + + header_class = allowed_networks.get(self.network) + if not header_class: + raise ValueError("Incorrect value in network field: " + self.network) + header_class(**self.genesisBlockHeader) + for block in self.blocks: + header_class(**block.blockHeader) diff --git a/src/ethereum_test_fixtures/schemas/common/types.py b/src/ethereum_test_fixtures/schemas/common/types.py new file mode 100644 index 0000000000..93f50d33d8 --- /dev/null +++ b/src/ethereum_test_fixtures/schemas/common/types.py @@ -0,0 +1,139 @@ +""" +Base types for Pydantic json test fixtures +""" + +import re +from typing import Generic, TypeVar + +from pydantic import RootModel, model_validator + +T = TypeVar("T", bound="FixedHash") + + +class FixedHash(RootModel[str], Generic[T]): + """Base class for fixed-length hashes.""" + + _length_in_bytes: int + + @model_validator(mode="after") + def validate_hex_hash(self): + """ + Validate that the field is a 0x-prefixed hash of specified byte length. + """ + expected_length = 2 + 2 * self._length_in_bytes # 2 for '0x' + 2 hex chars per byte + if not self.root.startswith("0x"): + raise ValueError("The hash must start with '0x'.") + if len(self.root) != expected_length: + raise ValueError( + f"The hash must be {expected_length} characters long " + f"(2 for '0x' and {2 * self._length_in_bytes} hex characters)." + ) + if not re.fullmatch(rf"0x[a-fA-F0-9]{{{2 * self._length_in_bytes}}}", self.root): + raise ValueError( + f"The hash must be a valid hexadecimal string of " + f"{2 * self._length_in_bytes} characters after '0x'." + ) + + +class FixedHash32(FixedHash): + """FixedHash32 type (32 bytes)""" + + _length_in_bytes = 32 + + +class FixedHash20(FixedHash): + """FixedHash20 type (20 bytes)""" + + _length_in_bytes = 20 + + +class FixedHash8(FixedHash): + """FixedHash8 type (8 bytes)""" + + _length_in_bytes = 8 + + +class FixedHash256(FixedHash): + """FixedHash256 type (256 bytes)""" + + _length_in_bytes = 256 + + +class PrefixedEvenHex(RootModel[str]): + """Class to validate a hexadecimal integer encoding in test files.""" + + def __eq__(self, other): + """ + For python str comparison + """ + if isinstance(other, str): + return self.root == other + return NotImplemented + + @model_validator(mode="after") + def validate_hex_integer(self): + """ + Validate that the field is a hexadecimal integer with specific rules: + - Must start with '0x'. + - Must be even in length after '0x'. + - Cannot be '0x0', '0x0000', etc. (minimum is '0x00'). + """ + # Ensure it starts with '0x' + if not self.root.startswith("0x"): + raise ValueError("The value must start with '0x'.") + + # Extract the hex portion (after '0x') + hex_part = self.root[2:] + + # Ensure the length of the hex part is even + if len(hex_part) % 2 != 0: + raise ValueError( + "The hexadecimal value must have an even number of characters after '0x'." + ) + + # Special rule: Only '0x00' is allowed; disallow '0x0000', '0x0001', etc. + if hex_part.startswith("00") and hex_part != "00": + raise ValueError("Leading zeros are not allowed except for '0x00'.") + + # Ensure it's a valid hexadecimal string + if not re.fullmatch(r"[a-fA-F0-9]+", hex_part): + raise ValueError("The value must be a valid hexadecimal string.") + + return self + + +class DataBytes(RootModel[str]): + """Class to validate DataBytes.""" + + @model_validator(mode="after") + def validate_data_bytes(self): + """ + Validate that the field follows the rules for DataBytes: + - Must start with '0x'. + - Can be empty (just '0x'). + - Must be even in length after '0x'. + - Allows prefixed '00' values (e.g., '0x00000001'). + - Must be a valid hexadecimal string. + """ + # Ensure it starts with '0x' + if not self.root.startswith("0x"): + raise ValueError("The value must start with '0x'.") + + # Extract the hex portion (after '0x') + hex_part = self.root[2:] + + # Allow empty '0x' + if len(hex_part) == 0: + return self + + # Ensure the length of the hex part is even + if len(hex_part) % 2 != 0: + raise ValueError( + "The hexadecimal value must have an even number of characters after '0x'." + ) + + # Ensure it's a valid hexadecimal string + if not re.fullmatch(r"[a-fA-F0-9]*", hex_part): # `*` allows empty hex_part for '0x' + raise ValueError("The value must be a valid hexadecimal string.") + + return self diff --git a/src/ethereum_test_fixtures/verify_format.py b/src/ethereum_test_fixtures/verify_format.py new file mode 100644 index 0000000000..b3e6a4be17 --- /dev/null +++ b/src/ethereum_test_fixtures/verify_format.py @@ -0,0 +1,30 @@ +""" +Verify the sanity of fixture .json format +""" + +from pydantic import ValidationError + +from .schemas.blockchain.test import BlockchainTestFixtureModel + + +class VerifyFixtureJson: + """ + Class to verify the correctness of a fixture JSON. + """ + + def __init__(self, name: str, fixture_json: dict): + self.fixture_json = fixture_json + self.fixture_name = name + if self.fixture_json.get("network") and not self.fixture_json.get("engineNewPayloads"): + self.verify_blockchain_fixture_json() + + def verify_blockchain_fixture_json(self): + """ + Function to verify blockchain json fixture + """ + try: + BlockchainTestFixtureModel(**self.fixture_json) + except ValidationError as e: + raise Exception( + f"Error in generated blockchain test json ({self.fixture_name})" + e.json() + ) diff --git a/whitelist.txt b/whitelist.txt index 38e3d344b6..cc92c4273d 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -177,6 +177,7 @@ fp2 fromhex frozenbidict func +fullmatch g1 g1add g1msm @@ -207,7 +208,10 @@ Golang gwei hacky hardfork +hash8 hash32 +hash20 +hash256 Hashable hasher HeaderNonce @@ -810,4 +814,4 @@ E9 EB EF F6 -FC +FC \ No newline at end of file