From 6ca6afc232701cb2e6836d8e80983fa18b0a4e16 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 31 May 2023 01:03:26 +0800 Subject: [PATCH 1/4] Refactor setup.py --- pysetup/__init__.py | 0 pysetup/constants.py | 31 ++ pysetup/helpers.py | 229 +++++++++++ pysetup/md_doc_paths.py | 76 ++++ pysetup/spec_builders.py | 447 +++++++++++++++++++++ pysetup/typing.py | 32 ++ setup.py | 821 ++------------------------------------- 7 files changed, 853 insertions(+), 783 deletions(-) create mode 100644 pysetup/__init__.py create mode 100644 pysetup/constants.py create mode 100644 pysetup/helpers.py create mode 100644 pysetup/md_doc_paths.py create mode 100644 pysetup/spec_builders.py create mode 100644 pysetup/typing.py diff --git a/pysetup/__init__.py b/pysetup/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pysetup/constants.py b/pysetup/constants.py new file mode 100644 index 0000000000..3b0998742f --- /dev/null +++ b/pysetup/constants.py @@ -0,0 +1,31 @@ +# Definitions in context.py +PHASE0 = 'phase0' +ALTAIR = 'altair' +BELLATRIX = 'bellatrix' +CAPELLA = 'capella' +DENEB = 'deneb' +EIP6110 = 'eip6110' + + +# The helper functions that are used when defining constants +CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS = ''' +def ceillog2(x: int) -> uint64: + if x < 1: + raise ValueError(f"ceillog2 accepts only positive values, x={x}") + return uint64((x - 1).bit_length()) + + +def floorlog2(x: int) -> uint64: + if x < 1: + raise ValueError(f"floorlog2 accepts only positive values, x={x}") + return uint64(x.bit_length() - 1) +''' + + +OPTIMIZED_BLS_AGGREGATE_PUBKEYS = ''' +def eth_aggregate_pubkeys(pubkeys: Sequence[BLSPubkey]) -> BLSPubkey: + return bls.AggregatePKs(pubkeys) +''' + + +ETH2_SPEC_COMMENT_PREFIX = "eth2spec:" diff --git a/pysetup/helpers.py b/pysetup/helpers.py new file mode 100644 index 0000000000..bda1006842 --- /dev/null +++ b/pysetup/helpers.py @@ -0,0 +1,229 @@ +from pathlib import Path +import re +from typing import TypeVar, Dict, Sequence +import textwrap + +from .constants import CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS +from .spec_builders import SpecBuilder +from .typing import ( + ProtocolDefinition, + SpecObject, + VariableDefinition, +) + + +def is_byte_vector(value: str) -> bool: + return value.startswith(('ByteVector')) + + +def make_function_abstract(protocol_def: ProtocolDefinition, key: str): + function = protocol_def.functions[key].split('"""') + protocol_def.functions[key] = function[0] + "..." + + +def objects_to_spec(preset_name: str, + spec_object: SpecObject, + builder: SpecBuilder, + ordered_class_objects: Dict[str, str]) -> str: + """ + Given all the objects that constitute a spec, combine them into a single pyfile. + """ + new_type_definitions = ( + '\n\n'.join( + [ + f"class {key}({value}):\n pass\n" if not is_byte_vector(value) else f"class {key}({value}): # type: ignore\n pass\n" + for key, value in spec_object.custom_types.items() + ] + ) + ) + + def format_protocol(protocol_name: str, protocol_def: ProtocolDefinition) -> str: + abstract_functions = ["verify_and_notify_new_payload"] + for key in protocol_def.functions.keys(): + if key in abstract_functions: + make_function_abstract(protocol_def, key) + + protocol = f"class {protocol_name}(Protocol):" + for fn_source in protocol_def.functions.values(): + fn_source = fn_source.replace("self: "+protocol_name, "self") + protocol += "\n\n" + textwrap.indent(fn_source, " ") + return protocol + + protocols_spec = '\n\n\n'.join(format_protocol(k, v) for k, v in spec_object.protocols.items()) + for k in list(spec_object.functions): + if k in [ + "ceillog2", + "floorlog2", + "compute_merkle_proof_for_block_body", + "compute_merkle_proof_for_state", + ]: + del spec_object.functions[k] + functions = builder.implement_optimizations(spec_object.functions) + functions_spec = '\n\n\n'.join(functions.values()) + + # Access global dict of config vars for runtime configurables + for name in spec_object.config_vars.keys(): + functions_spec = re.sub(r"\b%s\b" % name, 'config.' + name, functions_spec) + + def format_config_var(name: str, vardef: VariableDefinition) -> str: + if vardef.type_name is None: + out = f'{name}={vardef.value},' + else: + out = f'{name}={vardef.type_name}({vardef.value}),' + if vardef.comment is not None: + out += f' # {vardef.comment}' + return out + + config_spec = 'class Configuration(NamedTuple):\n' + config_spec += ' PRESET_BASE: str\n' + config_spec += '\n'.join(f' {k}: {v.type_name if v.type_name is not None else "int"}' + for k, v in spec_object.config_vars.items()) + config_spec += '\n\n\nconfig = Configuration(\n' + config_spec += f' PRESET_BASE="{preset_name}",\n' + config_spec += '\n'.join(' ' + format_config_var(k, v) for k, v in spec_object.config_vars.items()) + config_spec += '\n)\n' + + def format_constant(name: str, vardef: VariableDefinition) -> str: + if vardef.type_name is None: + if vardef.type_hint is None: + out = f'{name} = {vardef.value}' + else: + out = f'{name}: {vardef.type_hint} = {vardef.value}' + else: + out = f'{name} = {vardef.type_name}({vardef.value})' + if vardef.comment is not None: + out += f' # {vardef.comment}' + return out + + constant_vars_spec = '# Constant vars\n' + '\n'.join(format_constant(k, v) for k, v in spec_object.constant_vars.items()) + preset_vars_spec = '# Preset vars\n' + '\n'.join(format_constant(k, v) for k, v in spec_object.preset_vars.items()) + ordered_class_objects_spec = '\n\n\n'.join(ordered_class_objects.values()) + ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_ssz_dep_constants()[x]), builder.hardcoded_ssz_dep_constants())) + ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), builder.hardcoded_ssz_dep_constants())) + custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_custom_type_dep_constants(spec_object)[x]), builder.hardcoded_custom_type_dep_constants(spec_object))) + spec = ( + builder.imports(preset_name) + + builder.preparations() + + '\n\n' + f"fork = \'{builder.fork}\'\n" + # The constants that some SSZ containers require. Need to be defined before `new_type_definitions` + + ('\n\n' + custom_type_dep_constants + '\n' if custom_type_dep_constants != '' else '') + + '\n\n' + new_type_definitions + + '\n' + CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS + # The constants that some SSZ containers require. Need to be defined before `constants_spec` + + ('\n\n' + ssz_dep_constants if ssz_dep_constants != '' else '') + + '\n\n' + constant_vars_spec + + '\n\n' + preset_vars_spec + + '\n\n\n' + config_spec + + '\n\n' + ordered_class_objects_spec + + ('\n\n\n' + protocols_spec if protocols_spec != '' else '') + + '\n\n\n' + functions_spec + + '\n\n' + builder.sundry_functions() + + builder.execution_engine_cls() + # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are + # as same as the spec definition. + + ('\n\n\n' + ssz_dep_constants_verification if ssz_dep_constants_verification != '' else '') + + '\n' + ) + return spec + + +def combine_protocols(old_protocols: Dict[str, ProtocolDefinition], + new_protocols: Dict[str, ProtocolDefinition]) -> Dict[str, ProtocolDefinition]: + for key, value in new_protocols.items(): + if key not in old_protocols: + old_protocols[key] = value + else: + functions = combine_dicts(old_protocols[key].functions, value.functions) + old_protocols[key] = ProtocolDefinition(functions=functions) + return old_protocols + + +T = TypeVar('T') + + +def combine_dicts(old_dict: Dict[str, T], new_dict: Dict[str, T]) -> Dict[str, T]: + return {**old_dict, **new_dict} + + +ignored_dependencies = [ + 'bit', 'boolean', 'Vector', 'List', 'Container', 'BLSPubkey', 'BLSSignature', + 'Bytes1', 'Bytes4', 'Bytes8', 'Bytes20', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', + 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', + 'bytes', 'byte', 'ByteList', 'ByteVector', + 'Dict', 'dict', 'field', 'ceillog2', 'floorlog2', 'Set', + 'Optional', 'Sequence', +] + + +def dependency_order_class_objects(objects: Dict[str, str], custom_types: Dict[str, str]) -> None: + """ + Determines which SSZ Object is dependent on which other and orders them appropriately + """ + items = list(objects.items()) + for key, value in items: + dependencies = [] + for line in value.split('\n'): + if not re.match(r'\s+\w+: .+', line): + continue # skip whitespace etc. + line = line[line.index(':') + 1:] # strip of field name + if '#' in line: + line = line[:line.index('#')] # strip of comment + dependencies.extend(re.findall(r'(\w+)', line)) # catch all legible words, potential dependencies + dependencies = filter(lambda x: '_' not in x and x.upper() != x, dependencies) # filter out constants + dependencies = filter(lambda x: x not in ignored_dependencies, dependencies) + dependencies = filter(lambda x: x not in custom_types, dependencies) + for dep in dependencies: + key_list = list(objects.keys()) + for item in [dep, key] + key_list[key_list.index(dep)+1:]: + objects[item] = objects.pop(item) + + +def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str], custom_types) -> Dict[str, str]: + """ + Takes in old spec and new spec ssz objects, combines them, + and returns the newer versions of the objects in dependency order. + """ + for key, value in new_objects.items(): + old_objects[key] = value + return old_objects + + +def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: + """ + Takes in two spec variants (as tuples of their objects) and combines them using the appropriate combiner function. + """ + protocols = combine_protocols(spec0.protocols, spec1.protocols) + functions = combine_dicts(spec0.functions, spec1.functions) + custom_types = combine_dicts(spec0.custom_types, spec1.custom_types) + constant_vars = combine_dicts(spec0.constant_vars, spec1.constant_vars) + preset_vars = combine_dicts(spec0.preset_vars, spec1.preset_vars) + config_vars = combine_dicts(spec0.config_vars, spec1.config_vars) + ssz_dep_constants = combine_dicts(spec0.ssz_dep_constants, spec1.ssz_dep_constants) + ssz_objects = combine_ssz_objects(spec0.ssz_objects, spec1.ssz_objects, custom_types) + dataclasses = combine_dicts(spec0.dataclasses, spec1.dataclasses) + return SpecObject( + functions=functions, + protocols=protocols, + custom_types=custom_types, + constant_vars=constant_vars, + preset_vars=preset_vars, + config_vars=config_vars, + ssz_dep_constants=ssz_dep_constants, + ssz_objects=ssz_objects, + dataclasses=dataclasses, + ) + + +def parse_config_vars(conf: Dict[str, str]) -> Dict[str, str]: + """ + Parses a dict of basic str/int/list types into a dict for insertion into the spec code. + """ + out: Dict[str, str] = dict() + for k, v in conf.items(): + if isinstance(v, str) and (v.startswith("0x") or k == 'PRESET_BASE' or k == 'CONFIG_NAME'): + # Represent byte data with string, to avoid misinterpretation as big-endian int. + # Everything except PRESET_BASE and CONFIG_NAME is either byte data or an integer. + out[k] = f"'{v}'" + else: + out[k] = str(int(v)) + return out diff --git a/pysetup/md_doc_paths.py b/pysetup/md_doc_paths.py new file mode 100644 index 0000000000..6d6273da85 --- /dev/null +++ b/pysetup/md_doc_paths.py @@ -0,0 +1,76 @@ +from .constants import ( + PHASE0, + ALTAIR, + BELLATRIX, + CAPELLA, + DENEB, + EIP6110, +) + + +def get_md_doc_paths(spec_fork: str) -> str: + md_doc_paths = "" + if spec_fork in (PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110): + md_doc_paths += """ + specs/phase0/beacon-chain.md + specs/phase0/fork-choice.md + specs/phase0/validator.md + specs/phase0/weak-subjectivity.md + specs/phase0/p2p-interface.md + """ + if spec_fork in (ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110): + md_doc_paths += """ + specs/altair/light-client/full-node.md + specs/altair/light-client/light-client.md + specs/altair/light-client/p2p-interface.md + specs/altair/light-client/sync-protocol.md + specs/altair/beacon-chain.md + specs/altair/bls.md + specs/altair/fork.md + specs/altair/validator.md + specs/altair/p2p-interface.md + """ + if spec_fork in (BELLATRIX, CAPELLA, DENEB, EIP6110): + md_doc_paths += """ + specs/bellatrix/beacon-chain.md + specs/bellatrix/fork.md + specs/bellatrix/fork-choice.md + specs/bellatrix/validator.md + specs/bellatrix/p2p-interface.md + sync/optimistic.md + """ + if spec_fork in (CAPELLA, DENEB, EIP6110): + md_doc_paths += """ + specs/capella/light-client/fork.md + specs/capella/light-client/full-node.md + specs/capella/light-client/p2p-interface.md + specs/capella/light-client/sync-protocol.md + specs/capella/beacon-chain.md + specs/capella/fork.md + specs/capella/fork-choice.md + specs/capella/validator.md + specs/capella/p2p-interface.md + """ + if spec_fork in (DENEB, EIP6110): + md_doc_paths += """ + specs/deneb/light-client/fork.md + specs/deneb/light-client/full-node.md + specs/deneb/light-client/p2p-interface.md + specs/deneb/light-client/sync-protocol.md + specs/deneb/beacon-chain.md + specs/deneb/fork.md + specs/deneb/fork-choice.md + specs/deneb/polynomial-commitments.md + specs/deneb/p2p-interface.md + specs/deneb/validator.md + """ + if spec_fork == EIP6110: + md_doc_paths += """ + specs/_features/eip6110/light-client/fork.md + specs/_features/eip6110/light-client/full-node.md + specs/_features/eip6110/light-client/p2p-interface.md + specs/_features/eip6110/light-client/sync-protocol.md + specs/_features/eip6110/beacon-chain.md + specs/_features/eip6110/fork.md + """ + return md_doc_paths diff --git a/pysetup/spec_builders.py b/pysetup/spec_builders.py new file mode 100644 index 0000000000..78fa50a1fc --- /dev/null +++ b/pysetup/spec_builders.py @@ -0,0 +1,447 @@ +from abc import ABC, abstractmethod +from typing import Dict, List, Sequence +from pathlib import Path + +from .constants import ( + PHASE0, + ALTAIR, + BELLATRIX, + CAPELLA, + DENEB, + EIP6110, + OPTIMIZED_BLS_AGGREGATE_PUBKEYS, +) + + +class SpecBuilder(ABC): + @property + @abstractmethod + def fork(self) -> str: + raise NotImplementedError() + + @classmethod + @abstractmethod + def imports(cls, preset_name: str) -> str: + """ + Import objects from other libraries. + """ + raise NotImplementedError() + + @classmethod + @abstractmethod + def preparations(cls) -> str: + """ + Define special types/constants for building pyspec or call functions. + """ + raise NotImplementedError() + + @classmethod + @abstractmethod + def sundry_functions(cls) -> str: + """ + The functions that are (1) defined abstractly in specs or (2) adjusted for getting better performance. + """ + raise NotImplementedError() + + @classmethod + def execution_engine_cls(cls) -> str: + raise NotImplementedError() + + @classmethod + @abstractmethod + def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: + """ + The constants that are required for SSZ objects. + """ + raise NotImplementedError() + + @classmethod + @abstractmethod + def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]: # TODO + """ + The constants that are required for custom types. + """ + raise NotImplementedError() + + @classmethod + @abstractmethod + def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]: + raise NotImplementedError() + + @classmethod + @abstractmethod + def build_spec(cls, preset_name: str, + source_files: List[Path], preset_files: Sequence[Path], config_file: Path) -> str: + raise NotImplementedError() + + +# +# Phase0SpecBuilder +# +class Phase0SpecBuilder(SpecBuilder): + fork: str = PHASE0 + + @classmethod + def imports(cls, preset_name: str) -> str: + return '''from lru import LRU +from dataclasses import ( + dataclass, + field, +) +from typing import ( + Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar, NamedTuple, Final +) + +from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes +from eth2spec.utils.ssz.ssz_typing import ( + View, boolean, Container, List, Vector, uint8, uint32, uint64, uint256, + Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist) +from eth2spec.utils.ssz.ssz_typing import Bitvector # noqa: F401 +from eth2spec.utils import bls +from eth2spec.utils.hash_function import hash +''' + + @classmethod + def preparations(cls) -> str: + return ''' +SSZObject = TypeVar('SSZObject', bound=View) +''' + + @classmethod + def sundry_functions(cls) -> str: + return ''' +def get_eth1_data(block: Eth1Block) -> Eth1Data: + """ + A stub function return mocking Eth1Data. + """ + return Eth1Data( + deposit_root=block.deposit_root, + deposit_count=block.deposit_count, + block_hash=hash_tree_root(block)) + + +def cache_this(key_fn, value_fn, lru_size): # type: ignore + cache_dict = LRU(size=lru_size) + + def wrapper(*args, **kw): # type: ignore + key = key_fn(*args, **kw) + nonlocal cache_dict + if key not in cache_dict: + cache_dict[key] = value_fn(*args, **kw) + return cache_dict[key] + return wrapper + + +_compute_shuffled_index = compute_shuffled_index +compute_shuffled_index = cache_this( + lambda index, index_count, seed: (index, index_count, seed), + _compute_shuffled_index, lru_size=SLOTS_PER_EPOCH * 3) + +_get_total_active_balance = get_total_active_balance +get_total_active_balance = cache_this( + lambda state: (state.validators.hash_tree_root(), compute_epoch_at_slot(state.slot)), + _get_total_active_balance, lru_size=10) + +_get_base_reward = get_base_reward +get_base_reward = cache_this( + lambda state, index: (state.validators.hash_tree_root(), state.slot, index), + _get_base_reward, lru_size=2048) + +_get_committee_count_per_slot = get_committee_count_per_slot +get_committee_count_per_slot = cache_this( + lambda state, epoch: (state.validators.hash_tree_root(), epoch), + _get_committee_count_per_slot, lru_size=SLOTS_PER_EPOCH * 3) + +_get_active_validator_indices = get_active_validator_indices +get_active_validator_indices = cache_this( + lambda state, epoch: (state.validators.hash_tree_root(), epoch), + _get_active_validator_indices, lru_size=3) + +_get_beacon_committee = get_beacon_committee +get_beacon_committee = cache_this( + lambda state, slot, index: (state.validators.hash_tree_root(), state.randao_mixes.hash_tree_root(), slot, index), + _get_beacon_committee, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3) + +_get_matching_target_attestations = get_matching_target_attestations +get_matching_target_attestations = cache_this( + lambda state, epoch: (state.hash_tree_root(), epoch), + _get_matching_target_attestations, lru_size=10) + +_get_matching_head_attestations = get_matching_head_attestations +get_matching_head_attestations = cache_this( + lambda state, epoch: (state.hash_tree_root(), epoch), + _get_matching_head_attestations, lru_size=10) + +_get_attesting_indices = get_attesting_indices +get_attesting_indices = cache_this( + lambda state, data, bits: ( + state.randao_mixes.hash_tree_root(), + state.validators.hash_tree_root(), data.hash_tree_root(), bits.hash_tree_root() + ), + _get_attesting_indices, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3)''' + + + @classmethod + def execution_engine_cls(cls) -> str: + return "" + + + @classmethod + def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: + return {} + + @classmethod + def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]: + return {} + + @classmethod + def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]: + return functions + + +# AltairSpecBuilder +# +class AltairSpecBuilder(Phase0SpecBuilder): + fork: str = ALTAIR + + @classmethod + def imports(cls, preset_name: str) -> str: + return super().imports(preset_name) + '\n' + f''' +from typing import NewType, Union as PyUnion + +from eth2spec.phase0 import {preset_name} as phase0 +from eth2spec.test.helpers.merkle import build_proof +from eth2spec.utils.ssz.ssz_typing import Path +''' + + @classmethod + def preparations(cls): + return super().preparations() + '\n' + ''' +SSZVariableName = str +GeneralizedIndex = NewType('GeneralizedIndex', int) +''' + + @classmethod + def sundry_functions(cls) -> str: + return super().sundry_functions() + '\n\n' + ''' +def get_generalized_index(ssz_class: Any, *path: Sequence[PyUnion[int, SSZVariableName]]) -> GeneralizedIndex: + ssz_path = Path(ssz_class) + for item in path: + ssz_path = ssz_path / item + return GeneralizedIndex(ssz_path.gindex()) + + +def compute_merkle_proof_for_state(state: BeaconState, + index: GeneralizedIndex) -> Sequence[Bytes32]: + return build_proof(state.get_backing(), index)''' + + + @classmethod + def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: + constants = { + 'FINALIZED_ROOT_INDEX': 'GeneralizedIndex(105)', + 'CURRENT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(54)', + 'NEXT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(55)', + } + return {**super().hardcoded_ssz_dep_constants(), **constants} + + @classmethod + def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]: + if "eth_aggregate_pubkeys" in functions: + functions["eth_aggregate_pubkeys"] = OPTIMIZED_BLS_AGGREGATE_PUBKEYS.strip() + return super().implement_optimizations(functions) + +# +# BellatrixSpecBuilder +# +class BellatrixSpecBuilder(AltairSpecBuilder): + fork: str = BELLATRIX + + @classmethod + def imports(cls, preset_name: str): + return super().imports(preset_name) + f''' +from typing import Protocol +from eth2spec.altair import {preset_name} as altair +from eth2spec.utils.ssz.ssz_typing import Bytes8, Bytes20, ByteList, ByteVector +''' + + @classmethod + def preparations(cls): + return super().preparations() + + @classmethod + def sundry_functions(cls) -> str: + return super().sundry_functions() + '\n\n' + """ +ExecutionState = Any + + +def get_pow_block(hash: Bytes32) -> Optional[PowBlock]: + return PowBlock(block_hash=hash, parent_hash=Bytes32(), total_difficulty=uint256(0)) + + +def get_execution_state(_execution_state_root: Bytes32) -> ExecutionState: + pass + + +def get_pow_chain_head() -> PowBlock: + pass""" + + @classmethod + def execution_engine_cls(cls) -> str: + return "\n\n" + """ +class NoopExecutionEngine(ExecutionEngine): + + def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + return True + + def notify_forkchoice_updated(self: ExecutionEngine, + head_block_hash: Hash32, + safe_block_hash: Hash32, + finalized_block_hash: Hash32, + payload_attributes: Optional[PayloadAttributes]) -> Optional[PayloadId]: + pass + + def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: + # pylint: disable=unused-argument + raise NotImplementedError("no default block production") + + def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + return True + + def verify_and_notify_new_payload(self: ExecutionEngine, + new_payload_request: NewPayloadRequest) -> bool: + return True + + +EXECUTION_ENGINE = NoopExecutionEngine()""" + + + @classmethod + def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: + constants = { + 'MAX_BYTES_PER_TRANSACTION': spec_object.preset_vars['MAX_BYTES_PER_TRANSACTION'].value, + } + return {**super().hardcoded_custom_type_dep_constants(spec_object), **constants} + + +# +# CapellaSpecBuilder +# +class CapellaSpecBuilder(BellatrixSpecBuilder): + fork: str = CAPELLA + + @classmethod + def imports(cls, preset_name: str): + return super().imports(preset_name) + f''' +from eth2spec.bellatrix import {preset_name} as bellatrix +''' + + + @classmethod + def sundry_functions(cls) -> str: + return super().sundry_functions() + '\n\n' + ''' +def compute_merkle_proof_for_block_body(body: BeaconBlockBody, + index: GeneralizedIndex) -> Sequence[Bytes32]: + return build_proof(body.get_backing(), index)''' + + + @classmethod + def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: + constants = { + 'EXECUTION_PAYLOAD_INDEX': 'GeneralizedIndex(25)', + } + return {**super().hardcoded_ssz_dep_constants(), **constants} + +# +# DenebSpecBuilder +# +class DenebSpecBuilder(CapellaSpecBuilder): + fork: str = DENEB + + @classmethod + def imports(cls, preset_name: str): + return super().imports(preset_name) + f''' +from eth2spec.capella import {preset_name} as capella +''' + + + @classmethod + def preparations(cls): + return super().preparations() + '\n' + ''' +T = TypeVar('T') # For generic function +''' + + @classmethod + def sundry_functions(cls) -> str: + return super().sundry_functions() + '\n\n' + ''' +def retrieve_blobs_and_proofs(beacon_block_root: Root) -> PyUnion[Tuple[Blob, KZGProof], Tuple[str, str]]: + # pylint: disable=unused-argument + return ("TEST", "TEST")''' + + @classmethod + def execution_engine_cls(cls) -> str: + return "\n\n" + """ +class NoopExecutionEngine(ExecutionEngine): + + def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + return True + + def notify_forkchoice_updated(self: ExecutionEngine, + head_block_hash: Hash32, + safe_block_hash: Hash32, + finalized_block_hash: Hash32, + payload_attributes: Optional[PayloadAttributes]) -> Optional[PayloadId]: + pass + + def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: + # pylint: disable=unused-argument + raise NotImplementedError("no default block production") + + def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + return True + + def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: + return True + + def verify_and_notify_new_payload(self: ExecutionEngine, + new_payload_request: NewPayloadRequest) -> bool: + return True + + +EXECUTION_ENGINE = NoopExecutionEngine()""" + + + @classmethod + def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: + constants = { + 'BYTES_PER_FIELD_ELEMENT': spec_object.constant_vars['BYTES_PER_FIELD_ELEMENT'].value, + 'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value, + 'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value, + } + return {**super().hardcoded_custom_type_dep_constants(spec_object), **constants} + + +# +# EIP6110SpecBuilder +# +class EIP6110SpecBuilder(DenebSpecBuilder): + fork: str = EIP6110 + + @classmethod + def imports(cls, preset_name: str): + return super().imports(preset_name) + f''' +from eth2spec.deneb import {preset_name} as deneb +''' + + +spec_builders = { + builder.fork: builder + for builder in ( + Phase0SpecBuilder, + AltairSpecBuilder, + BellatrixSpecBuilder, + CapellaSpecBuilder, + DenebSpecBuilder, + EIP6110SpecBuilder, + ) +} diff --git a/pysetup/typing.py b/pysetup/typing.py new file mode 100644 index 0000000000..56f0cf3b10 --- /dev/null +++ b/pysetup/typing.py @@ -0,0 +1,32 @@ +from pathlib import Path +from typing import Dict, NamedTuple, Optional, List + + +class ProtocolDefinition(NamedTuple): + # just function definitions currently. May expand with configuration vars in future. + functions: Dict[str, str] + + +class VariableDefinition(NamedTuple): + type_name: Optional[str] + value: str + comment: Optional[str] # e.g. "noqa: E501" + type_hint: Optional[str] # e.g., "Final" + + +class SpecObject(NamedTuple): + functions: Dict[str, str] + protocols: Dict[str, ProtocolDefinition] + custom_types: Dict[str, str] + constant_vars: Dict[str, VariableDefinition] + preset_vars: Dict[str, VariableDefinition] + config_vars: Dict[str, VariableDefinition] + ssz_dep_constants: Dict[str, str] # the constants that depend on ssz_objects + ssz_objects: Dict[str, str] + dataclasses: Dict[str, str] + + +class BuildTarget(NamedTuple): + name: str + preset_paths: List[Path] + config_path: Path diff --git a/setup.py b/setup.py index 6013eff756..8a02f43c1a 100644 --- a/setup.py +++ b/setup.py @@ -4,11 +4,8 @@ from distutils.util import convert_path from pathlib import Path import os -import re import string -import textwrap -from typing import Dict, NamedTuple, List, Sequence, Optional, TypeVar, Tuple -from abc import ABC, abstractmethod +from typing import Dict, List, Sequence, Optional, Tuple import ast import subprocess import sys @@ -16,6 +13,27 @@ from collections import OrderedDict import json +from pysetup.constants import ( + # code names + PHASE0, + # misc + ETH2_SPEC_COMMENT_PREFIX, +) +from pysetup.spec_builders import spec_builders +from pysetup.typing import ( + BuildTarget, + ProtocolDefinition, + SpecObject, + VariableDefinition, +) +from pysetup.helpers import ( + combine_spec_objects, + dependency_order_class_objects, + objects_to_spec, + parse_config_vars, +) +from pysetup.md_doc_paths import get_md_doc_paths + # NOTE: have to programmatically include third-party dependencies in `setup.py`. def installPackage(package: str): @@ -41,60 +59,6 @@ def installPackage(package: str): from marko.ext.gfm.elements import Table -# Definitions in context.py -PHASE0 = 'phase0' -ALTAIR = 'altair' -BELLATRIX = 'bellatrix' -CAPELLA = 'capella' -DENEB = 'deneb' -EIP6110 = 'eip6110' - - -# The helper functions that are used when defining constants -CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS = ''' -def ceillog2(x: int) -> uint64: - if x < 1: - raise ValueError(f"ceillog2 accepts only positive values, x={x}") - return uint64((x - 1).bit_length()) - - -def floorlog2(x: int) -> uint64: - if x < 1: - raise ValueError(f"floorlog2 accepts only positive values, x={x}") - return uint64(x.bit_length() - 1) -''' - - -OPTIMIZED_BLS_AGGREGATE_PUBKEYS = ''' -def eth_aggregate_pubkeys(pubkeys: Sequence[BLSPubkey]) -> BLSPubkey: - return bls.AggregatePKs(pubkeys) -''' - - -class ProtocolDefinition(NamedTuple): - # just function definitions currently. May expand with configuration vars in future. - functions: Dict[str, str] - - -class VariableDefinition(NamedTuple): - type_name: Optional[str] - value: str - comment: Optional[str] # e.g. "noqa: E501" - type_hint: Optional[str] # e.g., "Final" - - -class SpecObject(NamedTuple): - functions: Dict[str, str] - protocols: Dict[str, ProtocolDefinition] - custom_types: Dict[str, str] - constant_vars: Dict[str, VariableDefinition] - preset_vars: Dict[str, VariableDefinition] - config_vars: Dict[str, VariableDefinition] - ssz_dep_constants: Dict[str, str] # the constants that depend on ssz_objects - ssz_objects: Dict[str, str] - dataclasses: Dict[str, str] - - def _get_name_from_heading(heading: Heading) -> Optional[str]: last_child = heading.children[-1] if isinstance(last_child, CodeSpan): @@ -159,13 +123,12 @@ def _load_kzg_trusted_setups(preset_name): return trusted_setup_G1, trusted_setup_G2, trusted_setup_G1_lagrange, roots_of_unity + ALL_KZG_SETUPS = { 'minimal': _load_kzg_trusted_setups('minimal'), 'mainnet': _load_kzg_trusted_setups('mainnet') } -ETH2_SPEC_COMMENT_PREFIX = "eth2spec:" - def _get_eth2_spec_comment(child: LinkRefDef) -> Optional[str]: _, _, title = child._parse_info @@ -177,7 +140,7 @@ def _get_eth2_spec_comment(child: LinkRefDef) -> Optional[str]: return title[len(ETH2_SPEC_COMMENT_PREFIX):].strip() -def _parse_value(name: str, typed_value: str, type_hint: Optional[str]=None) -> VariableDefinition: +def _parse_value(name: str, typed_value: str, type_hint: Optional[str] = None) -> VariableDefinition: comment = None if name == "BLS12_381_Q": comment = "noqa: E501" @@ -306,655 +269,6 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr ) -class SpecBuilder(ABC): - @property - @abstractmethod - def fork(self) -> str: - raise NotImplementedError() - - @classmethod - @abstractmethod - def imports(cls, preset_name: str) -> str: - """ - Import objects from other libraries. - """ - raise NotImplementedError() - - @classmethod - @abstractmethod - def preparations(cls) -> str: - """ - Define special types/constants for building pyspec or call functions. - """ - raise NotImplementedError() - - @classmethod - @abstractmethod - def sundry_functions(cls) -> str: - """ - The functions that are (1) defined abstractly in specs or (2) adjusted for getting better performance. - """ - raise NotImplementedError() - - @classmethod - def execution_engine_cls(cls) -> str: - raise NotImplementedError() - - @classmethod - @abstractmethod - def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: - """ - The constants that are required for SSZ objects. - """ - raise NotImplementedError() - - @classmethod - @abstractmethod - def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]: # TODO - """ - The constants that are required for custom types. - """ - raise NotImplementedError() - - @classmethod - @abstractmethod - def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]: - raise NotImplementedError() - - @classmethod - @abstractmethod - def build_spec(cls, preset_name: str, - source_files: List[Path], preset_files: Sequence[Path], config_file: Path) -> str: - raise NotImplementedError() - - -# -# Phase0SpecBuilder -# -class Phase0SpecBuilder(SpecBuilder): - fork: str = PHASE0 - - @classmethod - def imports(cls, preset_name: str) -> str: - return '''from lru import LRU -from dataclasses import ( - dataclass, - field, -) -from typing import ( - Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar, NamedTuple, Final -) - -from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes -from eth2spec.utils.ssz.ssz_typing import ( - View, boolean, Container, List, Vector, uint8, uint32, uint64, uint256, - Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist) -from eth2spec.utils.ssz.ssz_typing import Bitvector # noqa: F401 -from eth2spec.utils import bls -from eth2spec.utils.hash_function import hash -''' - - @classmethod - def preparations(cls) -> str: - return ''' -SSZObject = TypeVar('SSZObject', bound=View) -''' - - @classmethod - def sundry_functions(cls) -> str: - return ''' -def get_eth1_data(block: Eth1Block) -> Eth1Data: - """ - A stub function return mocking Eth1Data. - """ - return Eth1Data( - deposit_root=block.deposit_root, - deposit_count=block.deposit_count, - block_hash=hash_tree_root(block)) - - -def cache_this(key_fn, value_fn, lru_size): # type: ignore - cache_dict = LRU(size=lru_size) - - def wrapper(*args, **kw): # type: ignore - key = key_fn(*args, **kw) - nonlocal cache_dict - if key not in cache_dict: - cache_dict[key] = value_fn(*args, **kw) - return cache_dict[key] - return wrapper - - -_compute_shuffled_index = compute_shuffled_index -compute_shuffled_index = cache_this( - lambda index, index_count, seed: (index, index_count, seed), - _compute_shuffled_index, lru_size=SLOTS_PER_EPOCH * 3) - -_get_total_active_balance = get_total_active_balance -get_total_active_balance = cache_this( - lambda state: (state.validators.hash_tree_root(), compute_epoch_at_slot(state.slot)), - _get_total_active_balance, lru_size=10) - -_get_base_reward = get_base_reward -get_base_reward = cache_this( - lambda state, index: (state.validators.hash_tree_root(), state.slot, index), - _get_base_reward, lru_size=2048) - -_get_committee_count_per_slot = get_committee_count_per_slot -get_committee_count_per_slot = cache_this( - lambda state, epoch: (state.validators.hash_tree_root(), epoch), - _get_committee_count_per_slot, lru_size=SLOTS_PER_EPOCH * 3) - -_get_active_validator_indices = get_active_validator_indices -get_active_validator_indices = cache_this( - lambda state, epoch: (state.validators.hash_tree_root(), epoch), - _get_active_validator_indices, lru_size=3) - -_get_beacon_committee = get_beacon_committee -get_beacon_committee = cache_this( - lambda state, slot, index: (state.validators.hash_tree_root(), state.randao_mixes.hash_tree_root(), slot, index), - _get_beacon_committee, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3) - -_get_matching_target_attestations = get_matching_target_attestations -get_matching_target_attestations = cache_this( - lambda state, epoch: (state.hash_tree_root(), epoch), - _get_matching_target_attestations, lru_size=10) - -_get_matching_head_attestations = get_matching_head_attestations -get_matching_head_attestations = cache_this( - lambda state, epoch: (state.hash_tree_root(), epoch), - _get_matching_head_attestations, lru_size=10) - -_get_attesting_indices = get_attesting_indices -get_attesting_indices = cache_this( - lambda state, data, bits: ( - state.randao_mixes.hash_tree_root(), - state.validators.hash_tree_root(), data.hash_tree_root(), bits.hash_tree_root() - ), - _get_attesting_indices, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3)''' - - - @classmethod - def execution_engine_cls(cls) -> str: - return "" - - - @classmethod - def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: - return {} - - @classmethod - def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]: - return {} - - @classmethod - def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]: - return functions - - @classmethod - def build_spec(cls, preset_name: str, - source_files: Sequence[Path], preset_files: Sequence[Path], config_file: Path) -> str: - return _build_spec(preset_name, cls.fork, source_files, preset_files, config_file) - - -# -# AltairSpecBuilder -# -class AltairSpecBuilder(Phase0SpecBuilder): - fork: str = ALTAIR - - @classmethod - def imports(cls, preset_name: str) -> str: - return super().imports(preset_name) + '\n' + f''' -from typing import NewType, Union as PyUnion - -from eth2spec.phase0 import {preset_name} as phase0 -from eth2spec.test.helpers.merkle import build_proof -from eth2spec.utils.ssz.ssz_typing import Path -''' - - @classmethod - def preparations(cls): - return super().preparations() + '\n' + ''' -SSZVariableName = str -GeneralizedIndex = NewType('GeneralizedIndex', int) -''' - - @classmethod - def sundry_functions(cls) -> str: - return super().sundry_functions() + '\n\n' + ''' -def get_generalized_index(ssz_class: Any, *path: Sequence[PyUnion[int, SSZVariableName]]) -> GeneralizedIndex: - ssz_path = Path(ssz_class) - for item in path: - ssz_path = ssz_path / item - return GeneralizedIndex(ssz_path.gindex()) - - -def compute_merkle_proof_for_state(state: BeaconState, - index: GeneralizedIndex) -> Sequence[Bytes32]: - return build_proof(state.get_backing(), index)''' - - - @classmethod - def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: - constants = { - 'FINALIZED_ROOT_INDEX': 'GeneralizedIndex(105)', - 'CURRENT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(54)', - 'NEXT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(55)', - } - return {**super().hardcoded_ssz_dep_constants(), **constants} - - @classmethod - def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]: - if "eth_aggregate_pubkeys" in functions: - functions["eth_aggregate_pubkeys"] = OPTIMIZED_BLS_AGGREGATE_PUBKEYS.strip() - return super().implement_optimizations(functions) - -# -# BellatrixSpecBuilder -# -class BellatrixSpecBuilder(AltairSpecBuilder): - fork: str = BELLATRIX - - @classmethod - def imports(cls, preset_name: str): - return super().imports(preset_name) + f''' -from typing import Protocol -from eth2spec.altair import {preset_name} as altair -from eth2spec.utils.ssz.ssz_typing import Bytes8, Bytes20, ByteList, ByteVector -''' - - @classmethod - def preparations(cls): - return super().preparations() - - @classmethod - def sundry_functions(cls) -> str: - return super().sundry_functions() + '\n\n' + """ -ExecutionState = Any - - -def get_pow_block(hash: Bytes32) -> Optional[PowBlock]: - return PowBlock(block_hash=hash, parent_hash=Bytes32(), total_difficulty=uint256(0)) - - -def get_execution_state(_execution_state_root: Bytes32) -> ExecutionState: - pass - - -def get_pow_chain_head() -> PowBlock: - pass""" - - @classmethod - def execution_engine_cls(cls) -> str: - return "\n\n" + """ -class NoopExecutionEngine(ExecutionEngine): - - def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: - return True - - def notify_forkchoice_updated(self: ExecutionEngine, - head_block_hash: Hash32, - safe_block_hash: Hash32, - finalized_block_hash: Hash32, - payload_attributes: Optional[PayloadAttributes]) -> Optional[PayloadId]: - pass - - def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: - # pylint: disable=unused-argument - raise NotImplementedError("no default block production") - - def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: - return True - - def verify_and_notify_new_payload(self: ExecutionEngine, - new_payload_request: NewPayloadRequest) -> bool: - return True - - -EXECUTION_ENGINE = NoopExecutionEngine()""" - - - @classmethod - def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: - constants = { - 'MAX_BYTES_PER_TRANSACTION': spec_object.preset_vars['MAX_BYTES_PER_TRANSACTION'].value, - } - return {**super().hardcoded_custom_type_dep_constants(spec_object), **constants} - - -# -# CapellaSpecBuilder -# -class CapellaSpecBuilder(BellatrixSpecBuilder): - fork: str = CAPELLA - - @classmethod - def imports(cls, preset_name: str): - return super().imports(preset_name) + f''' -from eth2spec.bellatrix import {preset_name} as bellatrix -''' - - - @classmethod - def sundry_functions(cls) -> str: - return super().sundry_functions() + '\n\n' + ''' -def compute_merkle_proof_for_block_body(body: BeaconBlockBody, - index: GeneralizedIndex) -> Sequence[Bytes32]: - return build_proof(body.get_backing(), index)''' - - - @classmethod - def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: - constants = { - 'EXECUTION_PAYLOAD_INDEX': 'GeneralizedIndex(25)', - } - return {**super().hardcoded_ssz_dep_constants(), **constants} - -# -# DenebSpecBuilder -# -class DenebSpecBuilder(CapellaSpecBuilder): - fork: str = DENEB - - @classmethod - def imports(cls, preset_name: str): - return super().imports(preset_name) + f''' -from eth2spec.capella import {preset_name} as capella -''' - - - @classmethod - def preparations(cls): - return super().preparations() + '\n' + ''' -T = TypeVar('T') # For generic function -''' - - @classmethod - def sundry_functions(cls) -> str: - return super().sundry_functions() + '\n\n' + ''' -def retrieve_blobs_and_proofs(beacon_block_root: Root) -> PyUnion[Tuple[Blob, KZGProof], Tuple[str, str]]: - # pylint: disable=unused-argument - return ("TEST", "TEST")''' - - @classmethod - def execution_engine_cls(cls) -> str: - return "\n\n" + """ -class NoopExecutionEngine(ExecutionEngine): - - def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: - return True - - def notify_forkchoice_updated(self: ExecutionEngine, - head_block_hash: Hash32, - safe_block_hash: Hash32, - finalized_block_hash: Hash32, - payload_attributes: Optional[PayloadAttributes]) -> Optional[PayloadId]: - pass - - def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: - # pylint: disable=unused-argument - raise NotImplementedError("no default block production") - - def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: - return True - - def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: - return True - - def verify_and_notify_new_payload(self: ExecutionEngine, - new_payload_request: NewPayloadRequest) -> bool: - return True - - -EXECUTION_ENGINE = NoopExecutionEngine()""" - - - @classmethod - def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: - constants = { - 'BYTES_PER_FIELD_ELEMENT': spec_object.constant_vars['BYTES_PER_FIELD_ELEMENT'].value, - 'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value, - 'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value, - } - return {**super().hardcoded_custom_type_dep_constants(spec_object), **constants} - - -# -# EIP6110SpecBuilder -# -class EIP6110SpecBuilder(DenebSpecBuilder): - fork: str = EIP6110 - - @classmethod - def imports(cls, preset_name: str): - return super().imports(preset_name) + f''' -from eth2spec.deneb import {preset_name} as deneb -''' - - -spec_builders = { - builder.fork: builder - for builder in (Phase0SpecBuilder, AltairSpecBuilder, BellatrixSpecBuilder, CapellaSpecBuilder, DenebSpecBuilder, EIP6110SpecBuilder) -} - - -def is_byte_vector(value: str) -> bool: - return value.startswith(('ByteVector')) - - -def make_function_abstract(protocol_def: ProtocolDefinition, key: str): - function = protocol_def.functions[key].split('"""') - protocol_def.functions[key] = function[0] + "..." - - -def objects_to_spec(preset_name: str, - spec_object: SpecObject, - builder: SpecBuilder, - ordered_class_objects: Dict[str, str]) -> str: - """ - Given all the objects that constitute a spec, combine them into a single pyfile. - """ - new_type_definitions = ( - '\n\n'.join( - [ - f"class {key}({value}):\n pass\n" if not is_byte_vector(value) else f"class {key}({value}): # type: ignore\n pass\n" - for key, value in spec_object.custom_types.items() - ] - ) - ) - - def format_protocol(protocol_name: str, protocol_def: ProtocolDefinition) -> str: - abstract_functions = ["verify_and_notify_new_payload"] - for key in protocol_def.functions.keys(): - if key in abstract_functions: - make_function_abstract(protocol_def, key) - - protocol = f"class {protocol_name}(Protocol):" - for fn_source in protocol_def.functions.values(): - fn_source = fn_source.replace("self: "+protocol_name, "self") - protocol += "\n\n" + textwrap.indent(fn_source, " ") - return protocol - - protocols_spec = '\n\n\n'.join(format_protocol(k, v) for k, v in spec_object.protocols.items()) - for k in list(spec_object.functions): - if k in [ - "ceillog2", - "floorlog2", - "compute_merkle_proof_for_block_body", - "compute_merkle_proof_for_state", - ]: - del spec_object.functions[k] - functions = builder.implement_optimizations(spec_object.functions) - functions_spec = '\n\n\n'.join(functions.values()) - - # Access global dict of config vars for runtime configurables - for name in spec_object.config_vars.keys(): - functions_spec = re.sub(r"\b%s\b" % name, 'config.' + name, functions_spec) - - def format_config_var(name: str, vardef: VariableDefinition) -> str: - if vardef.type_name is None: - out = f'{name}={vardef.value},' - else: - out = f'{name}={vardef.type_name}({vardef.value}),' - if vardef.comment is not None: - out += f' # {vardef.comment}' - return out - - config_spec = 'class Configuration(NamedTuple):\n' - config_spec += ' PRESET_BASE: str\n' - config_spec += '\n'.join(f' {k}: {v.type_name if v.type_name is not None else "int"}' - for k, v in spec_object.config_vars.items()) - config_spec += '\n\n\nconfig = Configuration(\n' - config_spec += f' PRESET_BASE="{preset_name}",\n' - config_spec += '\n'.join(' ' + format_config_var(k, v) for k, v in spec_object.config_vars.items()) - config_spec += '\n)\n' - - def format_constant(name: str, vardef: VariableDefinition) -> str: - if vardef.type_name is None: - if vardef.type_hint is None: - out = f'{name} = {vardef.value}' - else: - out = f'{name}: {vardef.type_hint} = {vardef.value}' - else: - out = f'{name} = {vardef.type_name}({vardef.value})' - if vardef.comment is not None: - out += f' # {vardef.comment}' - return out - - constant_vars_spec = '# Constant vars\n' + '\n'.join(format_constant(k, v) for k, v in spec_object.constant_vars.items()) - preset_vars_spec = '# Preset vars\n' + '\n'.join(format_constant(k, v) for k, v in spec_object.preset_vars.items()) - ordered_class_objects_spec = '\n\n\n'.join(ordered_class_objects.values()) - ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_ssz_dep_constants()[x]), builder.hardcoded_ssz_dep_constants())) - ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), builder.hardcoded_ssz_dep_constants())) - custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_custom_type_dep_constants(spec_object)[x]), builder.hardcoded_custom_type_dep_constants(spec_object))) - spec = ( - builder.imports(preset_name) - + builder.preparations() - + '\n\n' + f"fork = \'{builder.fork}\'\n" - # The constants that some SSZ containers require. Need to be defined before `new_type_definitions` - + ('\n\n' + custom_type_dep_constants + '\n' if custom_type_dep_constants != '' else '') - + '\n\n' + new_type_definitions - + '\n' + CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS - # The constants that some SSZ containers require. Need to be defined before `constants_spec` - + ('\n\n' + ssz_dep_constants if ssz_dep_constants != '' else '') - + '\n\n' + constant_vars_spec - + '\n\n' + preset_vars_spec - + '\n\n\n' + config_spec - + '\n\n' + ordered_class_objects_spec - + ('\n\n\n' + protocols_spec if protocols_spec != '' else '') - + '\n\n\n' + functions_spec - + '\n\n' + builder.sundry_functions() - + builder.execution_engine_cls() - # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are - # as same as the spec definition. - + ('\n\n\n' + ssz_dep_constants_verification if ssz_dep_constants_verification != '' else '') - + '\n' - ) - return spec - - -def combine_protocols(old_protocols: Dict[str, ProtocolDefinition], - new_protocols: Dict[str, ProtocolDefinition]) -> Dict[str, ProtocolDefinition]: - for key, value in new_protocols.items(): - if key not in old_protocols: - old_protocols[key] = value - else: - functions = combine_dicts(old_protocols[key].functions, value.functions) - old_protocols[key] = ProtocolDefinition(functions=functions) - return old_protocols - - -T = TypeVar('T') - - -def combine_dicts(old_dict: Dict[str, T], new_dict: Dict[str, T]) -> Dict[str, T]: - return {**old_dict, **new_dict} - - -ignored_dependencies = [ - 'bit', 'boolean', 'Vector', 'List', 'Container', 'BLSPubkey', 'BLSSignature', - 'Bytes1', 'Bytes4', 'Bytes8', 'Bytes20', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', - 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', - 'bytes', 'byte', 'ByteList', 'ByteVector', - 'Dict', 'dict', 'field', 'ceillog2', 'floorlog2', 'Set', - 'Optional', 'Sequence', -] - - -def dependency_order_class_objects(objects: Dict[str, str], custom_types: Dict[str, str]) -> None: - """ - Determines which SSZ Object is dependent on which other and orders them appropriately - """ - items = list(objects.items()) - for key, value in items: - dependencies = [] - for line in value.split('\n'): - if not re.match(r'\s+\w+: .+', line): - continue # skip whitespace etc. - line = line[line.index(':') + 1:] # strip of field name - if '#' in line: - line = line[:line.index('#')] # strip of comment - dependencies.extend(re.findall(r'(\w+)', line)) # catch all legible words, potential dependencies - dependencies = filter(lambda x: '_' not in x and x.upper() != x, dependencies) # filter out constants - dependencies = filter(lambda x: x not in ignored_dependencies, dependencies) - dependencies = filter(lambda x: x not in custom_types, dependencies) - for dep in dependencies: - key_list = list(objects.keys()) - for item in [dep, key] + key_list[key_list.index(dep)+1:]: - objects[item] = objects.pop(item) - -def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str], custom_types) -> Dict[str, str]: - """ - Takes in old spec and new spec ssz objects, combines them, - and returns the newer versions of the objects in dependency order. - """ - for key, value in new_objects.items(): - old_objects[key] = value - return old_objects - - -def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: - """ - Takes in two spec variants (as tuples of their objects) and combines them using the appropriate combiner function. - """ - protocols = combine_protocols(spec0.protocols, spec1.protocols) - functions = combine_dicts(spec0.functions, spec1.functions) - custom_types = combine_dicts(spec0.custom_types, spec1.custom_types) - constant_vars = combine_dicts(spec0.constant_vars, spec1.constant_vars) - preset_vars = combine_dicts(spec0.preset_vars, spec1.preset_vars) - config_vars = combine_dicts(spec0.config_vars, spec1.config_vars) - ssz_dep_constants = combine_dicts(spec0.ssz_dep_constants, spec1.ssz_dep_constants) - ssz_objects = combine_ssz_objects(spec0.ssz_objects, spec1.ssz_objects, custom_types) - dataclasses = combine_dicts(spec0.dataclasses, spec1.dataclasses) - return SpecObject( - functions=functions, - protocols=protocols, - custom_types=custom_types, - constant_vars=constant_vars, - preset_vars=preset_vars, - config_vars=config_vars, - ssz_dep_constants=ssz_dep_constants, - ssz_objects=ssz_objects, - dataclasses=dataclasses, - ) - - -def parse_config_vars(conf: Dict[str, str]) -> Dict[str, str]: - """ - Parses a dict of basic str/int/list types into a dict for insertion into the spec code. - """ - out: Dict[str, str] = dict() - for k, v in conf.items(): - if isinstance(v, str) and (v.startswith("0x") or k == 'PRESET_BASE' or k == 'CONFIG_NAME'): - # Represent byte data with string, to avoid misinterpretation as big-endian int. - # Everything except PRESET_BASE and CONFIG_NAME is either byte data or an integer. - out[k] = f"'{v}'" - else: - out[k] = str(int(v)) - return out - - def load_preset(preset_files: Sequence[Path]) -> Dict[str, str]: """ Loads the a directory of preset files, merges the result into one preset. @@ -982,8 +296,11 @@ def load_config(config_path: Path) -> Dict[str, str]: return parse_config_vars(config_data) -def _build_spec(preset_name: str, fork: str, - source_files: Sequence[Path], preset_files: Sequence[Path], config_file: Path) -> str: +def build_spec(fork: str, + preset_name: str, + source_files: Sequence[Path], + preset_files: Sequence[Path], + config_file: Path) -> str: preset = load_preset(preset_files) config = load_config(config_file) all_specs = [get_spec(spec, preset, config, preset_name) for spec in source_files] @@ -1003,12 +320,6 @@ def _build_spec(preset_name: str, fork: str, return objects_to_spec(preset_name, spec_object, spec_builders[fork], class_objects) -class BuildTarget(NamedTuple): - name: str - preset_paths: List[Path] - config_path: Path - - class PySpecCommand(Command): """Convert spec markdown files to a spec python file""" @@ -1045,69 +356,7 @@ def finalize_options(self): if len(self.md_doc_paths) == 0: print("no paths were specified, using default markdown file paths for pyspec" " build (spec fork: %s)" % self.spec_fork) - if self.spec_fork in (PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110): - self.md_doc_paths = """ - specs/phase0/beacon-chain.md - specs/phase0/fork-choice.md - specs/phase0/validator.md - specs/phase0/weak-subjectivity.md - specs/phase0/p2p-interface.md - """ - if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110): - self.md_doc_paths += """ - specs/altair/light-client/full-node.md - specs/altair/light-client/light-client.md - specs/altair/light-client/p2p-interface.md - specs/altair/light-client/sync-protocol.md - specs/altair/beacon-chain.md - specs/altair/bls.md - specs/altair/fork.md - specs/altair/validator.md - specs/altair/p2p-interface.md - """ - if self.spec_fork in (BELLATRIX, CAPELLA, DENEB, EIP6110): - self.md_doc_paths += """ - specs/bellatrix/beacon-chain.md - specs/bellatrix/fork.md - specs/bellatrix/fork-choice.md - specs/bellatrix/validator.md - specs/bellatrix/p2p-interface.md - sync/optimistic.md - """ - if self.spec_fork in (CAPELLA, DENEB, EIP6110): - self.md_doc_paths += """ - specs/capella/light-client/fork.md - specs/capella/light-client/full-node.md - specs/capella/light-client/p2p-interface.md - specs/capella/light-client/sync-protocol.md - specs/capella/beacon-chain.md - specs/capella/fork.md - specs/capella/fork-choice.md - specs/capella/validator.md - specs/capella/p2p-interface.md - """ - if self.spec_fork in (DENEB, EIP6110): - self.md_doc_paths += """ - specs/deneb/light-client/fork.md - specs/deneb/light-client/full-node.md - specs/deneb/light-client/p2p-interface.md - specs/deneb/light-client/sync-protocol.md - specs/deneb/beacon-chain.md - specs/deneb/fork.md - specs/deneb/fork-choice.md - specs/deneb/polynomial-commitments.md - specs/deneb/p2p-interface.md - specs/deneb/validator.md - """ - if self.spec_fork == EIP6110: - self.md_doc_paths += """ - specs/_features/eip6110/light-client/fork.md - specs/_features/eip6110/light-client/full-node.md - specs/_features/eip6110/light-client/p2p-interface.md - specs/_features/eip6110/light-client/sync-protocol.md - specs/_features/eip6110/beacon-chain.md - specs/_features/eip6110/fork.md - """ + self.md_doc_paths = get_md_doc_paths(self.spec_fork) if len(self.md_doc_paths) == 0: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) @@ -1140,8 +389,13 @@ def run(self): dir_util.mkpath(self.out_dir) for (name, preset_paths, config_path) in self.parsed_build_targets: - spec_str = spec_builders[self.spec_fork].build_spec( - name, self.parsed_md_doc_paths, preset_paths, config_path) + spec_str = build_spec( + spec_builders[self.spec_fork].fork, + name, + self.parsed_md_doc_paths, + preset_paths, + config_path, + ) if self.dry_run: self.announce('dry run successfully prepared contents for spec.' f' out dir: "{self.out_dir}", spec fork: "{self.spec_fork}", build target: "{name}"') @@ -1202,6 +456,7 @@ def run(self): for spec_fork in spec_builders: self.run_pyspec_cmd(spec_fork=spec_fork) + commands = { 'pyspec': PySpecCommand, 'build_py': BuildPyCommand, From 453336c3029517d68565a50f245f3f8530feb50d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 31 May 2023 01:17:12 +0800 Subject: [PATCH 2/4] Update doc with the new file path --- docs/docs/new-feature.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/docs/new-feature.md b/docs/docs/new-feature.md index 5e6180329f..1b7cfd7206 100644 --- a/docs/docs/new-feature.md +++ b/docs/docs/new-feature.md @@ -53,17 +53,17 @@ For example, if the latest fork is Capella, use `./specs/capella` content as you ### 4. Add `fork.md` You can refer to the previous fork's `fork.md` file. ### 5. Make it executable -- Update [`constants.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/helpers/constants.py) with the new feature name. -- Update [`setup.py`](https://github.com/ethereum/consensus-specs/blob/dev/setup.py): - - Add a new `SpecBuilder` with the new feature name constant. e.g., `EIP9999SpecBuilder` - - Add the new `SpecBuilder` to `spec_builders` list. - - Add the path of the new markdown files in `finalize_options` function. +- Update Pyspec [`constants.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/helpers/constants.py) with the new feature name. +- Update helpers for [`setup.py`](https://github.com/ethereum/consensus-specs/blob/dev/setup.py) for building the spec: + - Update [`pysetup/constants.py`](https://github.com/ethereum/consensus-specs/blob/dev/constants.py) with the new feature name as Pyspec `constants.py` defined. + - Update [`pysetup/spec_builders.py`](https://github.com/ethereum/consensus-specs/blob/dev/pysetup/spec_builders.py) with a new `SpecBuilder` with the new feature name. e.g., `EIP9999SpecBuilder`. Append it to the `spec_builders` list. + - Update [`pysetup/md_doc_paths.py`](https://github.com/ethereum/consensus-specs/blob/dev/pysetup/md_doc_paths.py): add the path of the new markdown files in `get_md_doc_paths` function. ## B: Make it executable for pytest and test generator -### 1. Add `light-client/*` docs if you updated the content of `BeaconBlock` +### 1. [Optional] Add `light-client/*` docs if you updated the content of `BeaconBlock` - You can refer to the previous fork's `light-client/*` file. -- Add the path of the new markdown files in `setup.py`'s `finalize_options` function. +- Add the path of the new markdown files in [`pysetup/md_doc_paths.py`](https://github.com/ethereum/consensus-specs/blob/dev/pysetup/md_doc_paths.py)'s `get_md_doc_paths` function. ### 2. Add the mainnet and minimal presets and update the configs - Add presets: `presets/mainnet/.yaml` and `presets/minimal/.yaml` From 33d31b9634af641784182493c4907c9ebc248fda Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 26 Jun 2023 17:24:22 +0800 Subject: [PATCH 3/4] Rearrange spec_builders --- pysetup/helpers.py | 40 +-- pysetup/spec_builders.py | 437 ----------------------------- pysetup/spec_builders/__init__.py | 16 ++ pysetup/spec_builders/altair.py | 54 ++++ pysetup/spec_builders/base.py | 52 ++++ pysetup/spec_builders/bellatrix.py | 66 +++++ pysetup/spec_builders/capella.py | 29 ++ pysetup/spec_builders/deneb.py | 71 +++++ pysetup/spec_builders/eip6110.py | 12 + pysetup/spec_builders/phase0.py | 105 +++++++ pysetup/spec_builders/whisk.py | 20 ++ 11 files changed, 445 insertions(+), 457 deletions(-) delete mode 100644 pysetup/spec_builders.py create mode 100644 pysetup/spec_builders/__init__.py create mode 100644 pysetup/spec_builders/altair.py create mode 100644 pysetup/spec_builders/base.py create mode 100644 pysetup/spec_builders/bellatrix.py create mode 100644 pysetup/spec_builders/capella.py create mode 100644 pysetup/spec_builders/deneb.py create mode 100644 pysetup/spec_builders/eip6110.py create mode 100644 pysetup/spec_builders/phase0.py create mode 100644 pysetup/spec_builders/whisk.py diff --git a/pysetup/helpers.py b/pysetup/helpers.py index 4a06ccc2ee..692aaa0d7e 100644 --- a/pysetup/helpers.py +++ b/pysetup/helpers.py @@ -127,26 +127,26 @@ def format_constant(name: str, vardef: VariableDefinition) -> str: ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), hardcoded_ssz_dep_constants)) custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_custom_type_dep_constants[x]), hardcoded_custom_type_dep_constants)) spec_strs = [ - imports, - preparations, - f"fork = \'{fork}\'\n", - # The constants that some SSZ containers require. Need to be defined before `new_type_definitions` - custom_type_dep_constants, - new_type_definitions, - CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS, - # The constants that some SSZ containers require. Need to be defined before `constants_spec` - ssz_dep_constants, - constant_vars_spec, - preset_vars_spec, - config_spec, - ordered_class_objects_spec, - protocols_spec, - functions_spec, - sundry_functions, - execution_engine_cls, - # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are - # as same as the spec definition. - ssz_dep_constants_verification, + imports, + preparations, + f"fork = \'{fork}\'\n", + # The constants that some SSZ containers require. Need to be defined before `new_type_definitions` + custom_type_dep_constants, + new_type_definitions, + CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS, + # The constants that some SSZ containers require. Need to be defined before `constants_spec` + ssz_dep_constants, + constant_vars_spec, + preset_vars_spec, + config_spec, + ordered_class_objects_spec, + protocols_spec, + functions_spec, + sundry_functions, + execution_engine_cls, + # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are + # as same as the spec definition. + ssz_dep_constants_verification, ] return "\n\n\n".join([str.strip("\n") for str in spec_strs if str]) + "\n" diff --git a/pysetup/spec_builders.py b/pysetup/spec_builders.py deleted file mode 100644 index cffece07ee..0000000000 --- a/pysetup/spec_builders.py +++ /dev/null @@ -1,437 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Dict, Sequence -from pathlib import Path - -from .constants import ( - PHASE0, - ALTAIR, - BELLATRIX, - CAPELLA, - DENEB, - EIP6110, - WHISK, - OPTIMIZED_BLS_AGGREGATE_PUBKEYS, -) - - -class SpecBuilder(ABC): - @property - @abstractmethod - def fork(self) -> str: - raise NotImplementedError() - - @classmethod - def imports(cls, preset_name: str) -> str: - """ - Import objects from other libraries. - """ - return "" - - @classmethod - def preparations(cls) -> str: - """ - Define special types/constants for building pyspec or call functions. - """ - return "" - - @classmethod - def sundry_functions(cls) -> str: - """ - The functions that are (1) defined abstractly in specs or (2) adjusted for getting better performance. - """ - return "" - - @classmethod - def execution_engine_cls(cls) -> str: - return "" - - @classmethod - def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: - """ - The constants that are required for SSZ objects. - """ - return {} - - @classmethod - def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]: # TODO - """ - The constants that are required for custom types. - """ - return {} - - @classmethod - def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]: - return functions - - @classmethod - def build_spec(cls, preset_name: str, - source_files: Sequence[Path], preset_files: Sequence[Path], config_file: Path) -> str: - return _build_spec(preset_name, cls.fork, source_files, preset_files, config_file) - - -# -# Phase0SpecBuilder -# -class Phase0SpecBuilder(SpecBuilder): - fork: str = PHASE0 - - @classmethod - def imports(cls, preset_name: str) -> str: - return '''from lru import LRU -from dataclasses import ( - dataclass, - field, -) -from typing import ( - Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar, NamedTuple, Final -) - -from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes -from eth2spec.utils.ssz.ssz_typing import ( - View, boolean, Container, List, Vector, uint8, uint32, uint64, uint256, - Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist) -from eth2spec.utils.ssz.ssz_typing import Bitvector # noqa: F401 -from eth2spec.utils import bls -from eth2spec.utils.hash_function import hash -''' - - @classmethod - def preparations(cls) -> str: - return ''' -SSZObject = TypeVar('SSZObject', bound=View) -''' - - @classmethod - def sundry_functions(cls) -> str: - return ''' -def get_eth1_data(block: Eth1Block) -> Eth1Data: - """ - A stub function return mocking Eth1Data. - """ - return Eth1Data( - deposit_root=block.deposit_root, - deposit_count=block.deposit_count, - block_hash=hash_tree_root(block)) - - -def cache_this(key_fn, value_fn, lru_size): # type: ignore - cache_dict = LRU(size=lru_size) - - def wrapper(*args, **kw): # type: ignore - key = key_fn(*args, **kw) - nonlocal cache_dict - if key not in cache_dict: - cache_dict[key] = value_fn(*args, **kw) - return cache_dict[key] - return wrapper - - -_compute_shuffled_index = compute_shuffled_index -compute_shuffled_index = cache_this( - lambda index, index_count, seed: (index, index_count, seed), - _compute_shuffled_index, lru_size=SLOTS_PER_EPOCH * 3) - -_get_total_active_balance = get_total_active_balance -get_total_active_balance = cache_this( - lambda state: (state.validators.hash_tree_root(), compute_epoch_at_slot(state.slot)), - _get_total_active_balance, lru_size=10) - -_get_base_reward = get_base_reward -get_base_reward = cache_this( - lambda state, index: (state.validators.hash_tree_root(), state.slot, index), - _get_base_reward, lru_size=2048) - -_get_committee_count_per_slot = get_committee_count_per_slot -get_committee_count_per_slot = cache_this( - lambda state, epoch: (state.validators.hash_tree_root(), epoch), - _get_committee_count_per_slot, lru_size=SLOTS_PER_EPOCH * 3) - -_get_active_validator_indices = get_active_validator_indices -get_active_validator_indices = cache_this( - lambda state, epoch: (state.validators.hash_tree_root(), epoch), - _get_active_validator_indices, lru_size=3) - -_get_beacon_committee = get_beacon_committee -get_beacon_committee = cache_this( - lambda state, slot, index: (state.validators.hash_tree_root(), state.randao_mixes.hash_tree_root(), slot, index), - _get_beacon_committee, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3) - -_get_matching_target_attestations = get_matching_target_attestations -get_matching_target_attestations = cache_this( - lambda state, epoch: (state.hash_tree_root(), epoch), - _get_matching_target_attestations, lru_size=10) - -_get_matching_head_attestations = get_matching_head_attestations -get_matching_head_attestations = cache_this( - lambda state, epoch: (state.hash_tree_root(), epoch), - _get_matching_head_attestations, lru_size=10) - -_get_attesting_indices = get_attesting_indices -get_attesting_indices = cache_this( - lambda state, data, bits: ( - state.randao_mixes.hash_tree_root(), - state.validators.hash_tree_root(), data.hash_tree_root(), bits.hash_tree_root() - ), - _get_attesting_indices, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3)''' - - -# -# AltairSpecBuilder -# -class AltairSpecBuilder(SpecBuilder): - fork: str = ALTAIR - - @classmethod - def imports(cls, preset_name: str) -> str: - return f''' -from typing import NewType, Union as PyUnion - -from eth2spec.phase0 import {preset_name} as phase0 -from eth2spec.test.helpers.merkle import build_proof -from eth2spec.utils.ssz.ssz_typing import Path -''' - - @classmethod - def preparations(cls): - return ''' -SSZVariableName = str -GeneralizedIndex = NewType('GeneralizedIndex', int) -''' - - @classmethod - def sundry_functions(cls) -> str: - return ''' -def get_generalized_index(ssz_class: Any, *path: Sequence[PyUnion[int, SSZVariableName]]) -> GeneralizedIndex: - ssz_path = Path(ssz_class) - for item in path: - ssz_path = ssz_path / item - return GeneralizedIndex(ssz_path.gindex()) - - -def compute_merkle_proof_for_state(state: BeaconState, - index: GeneralizedIndex) -> Sequence[Bytes32]: - return build_proof(state.get_backing(), index)''' - - - @classmethod - def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: - return { - 'FINALIZED_ROOT_INDEX': 'GeneralizedIndex(105)', - 'CURRENT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(54)', - 'NEXT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(55)', - } - - @classmethod - def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]: - if "eth_aggregate_pubkeys" in functions: - functions["eth_aggregate_pubkeys"] = OPTIMIZED_BLS_AGGREGATE_PUBKEYS.strip() - return functions - -# -# BellatrixSpecBuilder -# -class BellatrixSpecBuilder(SpecBuilder): - fork: str = BELLATRIX - - @classmethod - def imports(cls, preset_name: str): - return f''' -from typing import Protocol -from eth2spec.altair import {preset_name} as altair -from eth2spec.utils.ssz.ssz_typing import Bytes8, Bytes20, ByteList, ByteVector -''' - - @classmethod - def sundry_functions(cls) -> str: - return """ -ExecutionState = Any - - -def get_pow_block(hash: Bytes32) -> Optional[PowBlock]: - return PowBlock(block_hash=hash, parent_hash=Bytes32(), total_difficulty=uint256(0)) - - -def get_execution_state(_execution_state_root: Bytes32) -> ExecutionState: - pass - - -def get_pow_chain_head() -> PowBlock: - pass""" - - @classmethod - def execution_engine_cls(cls) -> str: - return """ -class NoopExecutionEngine(ExecutionEngine): - - def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: - return True - - def notify_forkchoice_updated(self: ExecutionEngine, - head_block_hash: Hash32, - safe_block_hash: Hash32, - finalized_block_hash: Hash32, - payload_attributes: Optional[PayloadAttributes]) -> Optional[PayloadId]: - pass - - def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: - # pylint: disable=unused-argument - raise NotImplementedError("no default block production") - - def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: - return True - - def verify_and_notify_new_payload(self: ExecutionEngine, - new_payload_request: NewPayloadRequest) -> bool: - return True - - -EXECUTION_ENGINE = NoopExecutionEngine()""" - - - @classmethod - def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: - return { - 'MAX_BYTES_PER_TRANSACTION': spec_object.preset_vars['MAX_BYTES_PER_TRANSACTION'].value, - } - - -# -# CapellaSpecBuilder -# -class CapellaSpecBuilder(SpecBuilder): - fork: str = CAPELLA - - @classmethod - def imports(cls, preset_name: str): - return f''' -from eth2spec.bellatrix import {preset_name} as bellatrix -''' - - - @classmethod - def sundry_functions(cls) -> str: - return ''' -def compute_merkle_proof_for_block_body(body: BeaconBlockBody, - index: GeneralizedIndex) -> Sequence[Bytes32]: - return build_proof(body.get_backing(), index)''' - - - @classmethod - def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: - return { - 'EXECUTION_PAYLOAD_INDEX': 'GeneralizedIndex(25)', - } - - -# -# DenebSpecBuilder -# -class DenebSpecBuilder(SpecBuilder): - fork: str = DENEB - - @classmethod - def imports(cls, preset_name: str): - return f''' -from eth2spec.capella import {preset_name} as capella -''' - - - @classmethod - def preparations(cls): - return ''' -T = TypeVar('T') # For generic function -''' - - @classmethod - def sundry_functions(cls) -> str: - return ''' -def retrieve_blobs_and_proofs(beacon_block_root: Root) -> PyUnion[Tuple[Blob, KZGProof], Tuple[str, str]]: - # pylint: disable=unused-argument - return ("TEST", "TEST")''' - - @classmethod - def execution_engine_cls(cls) -> str: - return """ -class NoopExecutionEngine(ExecutionEngine): - - def notify_new_payload(self: ExecutionEngine, - execution_payload: ExecutionPayload, - parent_beacon_block_root: Root) -> bool: - return True - - def notify_forkchoice_updated(self: ExecutionEngine, - head_block_hash: Hash32, - safe_block_hash: Hash32, - finalized_block_hash: Hash32, - payload_attributes: Optional[PayloadAttributes]) -> Optional[PayloadId]: - pass - - def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: - # pylint: disable=unused-argument - raise NotImplementedError("no default block production") - - def is_valid_block_hash(self: ExecutionEngine, - execution_payload: ExecutionPayload, - parent_beacon_block_root: Root) -> bool: - return True - - def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: - return True - - def verify_and_notify_new_payload(self: ExecutionEngine, - new_payload_request: NewPayloadRequest) -> bool: - return True - - -EXECUTION_ENGINE = NoopExecutionEngine()""" - - - @classmethod - def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: - return { - 'BYTES_PER_FIELD_ELEMENT': spec_object.constant_vars['BYTES_PER_FIELD_ELEMENT'].value, - 'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value, - 'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value, - } - - -# -# EIP6110SpecBuilder -# -class EIP6110SpecBuilder(SpecBuilder): - fork: str = EIP6110 - - @classmethod - def imports(cls, preset_name: str): - return f''' -from eth2spec.deneb import {preset_name} as deneb -''' - -# -# WhiskSpecBuilder -# -class WhiskSpecBuilder(SpecBuilder): - fork: str = WHISK - - @classmethod - def imports(cls, preset_name: str): - return f''' -from eth2spec.capella import {preset_name} as capella -''' - - @classmethod - def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: - # Necessary for custom types `WhiskShuffleProof` and `WhiskTrackerProof` - return { - 'WHISK_MAX_SHUFFLE_PROOF_SIZE': spec_object.preset_vars['WHISK_MAX_SHUFFLE_PROOF_SIZE'].value, - 'WHISK_MAX_OPENING_PROOF_SIZE': spec_object.preset_vars['WHISK_MAX_OPENING_PROOF_SIZE'].value, - } - - -spec_builders = { - builder.fork: builder - for builder in ( - Phase0SpecBuilder, AltairSpecBuilder, BellatrixSpecBuilder, CapellaSpecBuilder, DenebSpecBuilder, - EIP6110SpecBuilder, WhiskSpecBuilder, - ) -} diff --git a/pysetup/spec_builders/__init__.py b/pysetup/spec_builders/__init__.py new file mode 100644 index 0000000000..123939b98c --- /dev/null +++ b/pysetup/spec_builders/__init__.py @@ -0,0 +1,16 @@ +from .phase0 import Phase0SpecBuilder +from .altair import AltairSpecBuilder +from .bellatrix import BellatrixSpecBuilder +from .capella import CapellaSpecBuilder +from .deneb import DenebSpecBuilder +from .eip6110 import EIP6110SpecBuilder +from .whisk import WhiskSpecBuilder + + +spec_builders = { + builder.fork: builder + for builder in ( + Phase0SpecBuilder, AltairSpecBuilder, BellatrixSpecBuilder, CapellaSpecBuilder, DenebSpecBuilder, + EIP6110SpecBuilder, WhiskSpecBuilder, + ) +} diff --git a/pysetup/spec_builders/altair.py b/pysetup/spec_builders/altair.py new file mode 100644 index 0000000000..4b35380de9 --- /dev/null +++ b/pysetup/spec_builders/altair.py @@ -0,0 +1,54 @@ +from typing import Dict + +from .base import BaseSpecBuilder +from ..constants import ALTAIR, OPTIMIZED_BLS_AGGREGATE_PUBKEYS + + +class AltairSpecBuilder(BaseSpecBuilder): + fork: str = ALTAIR + + @classmethod + def imports(cls, preset_name: str) -> str: + return f''' +from typing import NewType, Union as PyUnion + +from eth2spec.phase0 import {preset_name} as phase0 +from eth2spec.test.helpers.merkle import build_proof +from eth2spec.utils.ssz.ssz_typing import Path +''' + + @classmethod + def preparations(cls): + return ''' +SSZVariableName = str +GeneralizedIndex = NewType('GeneralizedIndex', int) +''' + + @classmethod + def sundry_functions(cls) -> str: + return ''' +def get_generalized_index(ssz_class: Any, *path: Sequence[PyUnion[int, SSZVariableName]]) -> GeneralizedIndex: + ssz_path = Path(ssz_class) + for item in path: + ssz_path = ssz_path / item + return GeneralizedIndex(ssz_path.gindex()) + + +def compute_merkle_proof_for_state(state: BeaconState, + index: GeneralizedIndex) -> Sequence[Bytes32]: + return build_proof(state.get_backing(), index)''' + + + @classmethod + def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: + return { + 'FINALIZED_ROOT_INDEX': 'GeneralizedIndex(105)', + 'CURRENT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(54)', + 'NEXT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(55)', + } + + @classmethod + def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]: + if "eth_aggregate_pubkeys" in functions: + functions["eth_aggregate_pubkeys"] = OPTIMIZED_BLS_AGGREGATE_PUBKEYS.strip() + return functions diff --git a/pysetup/spec_builders/base.py b/pysetup/spec_builders/base.py new file mode 100644 index 0000000000..44743682ab --- /dev/null +++ b/pysetup/spec_builders/base.py @@ -0,0 +1,52 @@ +from abc import ABC, abstractmethod +from typing import Sequence, Dict +from pathlib import Path + +class BaseSpecBuilder(ABC): + @property + @abstractmethod + def fork(self) -> str: + raise NotImplementedError() + + @classmethod + def imports(cls, preset_name: str) -> str: + """ + Import objects from other libraries. + """ + return "" + + @classmethod + def preparations(cls) -> str: + """ + Define special types/constants for building pyspec or call functions. + """ + return "" + + @classmethod + def sundry_functions(cls) -> str: + """ + The functions that are (1) defined abstractly in specs or (2) adjusted for getting better performance. + """ + return "" + + @classmethod + def execution_engine_cls(cls) -> str: + return "" + + @classmethod + def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: + """ + The constants that are required for SSZ objects. + """ + return {} + + @classmethod + def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]: # TODO + """ + The constants that are required for custom types. + """ + return {} + + @classmethod + def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]: + return functions diff --git a/pysetup/spec_builders/bellatrix.py b/pysetup/spec_builders/bellatrix.py new file mode 100644 index 0000000000..c5753d7df0 --- /dev/null +++ b/pysetup/spec_builders/bellatrix.py @@ -0,0 +1,66 @@ +from .base import BaseSpecBuilder +from ..constants import BELLATRIX + +class BellatrixSpecBuilder(BaseSpecBuilder): + fork: str = BELLATRIX + + @classmethod + def imports(cls, preset_name: str): + return f''' +from typing import Protocol +from eth2spec.altair import {preset_name} as altair +from eth2spec.utils.ssz.ssz_typing import Bytes8, Bytes20, ByteList, ByteVector +''' + + @classmethod + def sundry_functions(cls) -> str: + return """ +ExecutionState = Any + + +def get_pow_block(hash: Bytes32) -> Optional[PowBlock]: + return PowBlock(block_hash=hash, parent_hash=Bytes32(), total_difficulty=uint256(0)) + + +def get_execution_state(_execution_state_root: Bytes32) -> ExecutionState: + pass + + +def get_pow_chain_head() -> PowBlock: + pass""" + + @classmethod + def execution_engine_cls(cls) -> str: + return """ +class NoopExecutionEngine(ExecutionEngine): + + def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + return True + + def notify_forkchoice_updated(self: ExecutionEngine, + head_block_hash: Hash32, + safe_block_hash: Hash32, + finalized_block_hash: Hash32, + payload_attributes: Optional[PayloadAttributes]) -> Optional[PayloadId]: + pass + + def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: + # pylint: disable=unused-argument + raise NotImplementedError("no default block production") + + def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + return True + + def verify_and_notify_new_payload(self: ExecutionEngine, + new_payload_request: NewPayloadRequest) -> bool: + return True + + +EXECUTION_ENGINE = NoopExecutionEngine()""" + + + @classmethod + def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: + return { + 'MAX_BYTES_PER_TRANSACTION': spec_object.preset_vars['MAX_BYTES_PER_TRANSACTION'].value, + } diff --git a/pysetup/spec_builders/capella.py b/pysetup/spec_builders/capella.py new file mode 100644 index 0000000000..03b619b66a --- /dev/null +++ b/pysetup/spec_builders/capella.py @@ -0,0 +1,29 @@ +from typing import Dict + +from .base import BaseSpecBuilder +from ..constants import CAPELLA + + +class CapellaSpecBuilder(BaseSpecBuilder): + fork: str = CAPELLA + + @classmethod + def imports(cls, preset_name: str): + return f''' +from eth2spec.bellatrix import {preset_name} as bellatrix +''' + + + @classmethod + def sundry_functions(cls) -> str: + return ''' +def compute_merkle_proof_for_block_body(body: BeaconBlockBody, + index: GeneralizedIndex) -> Sequence[Bytes32]: + return build_proof(body.get_backing(), index)''' + + + @classmethod + def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: + return { + 'EXECUTION_PAYLOAD_INDEX': 'GeneralizedIndex(25)', + } diff --git a/pysetup/spec_builders/deneb.py b/pysetup/spec_builders/deneb.py new file mode 100644 index 0000000000..b4e180c2ae --- /dev/null +++ b/pysetup/spec_builders/deneb.py @@ -0,0 +1,71 @@ +from .base import BaseSpecBuilder +from ..constants import DENEB + + +class DenebSpecBuilder(BaseSpecBuilder): + fork: str = DENEB + + @classmethod + def imports(cls, preset_name: str): + return f''' +from eth2spec.capella import {preset_name} as capella +''' + + + @classmethod + def preparations(cls): + return ''' +T = TypeVar('T') # For generic function +''' + + @classmethod + def sundry_functions(cls) -> str: + return ''' +def retrieve_blobs_and_proofs(beacon_block_root: Root) -> PyUnion[Tuple[Blob, KZGProof], Tuple[str, str]]: + # pylint: disable=unused-argument + return ("TEST", "TEST")''' + + @classmethod + def execution_engine_cls(cls) -> str: + return """ +class NoopExecutionEngine(ExecutionEngine): + + def notify_new_payload(self: ExecutionEngine, + execution_payload: ExecutionPayload, + parent_beacon_block_root: Root) -> bool: + return True + + def notify_forkchoice_updated(self: ExecutionEngine, + head_block_hash: Hash32, + safe_block_hash: Hash32, + finalized_block_hash: Hash32, + payload_attributes: Optional[PayloadAttributes]) -> Optional[PayloadId]: + pass + + def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: + # pylint: disable=unused-argument + raise NotImplementedError("no default block production") + + def is_valid_block_hash(self: ExecutionEngine, + execution_payload: ExecutionPayload, + parent_beacon_block_root: Root) -> bool: + return True + + def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: + return True + + def verify_and_notify_new_payload(self: ExecutionEngine, + new_payload_request: NewPayloadRequest) -> bool: + return True + + +EXECUTION_ENGINE = NoopExecutionEngine()""" + + + @classmethod + def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: + return { + 'BYTES_PER_FIELD_ELEMENT': spec_object.constant_vars['BYTES_PER_FIELD_ELEMENT'].value, + 'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value, + 'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value, + } diff --git a/pysetup/spec_builders/eip6110.py b/pysetup/spec_builders/eip6110.py new file mode 100644 index 0000000000..e0fd253f19 --- /dev/null +++ b/pysetup/spec_builders/eip6110.py @@ -0,0 +1,12 @@ +from .base import BaseSpecBuilder +from ..constants import EIP6110 + + +class EIP6110SpecBuilder(BaseSpecBuilder): + fork: str = EIP6110 + + @classmethod + def imports(cls, preset_name: str): + return f''' +from eth2spec.deneb import {preset_name} as deneb +''' diff --git a/pysetup/spec_builders/phase0.py b/pysetup/spec_builders/phase0.py new file mode 100644 index 0000000000..6b3d826179 --- /dev/null +++ b/pysetup/spec_builders/phase0.py @@ -0,0 +1,105 @@ +from .base import BaseSpecBuilder +from ..constants import PHASE0 + + +class Phase0SpecBuilder(BaseSpecBuilder): + fork: str = PHASE0 + + @classmethod + def imports(cls, preset_name: str) -> str: + return '''from lru import LRU +from dataclasses import ( + dataclass, + field, +) +from typing import ( + Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar, NamedTuple, Final +) + +from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes +from eth2spec.utils.ssz.ssz_typing import ( + View, boolean, Container, List, Vector, uint8, uint32, uint64, uint256, + Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist) +from eth2spec.utils.ssz.ssz_typing import Bitvector # noqa: F401 +from eth2spec.utils import bls +from eth2spec.utils.hash_function import hash +''' + + @classmethod + def preparations(cls) -> str: + return ''' +SSZObject = TypeVar('SSZObject', bound=View) +''' + + @classmethod + def sundry_functions(cls) -> str: + return ''' +def get_eth1_data(block: Eth1Block) -> Eth1Data: + """ + A stub function return mocking Eth1Data. + """ + return Eth1Data( + deposit_root=block.deposit_root, + deposit_count=block.deposit_count, + block_hash=hash_tree_root(block)) + + +def cache_this(key_fn, value_fn, lru_size): # type: ignore + cache_dict = LRU(size=lru_size) + + def wrapper(*args, **kw): # type: ignore + key = key_fn(*args, **kw) + nonlocal cache_dict + if key not in cache_dict: + cache_dict[key] = value_fn(*args, **kw) + return cache_dict[key] + return wrapper + + +_compute_shuffled_index = compute_shuffled_index +compute_shuffled_index = cache_this( + lambda index, index_count, seed: (index, index_count, seed), + _compute_shuffled_index, lru_size=SLOTS_PER_EPOCH * 3) + +_get_total_active_balance = get_total_active_balance +get_total_active_balance = cache_this( + lambda state: (state.validators.hash_tree_root(), compute_epoch_at_slot(state.slot)), + _get_total_active_balance, lru_size=10) + +_get_base_reward = get_base_reward +get_base_reward = cache_this( + lambda state, index: (state.validators.hash_tree_root(), state.slot, index), + _get_base_reward, lru_size=2048) + +_get_committee_count_per_slot = get_committee_count_per_slot +get_committee_count_per_slot = cache_this( + lambda state, epoch: (state.validators.hash_tree_root(), epoch), + _get_committee_count_per_slot, lru_size=SLOTS_PER_EPOCH * 3) + +_get_active_validator_indices = get_active_validator_indices +get_active_validator_indices = cache_this( + lambda state, epoch: (state.validators.hash_tree_root(), epoch), + _get_active_validator_indices, lru_size=3) + +_get_beacon_committee = get_beacon_committee +get_beacon_committee = cache_this( + lambda state, slot, index: (state.validators.hash_tree_root(), state.randao_mixes.hash_tree_root(), slot, index), + _get_beacon_committee, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3) + +_get_matching_target_attestations = get_matching_target_attestations +get_matching_target_attestations = cache_this( + lambda state, epoch: (state.hash_tree_root(), epoch), + _get_matching_target_attestations, lru_size=10) + +_get_matching_head_attestations = get_matching_head_attestations +get_matching_head_attestations = cache_this( + lambda state, epoch: (state.hash_tree_root(), epoch), + _get_matching_head_attestations, lru_size=10) + +_get_attesting_indices = get_attesting_indices +get_attesting_indices = cache_this( + lambda state, data, bits: ( + state.randao_mixes.hash_tree_root(), + state.validators.hash_tree_root(), data.hash_tree_root(), bits.hash_tree_root() + ), + _get_attesting_indices, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3)''' diff --git a/pysetup/spec_builders/whisk.py b/pysetup/spec_builders/whisk.py new file mode 100644 index 0000000000..e9cd4a67da --- /dev/null +++ b/pysetup/spec_builders/whisk.py @@ -0,0 +1,20 @@ +from .base import BaseSpecBuilder +from ..constants import WHISK + + +class WhiskSpecBuilder(BaseSpecBuilder): + fork: str = WHISK + + @classmethod + def imports(cls, preset_name: str): + return f''' +from eth2spec.capella import {preset_name} as capella +''' + + @classmethod + def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: + # Necessary for custom types `WhiskShuffleProof` and `WhiskTrackerProof` + return { + 'WHISK_MAX_SHUFFLE_PROOF_SIZE': spec_object.preset_vars['WHISK_MAX_SHUFFLE_PROOF_SIZE'].value, + 'WHISK_MAX_OPENING_PROOF_SIZE': spec_object.preset_vars['WHISK_MAX_OPENING_PROOF_SIZE'].value, + } From 2a02d162813c3a80fd74a54a8dc17620069f05bb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 26 Jun 2023 17:51:00 +0800 Subject: [PATCH 4/4] Update doc --- docs/docs/new-feature.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/new-feature.md b/docs/docs/new-feature.md index 1b7cfd7206..b987e2e97d 100644 --- a/docs/docs/new-feature.md +++ b/docs/docs/new-feature.md @@ -56,8 +56,8 @@ You can refer to the previous fork's `fork.md` file. - Update Pyspec [`constants.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/helpers/constants.py) with the new feature name. - Update helpers for [`setup.py`](https://github.com/ethereum/consensus-specs/blob/dev/setup.py) for building the spec: - Update [`pysetup/constants.py`](https://github.com/ethereum/consensus-specs/blob/dev/constants.py) with the new feature name as Pyspec `constants.py` defined. - - Update [`pysetup/spec_builders.py`](https://github.com/ethereum/consensus-specs/blob/dev/pysetup/spec_builders.py) with a new `SpecBuilder` with the new feature name. e.g., `EIP9999SpecBuilder`. Append it to the `spec_builders` list. - - Update [`pysetup/md_doc_paths.py`](https://github.com/ethereum/consensus-specs/blob/dev/pysetup/md_doc_paths.py): add the path of the new markdown files in `get_md_doc_paths` function. + - Update [`pysetup/spec_builders/__init__.py`](https://github.com/ethereum/consensus-specs/blob/dev/pysetup/spec_builders/__init__.py). Implement a new `SpecBuilder` in `pysetup/spec_builders/.py` with the new feature name. e.g., `EIP9999SpecBuilder`. Append it to the `spec_builders` list. + - Update [`pysetup/md_doc_paths.py`](https://github.com/ethereum/consensus-specs/blob/dev/pysetup/md_doc_paths.py): add the path of the new markdown files in `get_md_doc_paths` function if needed. ## B: Make it executable for pytest and test generator