update test util

Makefile

@@ -96,11 +96,11 @@ install_test:
 
 test: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.altair.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
+	python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.mainnet --cov=eth2spec.altair.mainnet --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
 
 find_test: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.altair.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
+	python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.mainnet --cov=eth2spec.altair.mainnet --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
 
 citest: pyspec
 	mkdir -p tests/core/pyspec/test-reports/eth2spec; . venv/bin/activate; cd $(PY_SPEC_DIR); \

setup.py

@@ -61,7 +61,7 @@ class ProtocolDefinition(NamedTuple):
 
 
 class VariableDefinition(NamedTuple):
-    type_name: str
+    type_name: Optional[str]
     value: str
     comment: Optional[str]  # e.g. "noqa: E501"
 
@@ -145,7 +145,7 @@ def _parse_value(name: str, typed_value: str) -> VariableDefinition:
 
     typed_value = typed_value.strip()
     if '(' not in typed_value:
-        return VariableDefinition(type_name='int', value=typed_value, comment=comment)
+        return VariableDefinition(type_name=None, value=typed_value, comment=comment)
     i = typed_value.index('(')
     type_name = typed_value[:i]
 
@@ -259,7 +259,7 @@ def fork(self) -> str:
 
     @classmethod
     @abstractmethod
-    def imports(cls) -> str:
+    def imports(cls, preset_name: str) -> str:
         """
         Import objects from other libraries.
         """
@@ -307,7 +307,8 @@ def invariant_checks(cls) -> str:
 
     @classmethod
     @abstractmethod
-    def build_spec(cls, source_files: List[Path], preset_files: Sequence[Path], config_file: Path) -> str:
+    def build_spec(cls, preset_name: str,
+                   source_files: List[Path], preset_files: Sequence[Path], config_file: Path) -> str:
         raise NotImplementedError()
 
 
@@ -318,7 +319,7 @@ class Phase0SpecBuilder(SpecBuilder):
     fork: str = PHASE0
 
     @classmethod
-    def imports(cls) -> str:
+    def imports(cls, preset_name: str) -> str:
         return '''from lru import LRU
 from dataclasses import (
     dataclass,
@@ -429,8 +430,9 @@ def invariant_checks(cls) -> str:
         return ''
 
     @classmethod
-    def build_spec(cls, source_files: Sequence[Path], preset_files: Sequence[Path], config_file: Path) -> str:
-        return _build_spec(cls.fork, source_files, preset_files, config_file)
+    def build_spec(cls, preset_name: str,
+                   source_files: Sequence[Path], preset_files: Sequence[Path], config_file: Path) -> str:
+        return _build_spec(preset_name, cls.fork, source_files, preset_files, config_file)
 
 
 #
@@ -440,21 +442,17 @@ class AltairSpecBuilder(Phase0SpecBuilder):
     fork: str = ALTAIR
 
     @classmethod
-    def imports(cls) -> str:
-        return super().imports() + '\n' + '''
+    def imports(cls, preset_name: str) -> str:
+        return super().imports(preset_name) + '\n' + f'''
 from typing import NewType, Union
-from importlib import reload
 
-from eth2spec.phase0 import spec as phase0
+from eth2spec.phase0 import {preset_name} as phase0
 from eth2spec.utils.ssz.ssz_typing import Path
 '''
 
     @classmethod
     def preparations(cls):
         return super().preparations() + '\n' + '''
-# Whenever this spec version is loaded, make sure we have the latest phase0
-reload(phase0)
-
 SSZVariableName = str
 GeneralizedIndex = NewType('GeneralizedIndex', int)
 '''
@@ -492,19 +490,16 @@ class MergeSpecBuilder(Phase0SpecBuilder):
     fork: str = MERGE
 
     @classmethod
-    def imports(cls):
-        return super().imports() + '''
+    def imports(cls, preset_name: str):
+        return super().imports(preset_name) + f'''
 from typing import Protocol
-from eth2spec.phase0 import spec as phase0
+from eth2spec.phase0 import {preset_name} as phase0
 from eth2spec.utils.ssz.ssz_typing import Bytes20, ByteList, ByteVector, uint256
-from importlib import reload
 '''
 
     @classmethod
     def preparations(cls):
-        return super().preparations() + '\n' + '''
-reload(phase0)
-'''
+        return super().preparations()
 
     @classmethod
     def sundry_functions(cls) -> str:
@@ -513,7 +508,7 @@ def sundry_functions(cls) -> str:
 
 
 def get_pow_block(hash: Bytes32) -> PowBlock:
-    return PowBlock(block_hash=hash, is_valid=True, is_processed=True, total_difficulty=TRANSITION_TOTAL_DIFFICULTY)
+    return PowBlock(block_hash=hash, is_valid=True, is_processed=True, total_difficulty=config.TRANSITION_TOTAL_DIFFICULTY)
 
 
 def get_execution_state(execution_state_root: Bytes32) -> ExecutionState:
@@ -556,7 +551,10 @@ def hardcoded_custom_type_dep_constants(cls) -> str:
 }
 
 
-def objects_to_spec(spec_object: SpecObject, builder: SpecBuilder, ordered_class_objects: Dict[str, str]) -> str:
+def objects_to_spec(preset_name: str,
+                    spec_object: SpecObject,
+                    builder: SpecBuilder,
+                    ordered_class_objects: Dict[str, str]) -> str:
     """
     Given all the objects that constitute a spec, combine them into a single pyfile.
     """
@@ -596,19 +594,28 @@ def format_protocol(protocol_name: str, protocol_def: ProtocolDefinition) -> str
         functions_spec = functions_spec.replace(name, 'config.' + name)
 
     def format_config_var(name: str, vardef: VariableDefinition) -> str:
-        out = f'{name}={vardef.type_name}({vardef.value}),'
+        if vardef.type_name is None:
+            out = f'{name}={vardef.value}'
+        else:
+            out = f'{name}={vardef.type_name}({vardef.value}),'
         if vardef.comment is not None:
             out += f'  # {vardef.comment}'
         return out
 
     config_spec = '@dataclass\nclass Configuration(object):\n'
-    config_spec += '\n'.join(f'    {k}: {v.type_name}' for k, v in spec_object.config_vars.items())
+    config_spec += '    PRESET_BASE: str\n'
+    config_spec += '\n'.join(f'    {k}: {v.type_name if v.type_name is not None else "int"}'
+                             for k, v in spec_object.config_vars.items())
     config_spec += '\n\n\nconfig = Configuration(\n'
+    config_spec += f'    PRESET_BASE="{preset_name}",\n'
     config_spec += '\n'.join('    ' + format_config_var(k, v) for k, v in spec_object.config_vars.items())
     config_spec += '\n)\n'
 
     def format_constant(name: str, vardef: VariableDefinition) -> str:
-        out = f'{name} = {vardef.type_name}({vardef.value})'
+        if vardef.type_name is None:
+            out = f'{name} = {vardef.value}'
+        else:
+            out = f'{name} = {vardef.type_name}({vardef.value})'
         if vardef.comment is not None:
             out += f'  # {vardef.comment}'
         return out
@@ -620,7 +627,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
     ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), builder.hardcoded_ssz_dep_constants()))
     custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_custom_type_dep_constants()[x]), builder.hardcoded_custom_type_dep_constants()))
     spec = (
-            builder.imports()
+            builder.imports(preset_name)
             + builder.preparations()
             + '\n\n' + f"fork = \'{builder.fork}\'\n"
             # The constants that some SSZ containers require. Need to be defined before `new_type_definitions`
@@ -785,7 +792,8 @@ def load_config(config_path: Path) -> Dict[str, str]:
     return parse_config_vars(config_data)
 
 
-def _build_spec(fork: str, source_files: Sequence[Path], preset_files: Sequence[Path], config_file: Path) -> str:
+def _build_spec(preset_name: str, fork: str,
+                source_files: Sequence[Path], preset_files: Sequence[Path], config_file: Path) -> str:
     preset = load_preset(preset_files)
     config = load_config(config_file)
     all_specs = [get_spec(spec, preset, config) for spec in source_files]
@@ -797,7 +805,7 @@ def _build_spec(fork: str, source_files: Sequence[Path], preset_files: Sequence[
     class_objects = {**spec_object.ssz_objects, **spec_object.dataclasses}
     dependency_order_class_objects(class_objects, spec_object.custom_types)
 
-    return objects_to_spec(spec_object, spec_builders[fork], class_objects)
+    return objects_to_spec(preset_name, spec_object, spec_builders[fork], class_objects)
 
 
 class BuildTarget(NamedTuple):
@@ -903,7 +911,8 @@ def run(self):
             dir_util.mkpath(self.out_dir)
 
         for (name, preset_paths, config_path) in self.parsed_build_targets:
-            spec_str = spec_builders[self.spec_fork].build_spec(self.parsed_md_doc_paths, preset_paths, config_path)
+            spec_str = spec_builders[self.spec_fork].build_spec(
+                name, self.parsed_md_doc_paths, preset_paths, config_path)
             if self.dry_run:
                 self.announce('dry run successfully prepared contents for spec.'
                               f' out dir: "{self.out_dir}", spec fork: "{self.spec_fork}", build target: "{name}"')

tests/core/pyspec/eth2spec/test/conftest.py

@@ -28,8 +28,8 @@ def ignore():
 
 def pytest_addoption(parser):
     parser.addoption(
-        "--config", action="store", type=str, default="minimal",
-        help="config: make the pyspec use the specified configuration"
+        "--preset", action="store", type=str, default="minimal",
+        help="preset: make the pyspec use the specified preset"
     )
     parser.addoption(
         "--disable-bls", action="store_true", default=False,
@@ -42,18 +42,10 @@ def pytest_addoption(parser):
 
 
 @fixture(autouse=True)
-def config(request):
-    if not config_util.loaded_defaults:
-        config_util.load_defaults(Path("../../../configs"))
-
-    config_flag_value = request.config.getoption("--config")
-    if config_flag_value in ('minimal', 'mainnet'):
-        config_util.prepare_config(config_flag_value)
-    else:
-        # absolute network config path, e.g. run tests with testnet config
-        config_util.prepare_config(Path(config_flag_value))
-    # now that the presets are loaded, reload the specs to apply them
-    context.reload_specs()
+def preset(request):
+    # TODO: apply to tests, see context.py 'with_presets'
+    preset_flag_value = request.config.getoption("--preset")
+    print("preset:", preset_flag_value)
 
 
 @fixture(autouse=True)

tests/core/pyspec/eth2spec/test/context.py

@@ -1,31 +1,23 @@
 import pytest
-
 from copy import deepcopy
-from eth2spec.phase0 import spec as spec_phase0
-from eth2spec.altair import spec as spec_altair
-from eth2spec.merge import spec as spec_merge
+from eth2spec.phase0 import mainnet as spec_phase0_mainnet, minimal as spec_phase0_minimal
+from eth2spec.altair import mainnet as spec_altair_mainnet, minimal as spec_altair_minimal
+from eth2spec.merge import mainnet as spec_merge_mainnet, minimal as spec_merge_minimal
 from eth2spec.utils import bls
 
 from .exceptions import SkippedTest
 from .helpers.constants import (
-    PHASE0, ALTAIR, MERGE,
+    SpecForkName, PresetBaseName,
+    PHASE0, ALTAIR, MERGE, MINIMAL, MAINNET,
     ALL_PHASES, FORKS_BEFORE_ALTAIR, FORKS_BEFORE_MERGE,
 )
 from .helpers.genesis import create_genesis_state
 from .utils import vector_test, with_meta_tags, build_transition_test
 
 from random import Random
-from typing import Any, Callable, Sequence, TypedDict, Protocol
+from typing import Any, Callable, Sequence, TypedDict, Protocol, Dict
 
 from lru import LRU
-from eth2spec.config import config_util
-from importlib import reload
-
-
-def reload_specs():
-    reload(spec_phase0)
-    reload(spec_altair)
-    reload(spec_merge)
 
 
 # TODO: currently phases are defined as python modules.
@@ -48,6 +40,20 @@ class SpecMerge(Spec):
     ...
 
 
+spec_targets: Dict[PresetBaseName, Dict[SpecForkName, Spec]] = {
+    MINIMAL: {
+        PHASE0: spec_phase0_minimal,
+        ALTAIR: spec_altair_minimal,
+        MERGE: spec_merge_minimal,
+    },
+    MAINNET: {
+        PHASE0: spec_phase0_mainnet,
+        ALTAIR: spec_altair_mainnet,
+        MERGE: spec_merge_mainnet,
+    },
+}
+
+
 class SpecForks(TypedDict, total=False):
     PHASE0: SpecPhase0
     ALTAIR: SpecAltair
@@ -73,7 +79,7 @@ def deco(fn):
 
         def entry(*args, spec: Spec, phases: SpecForks, **kw):
             # make a key for the state
-            key = (spec.fork, spec.CONFIG_NAME, spec.__file__, balances_fn, threshold_fn)
+            key = (spec.fork, spec.config.PRESET_BASE, spec.__file__, balances_fn, threshold_fn)
             global _custom_state_cache_dict
             if key not in _custom_state_cache_dict:
                 state = _prepare_state(balances_fn, threshold_fn, spec, phases)
@@ -321,27 +327,32 @@ def wrapper(*args, **kw):
             if other_phases is not None:
                 available_phases |= set(other_phases)
 
+            preset_name = MINIMAL
+            if 'preset' in kw:
+                preset_name = kw.pop('preset')
+            targets = spec_targets[preset_name]
+
             # TODO: test state is dependent on phase0 but is immediately transitioned to later phases.
             #  A new state-creation helper for later phases may be in place, and then tests can run without phase0
             available_phases.add(PHASE0)
 
             # Populate all phases for multi-phase tests
             phase_dir = {}
             if PHASE0 in available_phases:
-                phase_dir[PHASE0] = spec_phase0
+                phase_dir[PHASE0] = targets[PHASE0]
             if ALTAIR in available_phases:
-                phase_dir[ALTAIR] = spec_altair
+                phase_dir[ALTAIR] = targets[ALTAIR]
             if MERGE in available_phases:
-                phase_dir[MERGE] = spec_merge
+                phase_dir[MERGE] = targets[MERGE]
 
             # return is ignored whenever multiple phases are ran.
             # This return is for test generators to emit python generators (yielding test vector outputs)
             if PHASE0 in run_phases:
-                ret = fn(spec=spec_phase0, phases=phase_dir, *args, **kw)
+                ret = fn(spec=targets[PHASE0], phases=phase_dir, *args, **kw)
             if ALTAIR in run_phases:
-                ret = fn(spec=spec_altair, phases=phase_dir, *args, **kw)
+                ret = fn(spec=targets[ALTAIR], phases=phase_dir, *args, **kw)
             if MERGE in run_phases:
-                ret = fn(spec=spec_merge, phases=phase_dir, *args, **kw)
+                ret = fn(spec=targets[MERGE], phases=phase_dir, *args, **kw)
 
             # TODO: merge, sharding, custody_game and das are not executable yet.
             #  Tests that specify these features will not run, and get ignored for these specific phases.
@@ -351,12 +362,12 @@ def wrapper(*args, **kw):
 
 
 def with_presets(preset_bases, reason=None):
-    available_configs = set(preset_bases)
+    available_presets = set(preset_bases)
 
     def decorator(fn):
         def wrapper(*args, spec: Spec, **kw):
-            if spec.PRESET_BASE not in available_configs:
-                message = f"doesn't support this preset base: {spec.PRESET_BASE}."
+            if spec.config.PRESET_BASE not in available_presets:
+                message = f"doesn't support this preset base: {spec.config.PRESET_BASE}."
                 if reason is not None:
                     message = f"{message} Reason: {reason}"
                 dump_skipping_message(message)
@@ -376,13 +387,12 @@ def with_config_overrides(config_overrides):
     def decorator(fn):
         def wrapper(*args, spec: Spec, **kw):
             # remember the old config
-            old_config = config_util.config
+            old_config = spec.config
 
             # apply our overrides to a copy of it, and apply it to the spec
             tmp_config = deepcopy(old_config)
             tmp_config.update(config_overrides)
-            config_util.config = tmp_config
-            reload_specs()  # Note this reloads the same module instance(s) that we passed into the test
+            spec.config = tmp_config
 
             # Run the function
             out = fn(*args, spec=spec, **kw)
@@ -392,8 +402,7 @@ def wrapper(*args, spec: Spec, **kw):
                 yield from out
 
             # Restore the old config and apply it
-            config_util.config = old_config
-            reload_specs()
+            spec.config = old_config
 
         return wrapper
     return decorator