From ad91a835c03248dbb8a220ca157788d3e88853c7 Mon Sep 17 00:00:00 2001 From: Parithosh Jayanthi Date: Wed, 21 Aug 2024 20:15:57 +0200 Subject: [PATCH] Update python to 3.12.4 (#3880) * update python to 3.12 * drop down to 3.12.4 for compatibility * setup rust toolchain for deps * fix typo * fix linter * fixing linter * adding rust dep * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying new image * trying new image * trying new image * trying new image * invalidate cache * bump reqs * remove cache invalidation * fix cache name * update to use latest image --- .circleci/config.yml | 30 ++++++++-------- .github/workflows/generate_vectors.yml | 2 +- .github/workflows/run-tests.yml | 10 ++++-- docker/Dockerfile | 2 +- requirements_preinstallation.txt | 8 ++--- .../gen_helpers/gen_base/gen_runner.py | 4 +-- .../ssz_generic/ssz_basic_vector.py | 14 ++++---- tests/generators/ssz_generic/ssz_bitlist.py | 6 ++-- tests/generators/ssz_generic/ssz_bitvector.py | 4 +-- tests/generators/ssz_generic/ssz_container.py | 35 +++++++++---------- tests/generators/ssz_generic/ssz_uints.py | 6 ++-- 11 files changed, 62 insertions(+), 59 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bf8b7ada8a..94dcefdb6f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,7 +60,7 @@ commands: jobs: checkout_specs: docker: - - image: circleci/python:3.9 + - image: cimg/python:3.12.4 working_directory: ~/specs-repo steps: # Restore git repo at point close to target branch/revision, to speed up checkout @@ -80,7 +80,7 @@ jobs: - ~/specs-repo install_pyspec_test: docker: - - image: circleci/python:3.9 + - image: ethpandaops/circleci-python-rust:latest working_directory: ~/specs-repo steps: - restore_cache: @@ -88,11 +88,11 @@ jobs: - restore_pyspec_cached_venv - run: name: Install pyspec requirements - command: make install_test + command: make install_test && make preinstallation - save_pyspec_cached_venv test-phase0: docker: - - image: circleci/python:3.9 + - image: cimg/python:3.12.4 working_directory: ~/specs-repo steps: - restore_cache: @@ -105,7 +105,7 @@ jobs: path: tests/core/pyspec/test-reports test-altair: docker: - - image: circleci/python:3.9 + - image: cimg/python:3.12.4 working_directory: ~/specs-repo steps: - restore_cache: @@ -118,7 +118,7 @@ jobs: path: tests/core/pyspec/test-reports test-bellatrix: docker: - - image: circleci/python:3.9 + - image: cimg/python:3.12.4 working_directory: ~/specs-repo steps: - restore_cache: @@ -131,7 +131,7 @@ jobs: path: tests/core/pyspec/test-reports test-capella: docker: - - image: circleci/python:3.9 + - image: cimg/python:3.12.4 working_directory: ~/specs-repo steps: - restore_cache: @@ -144,7 +144,7 @@ jobs: path: tests/core/pyspec/test-reports test-deneb: docker: - - image: circleci/python:3.9 + - image: cimg/python:3.12.4 working_directory: ~/specs-repo steps: - restore_cache: @@ -157,7 +157,7 @@ jobs: path: tests/core/pyspec/test-reports test-electra: docker: - - image: circleci/python:3.9 + - image: cimg/python:3.12.4 working_directory: ~/specs-repo steps: - restore_cache: @@ -170,7 +170,7 @@ jobs: path: tests/core/pyspec/test-reports test-whisk: docker: - - image: circleci/python:3.9 + - image: cimg/python:3.12.4 working_directory: ~/specs-repo steps: - restore_cache: @@ -183,7 +183,7 @@ jobs: path: tests/core/pyspec/test-reports test-eip7594: docker: - - image: circleci/python:3.9 + - image: cimg/python:3.12.4 working_directory: ~/specs-repo steps: - restore_cache: @@ -205,7 +205,7 @@ jobs: command: sudo npm install -g doctoc@2.2.0 && make check_toc codespell: docker: - - image: circleci/python:3.9 + - image: cimg/python:3.12.4 working_directory: ~/specs-repo steps: - checkout @@ -214,7 +214,7 @@ jobs: command: pip install 'codespell<3.0.0,>=2.0.0' --user && make codespell lint: docker: - - image: circleci/python:3.9 + - image: cimg/python:3.12.4 working_directory: ~/specs-repo steps: - restore_cache: @@ -270,7 +270,7 @@ jobs: - /nix install_deposit_contract_web3_tester: docker: - - image: circleci/python:3.9 + - image: cimg/python:3.12.4 working_directory: ~/specs-repo steps: - restore_cache: @@ -282,7 +282,7 @@ jobs: - save_deposit_contract_tester_cached_venv test_deposit_contract_web3_tests: docker: - - image: circleci/python:3.9 + - image: cimg/python:3.12.4 working_directory: ~/specs-repo steps: - restore_cache: diff --git a/.github/workflows/generate_vectors.yml b/.github/workflows/generate_vectors.yml index 37468d203d..7a45953b8a 100644 --- a/.github/workflows/generate_vectors.yml +++ b/.github/workflows/generate_vectors.yml @@ -32,7 +32,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12.4' cache: '' - name: Clean up Spec Repository run: | diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index a56270030f..83926c47c5 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -45,7 +45,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12.4' cache: '' - name: Check codespell run: make codespell @@ -55,10 +55,12 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Setup Rust for dependencies + uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Setup Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12.4' cache: '' - name: Install pyspec requirements run: make install_test @@ -76,10 +78,12 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Setup Rust for dependencies + uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Setup Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12.4' cache: '' - name: set TEST_PRESET_TYPE if: github.event.inputs.test_preset_type != '' diff --git a/docker/Dockerfile b/docker/Dockerfile index 7713099316..8ec384499d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ # Rename the build stage from 'base' to 'builder' for clarification and code readability -FROM python:3.11.0-slim-bullseye as builder +FROM python:3.12.4-slim-bullseye as builder ENV DEBIAN_FRONTEND=noninteractive \ WORKDIR=/consensus-specs \ diff --git a/requirements_preinstallation.txt b/requirements_preinstallation.txt index 02f6d2f0b8..1aca6ad0e0 100644 --- a/requirements_preinstallation.txt +++ b/requirements_preinstallation.txt @@ -1,4 +1,4 @@ -pip>=23.1.2 -wheel>=0.40.0 -setuptools>=68.0.0 -pylint>=3.0.0 \ No newline at end of file +pip>=24.0.0 +wheel>=0.44.0 +setuptools>=72.0.0 +pylint>=3.2.0 \ No newline at end of file diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 3ab2e9eea8..85ccec017a 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -132,7 +132,7 @@ def should_skip_case_dir(case_dir, is_force, diagnostics_obj): print(f'Skipping already existing test: {case_dir}') is_skip = True else: - print(f'Warning, output directory {case_dir} already exist,' + print(f'Warning, output directory {case_dir} already exist, ' ' old files will be deleted and it will generate test vector files with the latest version') # Clear the existing case_dir folder shutil.rmtree(case_dir) @@ -356,7 +356,7 @@ def generate_test_vector(test_case, case_dir, log_file, file_mode): test_end = time.time() span = round(test_end - test_start, 2) if span > TIME_THRESHOLD_TO_PRINT: - print(f' - generated in {span} seconds') + print(f'- generated in {span} seconds') return result diff --git a/tests/generators/ssz_generic/ssz_basic_vector.py b/tests/generators/ssz_generic/ssz_basic_vector.py index 51dfd4ba1a..41bd3a7f5c 100644 --- a/tests/generators/ssz_generic/ssz_basic_vector.py +++ b/tests/generators/ssz_generic/ssz_basic_vector.py @@ -33,7 +33,7 @@ def valid_cases(): for length in [1, 2, 3, 4, 5, 8, 16, 31, 512, 513]: for mode in random_modes: yield f'vec_{name}_{length}_{mode.to_name()}', \ - valid_test_case(lambda: basic_vector_case_fn(rng, mode, typ, length)) + valid_test_case(lambda: basic_vector_case_fn(rng, mode, typ, length)) def invalid_cases(): @@ -52,14 +52,14 @@ def invalid_cases(): if length == 1: # empty bytes, no elements. It may seem valid, but empty fixed-size elements are not valid SSZ. yield f'vec_{name}_{length}_{mode.to_name()}_one_less', \ - invalid_test_case(lambda: b"") + invalid_test_case(lambda: b"") else: yield f'vec_{name}_{length}_{mode.to_name()}_one_less', \ - invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length - 1))) + invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length - 1))) yield f'vec_{name}_{length}_{mode.to_name()}_one_more', \ - invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length + 1))) + invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length + 1))) yield f'vec_{name}_{length}_{mode.to_name()}_one_byte_less', \ - invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length))[:-1]) + invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length))[:-1]) yield f'vec_{name}_{length}_{mode.to_name()}_one_byte_more', \ - invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length)) - + serialize(basic_vector_case_fn(rng, mode, uint8, 1))) + invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length)) + + serialize(basic_vector_case_fn(rng, mode, uint8, 1))) diff --git a/tests/generators/ssz_generic/ssz_bitlist.py b/tests/generators/ssz_generic/ssz_bitlist.py index d1a940eee8..8b904dc511 100644 --- a/tests/generators/ssz_generic/ssz_bitlist.py +++ b/tests/generators/ssz_generic/ssz_bitlist.py @@ -22,7 +22,7 @@ def valid_cases(): RandomizationMode.mode_zero, RandomizationMode.mode_max]: yield f'bitlist_{size}_{mode.to_name()}_{variation}', \ - valid_test_case(lambda: bitlist_case_fn(rng, mode, size)) + valid_test_case(lambda: bitlist_case_fn(rng, mode, size)) def invalid_cases(): @@ -33,5 +33,5 @@ def invalid_cases(): for (typ_limit, test_limit) in [(1, 2), (1, 8), (1, 9), (2, 3), (3, 4), (4, 5), (5, 6), (8, 9), (32, 64), (32, 33), (512, 513)]: yield f'bitlist_{typ_limit}_but_{test_limit}', \ - invalid_test_case(lambda: serialize( - bitlist_case_fn(rng, RandomizationMode.mode_max_count, test_limit))) + invalid_test_case(lambda: serialize( + bitlist_case_fn(rng, RandomizationMode.mode_max_count, test_limit))) diff --git a/tests/generators/ssz_generic/ssz_bitvector.py b/tests/generators/ssz_generic/ssz_bitvector.py index d846140680..56e9f86c21 100644 --- a/tests/generators/ssz_generic/ssz_bitvector.py +++ b/tests/generators/ssz_generic/ssz_bitvector.py @@ -38,5 +38,5 @@ def invalid_cases(): (5, 6), (8, 9), (9, 8), (16, 8), (32, 33), (512, 513)]: for mode in [RandomizationMode.mode_random, RandomizationMode.mode_zero, RandomizationMode.mode_max]: yield f'bitvec_{typ_size}_{mode.to_name()}_{test_size}', \ - invalid_test_case(lambda: serialize(bitvector_case_fn(rng, mode, test_size, - invalid_making_pos=typ_size))) + invalid_test_case(lambda: serialize(bitvector_case_fn(rng, mode, test_size, + invalid_making_pos=typ_size))) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 2c1d37da88..b1a5f026b5 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -46,7 +46,7 @@ class BitsStruct(Container): E: Bitvector[8] -def container_case_fn(rng: Random, mode: RandomizationMode, typ: Type[View], chaos: bool=False): +def container_case_fn(rng: Random, mode: RandomizationMode, typ: Type[View], chaos: bool = False): return get_random_ssz_object(rng, typ, max_bytes_length=2000, max_list_length=2000, @@ -77,7 +77,7 @@ def valid_cases(): for mode in modes: for variation in range(3): yield f'{name}_{mode.to_name()}_chaos_{variation}', \ - valid_test_case(lambda: container_case_fn(rng, mode, typ, chaos=True)) + valid_test_case(lambda: container_case_fn(rng, mode, typ, chaos=True)) # Notes: Below is the second wave of iteration, and only the random mode is selected # for container without offset since ``RandomizationMode.mode_zero`` and ``RandomizationMode.mode_max`` # are deterministic. @@ -85,7 +85,7 @@ def valid_cases(): for mode in modes: for variation in range(10): yield f'{name}_{mode.to_name()}_{variation}', \ - valid_test_case(lambda: container_case_fn(rng, mode, typ)) + valid_test_case(lambda: container_case_fn(rng, mode, typ)) def mod_offset(b: bytes, offset_index: int, change: Callable[[int], int]): @@ -100,8 +100,8 @@ def invalid_cases(): for (name, (typ, offsets)) in PRESET_CONTAINERS.items(): # using mode_max_count, so that the extra byte cannot be picked up as normal list content yield f'{name}_extra_byte', \ - invalid_test_case(lambda: serialize( - container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff') + invalid_test_case(lambda: serialize( + container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff') if len(offsets) != 0: # Note: there are many more ways to have invalid offsets, @@ -112,23 +112,22 @@ def invalid_cases(): RandomizationMode.mode_max_count]: for index, offset_index in enumerate(offsets): yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \ - invalid_test_case(lambda: mod_offset( - b=serialize(container_case_fn(rng, mode, typ)), - offset_index=offset_index, - change=lambda x: x + 1 - )) + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x + 1)) yield f'{name}_{mode.to_name()}_offset_{offset_index}_zeroed', \ - invalid_test_case(lambda: mod_offset( - b=serialize(container_case_fn(rng, mode, typ)), - offset_index=offset_index, - change=lambda x: 0 - )) + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: 0 + )) if index == 0: yield f'{name}_{mode.to_name()}_offset_{offset_index}_minus_one', \ invalid_test_case(lambda: mod_offset( - b=serialize(container_case_fn(rng, mode, typ)), - offset_index=offset_index, - change=lambda x: x - 1 + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x - 1 )) if mode == RandomizationMode.mode_max_count: serialized = serialize(container_case_fn(rng, mode, typ)) diff --git a/tests/generators/ssz_generic/ssz_uints.py b/tests/generators/ssz_generic/ssz_uints.py index abf7fc75b2..32e54353d3 100644 --- a/tests/generators/ssz_generic/ssz_uints.py +++ b/tests/generators/ssz_generic/ssz_uints.py @@ -21,7 +21,7 @@ def valid_cases(): mode = RandomizationMode.mode_random byte_len = uint_type.type_byte_length() yield f'uint_{byte_len * 8}_last_byte_empty', \ - valid_test_case(lambda: uint_type((2 ** ((byte_len - 1) * 8)) - 1)) + valid_test_case(lambda: uint_type((2 ** ((byte_len - 1) * 8)) - 1)) for variation in range(5): yield f'uint_{byte_len * 8}_{mode.to_name()}_{variation}', \ valid_test_case(lambda: uint_case_fn(rng, mode, uint_type)) @@ -38,8 +38,8 @@ def invalid_cases(): for uint_type in [uint8, uint16, uint32, uint64, uint128, uint256]: byte_len = uint_type.type_byte_length() yield f'uint_{byte_len * 8}_one_byte_longer', \ - invalid_test_case(lambda: (2 ** (byte_len * 8) - 1).to_bytes(byte_len + 1, 'little')) + invalid_test_case(lambda: (2 ** (byte_len * 8) - 1).to_bytes(byte_len + 1, 'little')) for uint_type in [uint8, uint16, uint32, uint64, uint128, uint256]: byte_len = uint_type.type_byte_length() yield f'uint_{byte_len * 8}_one_byte_shorter', \ - invalid_test_case(lambda: (2 ** ((byte_len - 1) * 8) - 1).to_bytes(byte_len - 1, 'little')) + invalid_test_case(lambda: (2 ** ((byte_len - 1) * 8) - 1).to_bytes(byte_len - 1, 'little'))