feat(fill): add solc plugin and automatic installation of solc (#772)

ethereum · Aug 29, 2024 · bc820c6 · bc820c6
1 parent c4aa83c
commit bc820c6
Show file tree

Hide file tree

Showing 15 changed files with 386 additions and 86 deletions.
diff --git a/.github/actions/build-fixtures/action.yaml b/.github/actions/build-fixtures/action.yaml
@@ -30,8 +30,7 @@ runs:
         python -m venv env
         source env/bin/activate
         pip install -e .
-        solc-select use  ${{ steps.properties.outputs.solc }} --always-install
-        fill -n ${{ steps.evm-builder.outputs.x-dist }} --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }} --output=fixtures_${{ inputs.name }}.tar.gz --build-name ${{ inputs.name }}
+        fill -n ${{ steps.evm-builder.outputs.x-dist }} --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} --solc-version=${{ steps.properties.outputs.solc }} ${{ steps.properties.outputs.fill-params }} --output=fixtures_${{ inputs.name }}.tar.gz --build-name ${{ inputs.name }}
     - uses: actions/upload-artifact@v4
       with:
         name: fixtures_${{ inputs.name }}

diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml
@@ -3,7 +3,7 @@ name: Evmone Coverage Report
 on:
   pull_request:
     paths:
-      - 'converted-ethereum-tests.txt' # This triggers the workflow only for changes in file.txt
+      - "converted-ethereum-tests.txt" # This triggers the workflow only for changes in file.txt
 
 jobs:
   evmone-coverage-diff:
@@ -37,20 +37,19 @@ jobs:
           python3 -m venv ./venv/
           source ./venv/bin/activate
           pip install -e .
-          solc-select use 0.8.25 --always-install
 
       # Required to fill .py tests
       - name: Build GO EVM
         uses: ./.github/actions/build-evm-client/geth
         id: evm-builder
         with:
-          type: 'main'
+          type: "main"
 
       - name: Build EVMONE EVM
         uses: ./.github/actions/build-evm-client/evmone
         id: evm-builder2
         with:
-          type: 'main'
+          type: "main"
 
       - name: Checkout ethereum/tests
         uses: actions/checkout@v4
@@ -69,7 +68,6 @@ jobs:
           sparse-checkout: |
             Cancun/GeneralStateTests
 
-
       # This command diffs the file and filters in new lines
       - name: Parse converted tests from converted-ethereum-tests.txt
         run: |
@@ -78,7 +76,7 @@ jobs:
           files=$(echo "$lines" | grep -oP '(?<=\+).+\.json')
 
           if [ -z "$files" ]; then
-             echo "Error: No new JSON files found in converted-ethereum-tests.txt"
+              echo "Error: No new JSON files found in converted-ethereum-tests.txt"
               exit 1
           fi
 
@@ -114,13 +112,13 @@ jobs:
               fi
 
               if [ $file_found -eq 0 ]; then
-               echo "Error: Failed to find the test file $file in test repo"
+                echo "Error: Failed to find the test file $file in test repo"
                 exit 1
               fi
           done
           
 
-     # This command diffs the .py scripts introduced by a PR
+      # This command diffs the .py scripts introduced by a PR
       - name: Parse and fill introduced test sources
         run: |
           python3 -m venv ./venv/
@@ -153,12 +151,12 @@ jobs:
           mkdir -p fixtures/eof_tests
           echo "$files" | while read line; do
             file=$(echo "$line" | cut -c 3-)
-            fill $file --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1
-            (fill $file --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1
+            fill $file --until=Cancun --evm-bin evmone-t8n --solc-version=0.8.25 || true >> filloutput.log 2>&1
+            (fill $file --fork=CancunEIP7692 --evm-bin evmone-t8n --solc-version=0.8.25 -k eof_test || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1
           done
 
           if grep -q "FAILURES" filloutput.log; then
-             echo "Error: failed to generate .py tests."
+              echo "Error: failed to generate .py tests."
               exit 1
           fi
           if [ "${{ matrix.driver }}" = "retesteth" ] && grep -q "passed" filloutputEOF.log; then
@@ -174,7 +172,7 @@ jobs:
           filesEOF=$(find fixtures/eof_tests -type f -name "*.json")
           if [ -z "$filesState" ] && [ -z "$filesEOF" ]; then
               echo "Error: No filled JSON fixtures found in fixtures."
-               exit 1
+              exit 1
           fi
 
           PATCH_TEST_PATH=${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS

diff --git a/README.md b/README.md
@@ -82,20 +82,19 @@ The following requires a Python 3.10, 3.11 or 3.12 installation.
 
 ### Quick Start
 
-This guide installs stable versions of the required external (go-ethereum) `evm` and `solc` executables and will only enable generation of test fixtures for features deployed to mainnet. In order to generate fixtures for features under active development, you can follow the steps below and then follow the [additional steps in the online doc](https://ethereum.github.io/execution-spec-tests/getting_started/executing_tests_dev_fork/).
+This guide installs stable versions of the external (go-ethereum) `evm` executable and will only enable generation of test fixtures for features deployed to mainnet. In order to generate fixtures for features under active development, you can follow the steps below and then follow the [additional steps in the online doc](https://ethereum.github.io/execution-spec-tests/getting_started/executing_tests_dev_fork/).
 
-1. Ensure go-ethereum's `evm` tool and `solc` ([0.8.20](https://github.com/ethereum/solidity/releases/tag/v0.8.20), [0.8.21](https://github.com/ethereum/solidity/releases/tag/v0.8.21), [0.8.22](https://github.com/ethereum/solidity/releases/tag/v0.8.22), [0.8.23](https://github.com/ethereum/solidity/releases/tag/v0.8.23)  supported) are in your path. Either build the required versions, or alternatively:
+1. Ensure go-ethereum's `evm` tool is in your path. Either build the required version, or alternatively:
 
     ```console
     sudo add-apt-repository -y ppa:ethereum/ethereum
     sudo apt-get update
-    sudo apt-get install ethereum solc
+    sudo apt-get install ethereum
     ```
 
     More help:
 
     - [geth installation doc](https://geth.ethereum.org/docs/getting-started/installing-geth#ubuntu-via-ppas).
-    - [solc installation doc](https://docs.soliditylang.org/en/latest/installing-solidity.html#linux-packages).
 
     Help for other platforms is available in the [online doc](https://ethereum.github.io/execution-spec-tests/getting_started/quick_start/).
 
@@ -129,7 +128,7 @@ This guide installs stable versions of the required external (go-ethereum) `evm`
           ![Screenshot of pytest test collection console output](docs/getting_started/img/pytest_run_example.png)
         Check:
 
-        1. The versions of the `evm` and `solc` tools are as expected (your versions may differ from those in the highlighted box).
+        1. The versions of the `evm` tool is as expected (your versions may differ from those in the highlighted box).
         2. The corresponding fixture file has been generated:
 
            ```console

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -32,6 +32,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 - ✨ Added `Container.Init` to `ethereum_test_types.EOF.V1` package, which allows generation of an EOF init container more easily ([#739](https://github.com/ethereum/execution-spec-tests/pull/739)).
 - ✨ Introduce method valid_opcodes() to the fork class ([#748](https://github.com/ethereum/execution-spec-tests/pull/748)).
 - 🐞 Fixed `consume` exit code return values, ensuring that pytest's return value is correctly propagated to the shell. This allows the shell to accurately reflect the test results (e.g., failures) based on the pytest exit code ([#765](https://github.com/ethereum/execution-spec-tests/pull/765)).
+- ✨ Added a new flag `--solc-version` to the `fill` command, which allows the user to specify the version of the Solidity compiler to use when compiling Yul source code; this version will now be automatically downloaded by `fill` via [`solc-select`](https://github.com/crytic/solc-select) ([#772](https://github.com/ethereum/execution-spec-tests/pull/772).
 
 ### 🔧 EVM Tools
 

diff --git a/docs/getting_started/executing_tests_command_line.md b/docs/getting_started/executing_tests_command_line.md
@@ -114,17 +114,36 @@ fill --help
 Output:
 
 ```text
-usage: fill [-h] [--evm-bin EVM_BIN] [--traces] [--verify-fixtures]
-            [--verify-fixtures-bin VERIFY_FIXTURES_BIN] [--solc-bin SOLC_BIN]
+usage: fill [-h] [--strict-alloc] [--ca-start CA_START] [--ca-incr CA_INCR]
+            [--evm-code-type EVM_CODE_TYPE] [--solc-bin SOLC_BIN]
+            [--solc-version SOLC_VERSION] [--evm-bin EVM_BIN] [--traces]
+            [--verify-fixtures] [--verify-fixtures-bin VERIFY_FIXTURES_BIN]
             [--filler-path FILLER_PATH] [--output OUTPUT] [--flat-output]
-            [--single-fixture-per-file] [--no-html] [--strict-alloc]
-            [--ca-start CA_START] [--ca-incr CA_INCR] [--build-name BUILD_NAME]
-            [--evm-dump-dir EVM_DUMP_DIR] [--forks] [--fork FORK] [--from FROM]
-            [--until UNTIL] [--test-help]
+            [--single-fixture-per-file] [--no-html] [--build-name BUILD_NAME]
+            [--index] [--evm-dump-dir EVM_DUMP_DIR] [--forks] [--fork FORK]
+            [--from FROM] [--until UNTIL] [--test-help]
 
 options:
   -h, --help            show this help message and exit
 
+Arguments defining pre-allocation behavior.:
+  --strict-alloc        [DEBUG ONLY] Disallows deploying a contract in a
+                        predefined address.
+  --ca-start CA_START, --contract-address-start CA_START
+                        The starting address from which tests will deploy
+                        contracts.
+  --ca-incr CA_INCR, --contract-address-increment CA_INCR
+                        The address increment value to each deployed contract by
+                        a test.
+  --evm-code-type EVM_CODE_TYPE
+                        Type of EVM code to deploy in each test by default.
+
+Arguments defining the solc executable:
+  --solc-bin SOLC_BIN   Path to a solc executable (for Yul source compilation).
+                        No default; if unspecified `--solc-version` is used.
+  --solc-version SOLC_VERSION
+                        Version of the solc compiler to use. Default: 0.8.24.
+
 Arguments defining evm executable behavior:
   --evm-bin EVM_BIN     Path to an evm executable that provides `t8n`. Default:
                         First 'evm' entry in PATH.
@@ -140,18 +159,14 @@ Arguments defining evm executable behavior:
                         Path to an evm executable that provides the `blocktest`
                         command. Default: The first (geth) 'evm' entry in PATH.
 
-Arguments defining the solc executable:
-  --solc-bin SOLC_BIN   Path to a solc executable (for Yul source compilation).
-                        Default: First 'solc' entry in PATH.
-
 Arguments defining filler location and output:
   --filler-path FILLER_PATH
                         Path to filler directives
-  --output OUTPUT       Directory path to store the generated test fixtures. Can
-                        be deleted. If the specified path ends in '.tar.gz', then
-                        the specified tarball is additionally created (the
-                        fixtures are still written to the specified path without
-                        '.tar.gz' suffix). Default: './fixtures'.
+  --output OUTPUT       Directory path to store the generated test fixtures. If
+                        the specified path ends in '.tar.gz', then the specified
+                        tarball is additionally created (the fixtures are still
+                        written to the specified path without the '.tar.gz'
+                        suffix). Can be deleted. Default: './fixtures'.
   --flat-output         Output each test case in the directory without the folder
                         structure.
   --single-fixture-per-file
@@ -161,17 +176,10 @@ Arguments defining filler location and output:
   --no-html             Don't generate an HTML test report (in the output
                         directory). The --html flag can be used to specify a
                         different path.
-  --strict-alloc        [DEBUG ONLY] Disallows deploying a contract in a
-                        predefined address.
-  --ca-start CA_START, --contract-address-start CA_START
-                        The starting address from which tests will deploy
-                        contracts.
-  --ca-incr CA_INCR, --contract-address-increment CA_INCR
-                        The address increment value to each deployed contract by
-                        a test.
   --build-name BUILD_NAME
                         Specify a build name for the fixtures.ini file, e.g.,
                         'stable'.
+  --index               Generate an index file for all produced fixtures.
 
 Arguments defining debug behavior:
   --evm-dump-dir EVM_DUMP_DIR, --t8n-dump-dir EVM_DUMP_DIR

diff --git a/docs/getting_started/quick_start.md b/docs/getting_started/quick_start.md
@@ -1,25 +1,24 @@
 # Quick Start
 
 !!! info "Testing features under active development"
-    The EVM features under test must be implemented in the `evm` tool and `solc` executables that are used by the execution-spec-tests framework. The following guide installs stable versions of these tools.
+    The EVM features under test must be implemented in the `evm` tool and `solc` executables that are used by the execution-spec-tests framework. The following guide installs the stable version of the geth `evm`; `solc` will be installed by the `fill` command.
 
     To test features under active development, start with this base configuration and then follow the steps in [executing tests for features under development](./executing_tests_dev_fork.md). 
 
 The following requires a Python 3.10, 3.11 or 3.12 installation.
 
-1. Ensure `go-ethereum`'s `evm` tool and `solc` ([0.8.20](https://github.com/ethereum/solidity/releases/tag/v0.8.20), [0.8.21](https://github.com/ethereum/solidity/releases/tag/v0.8.21), [0.8.22](https://github.com/ethereum/solidity/releases/tag/v0.8.22), [0.8.23](https://github.com/ethereum/solidity/releases/tag/v0.8.23)  supported) are in your path. Either build the required versions, or alternatively:
+1. Ensure `go-ethereum`'s `evm` tool is in your path. Either build the required version, or alternatively:
 
     === "Ubuntu"
 
           ```console
           sudo add-apt-repository -y ppa:ethereum/ethereum
           sudo apt-get update
-          sudo apt-get install ethereum solc
+          sudo apt-get install ethereum
           ```
           More help:
 
           - [geth installation doc](https://geth.ethereum.org/docs/getting-started/installing-geth#ubuntu-via-ppas).
-          - [solc installation doc](https://docs.soliditylang.org/en/latest/installing-solidity.html#linux-packages).
 
     === "macOS"
 
@@ -32,7 +31,6 @@ The following requires a Python 3.10, 3.11 or 3.12 installation.
           More help:
 
           - [geth installation doc](https://geth.ethereum.org/docs/getting-started/installing-geth#macos-via-homebrew).
-          - [solc installation doc](https://docs.soliditylang.org/en/latest/installing-solidity.html#macos-packages).
 
     === "Windows"
 
@@ -44,7 +42,6 @@ The following requires a Python 3.10, 3.11 or 3.12 installation.
           More help:
 
           - [geth installation doc](https://geth.ethereum.org/docs/getting-started/installing-geth#windows).
-          - [solc static binaries doc](https://docs.soliditylang.org/en/latest/installing-solidity.html#static-binaries).
 
 2. Clone the [execution-spec-tests](https://github.com/ethereum/execution-spec-tests) repo and install its dependencies (it's recommended to use a virtual environment for the installation):
 
@@ -80,7 +77,7 @@ The following requires a Python 3.10, 3.11 or 3.12 installation.
         </figure>
         Check:
 
-        1. The versions of the `evm` and `solc` tools are as expected (your versions may differ from those in the highlighted box).
+        1. The versions of the `evm` tool is as expected (your versions may differ from those in the highlighted box).
         2. The generated HTML test report by clicking the link at the bottom of the console output.
         3. The corresponding fixture file has been generated:
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -35,7 +35,7 @@ dependencies = [
     "semver>=3.0.1,<4",
     "pydantic>=2.8.0,<2.9",
     "rich>=13.7.0,<14",
-    "solc-select>=1.0.4",
+    "solc-select>=1.0.4,<2",
     "filelock>=3.15.1,<4",
 ]
 

diff --git a/pytest-framework.ini b/pytest-framework.ini
@@ -5,6 +5,8 @@ python_files=
     test_*.py
 testpaths =
     src
+markers =
+    run_in_serial
 addopts = 
     -p pytester
     --ignore=src/pytest_plugins/consume/direct/test_via_direct.py

diff --git a/pytest.ini b/pytest.ini
@@ -8,6 +8,7 @@ markers =
     pre_alloc_modify
 addopts = 
     -p pytest_plugins.filler.pre_alloc
+    -p pytest_plugins.filler.solc
     -p pytest_plugins.filler.filler
     -p pytest_plugins.forks.forks
     -p pytest_plugins.spec_version_checker.spec_version_checker

diff --git a/src/pytest_plugins/filler/filler.py b/src/pytest_plugins/filler/filler.py
@@ -5,6 +5,7 @@
 and that modifies pytest hooks in order to fill test specs for all tests and
 writes the generated fixtures to file.
 """
+
 import argparse
 import configparser
 import datetime
@@ -23,14 +24,12 @@
 from ethereum_test_fixtures import FixtureCollector, FixtureFormats, TestInfo
 from ethereum_test_forks import (
     Fork,
-    Frontier,
     Paris,
     get_closest_fork_with_solc_support,
     get_forks_with_solc_support,
 )
 from ethereum_test_specs import SPEC_TYPES, BaseTest
 from ethereum_test_tools import Yul
-from ethereum_test_tools.code import Solc
 from ethereum_test_tools.utility.versioning import (
     generate_github_url,
     get_current_commit_hash_or_tag,
@@ -117,18 +116,6 @@ def pytest_addoption(parser: pytest.Parser):
         ),
     )
 
-    solc_group = parser.getgroup("solc", "Arguments defining the solc executable")
-    solc_group.addoption(
-        "--solc-bin",
-        action="store",
-        dest="solc_bin",
-        default=None,
-        help=(
-            "Path to a solc executable (for Yul source compilation). "
-            "Default: First 'solc' entry in PATH."
-        ),
-    )
-
     test_group = parser.getgroup("tests", "Arguments defining filler location and output")
     test_group.addoption(
         "--filler-path",
@@ -269,18 +256,14 @@ def pytest_configure(config):
             "The Besu t8n tool does not work well with the xdist plugin; use -n=0.",
             returncode=pytest.ExitCode.USAGE_ERROR,
         )
-    config.solc_version = Solc(config.getoption("solc_bin")).version
-    if config.solc_version < Frontier.solc_min_version():
-        pytest.exit(
-            f"Unsupported solc version: {config.solc_version}. Minimum required version is "
-            f"{Frontier.solc_min_version()}",
-            returncode=pytest.ExitCode.USAGE_ERROR,
-        )
 
-    config.stash[metadata_key]["Tools"] = {
-        "t8n": t8n.version(),
-        "solc": str(config.solc_version),
-    }
+    if "Tools" not in config.stash[metadata_key]:
+        config.stash[metadata_key]["Tools"] = {
+            "t8n": t8n.version(),
+        }
+    else:
+        config.stash[metadata_key]["Tools"]["t8n"] = t8n.version()
+
     args = ["fill"] + [str(arg) for arg in config.invocation_params.args]
     for i in range(len(args)):
         if " " in args[i]:
@@ -419,14 +402,6 @@ def verify_fixtures_bin(request: pytest.FixtureRequest) -> Path | None:
     return request.config.getoption("verify_fixtures_bin")
 
 
-@pytest.fixture(autouse=True, scope="session")
-def solc_bin(request: pytest.FixtureRequest):
-    """
-    Returns the configured solc binary path.
-    """
-    return request.config.getoption("solc_bin")
-
-
 @pytest.fixture(autouse=True, scope="session")
 def t8n(request: pytest.FixtureRequest, evm_bin: Path) -> Generator[TransitionTool, None, None]:
     """