feat(consume): Add consume cache, and --input flag accepts versioned release names

docs/CHANGELOG.md

@@ -67,6 +67,8 @@ Test fixtures for use by clients are available for each release on the [Github r
 - ✨ Introduce [`fork_covariant_parametrize`](https://ethereum.github.io/execution-spec-tests/main/writing_tests/test_markers/#custom-fork-covariant-markers) helper function ([#1019](https://github.com/ethereum/execution-spec-tests/pull/1019)).
 - 🔀 Update EIP-7251 according to [spec updates](https://github.com/ethereum/EIPs/pull/9127) ([#1024](https://github.com/ethereum/execution-spec-tests/pull/1024))
 - 🔀 Update EIP-7002 according to [spec updates](https://github.com/ethereum/EIPs/pull/9119) ([#1024](https://github.com/ethereum/execution-spec-tests/pull/1024))
+- ✨ Add the `consume cache` command to cache fixtures before running consume commands ([#1044](https://github.com/ethereum/execution-spec-tests/pull/1044)).
+- ✨ The `--input` flag of the consume commands now supports parsing of tagged release names in the format `<RELEASE_NAME>@<RELEASE_VERSION>` ([#1044](https://github.com/ethereum/execution-spec-tests/pull/1044)).
 
 ### 🔧 EVM Tools
 
@@ -91,6 +93,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 - The EOF fixture format contained in `eof_tests` may now contain multiple exceptions in the `"exception"` field in the form of a pipe (`|`) separated string ([#759](https://github.com/ethereum/execution-spec-tests/pull/759)).
 - Remove redundant tests within stable and develop fixture releases, moving them to a separate legacy release ([#788](https://github.com/ethereum/execution-spec-tests/pull/788)).
 - Ruff now replaces Flake8, Isort and Black resulting in significant changes to the entire code base including its usage ([#922](https://github.com/ethereum/execution-spec-tests/pull/922)).
+- `latest-stable-release` and `latest-develop-release` keywords for the `--input` flag in consume commands have been replaced with `stable@latest` and `develop@latest` respectively ([#1044](https://github.com/ethereum/execution-spec-tests/pull/1044)).
 
 ## [v3.0.0](https://github.com/ethereum/execution-spec-tests/releases/tag/v3.0.0) - 2024-07-22
 

docs/consuming_tests/index.md

@@ -30,7 +30,7 @@ The @ethereum/execution-spec-tests repository provides [releases](https://github
 
 | Release Artifact               | Consumer | Fork/feature scope |
 | ------------------------------ | -------- | ------------------ |
-| `fixtures.tar.gz`              | Clients  | All tests until the last stable fork ("must pass") |
+| `fixtures_stable.tar.gz`       | Clients  | All tests until the last stable fork ("must pass") |
 | `fixtures_develop.tar.gz`      | Clients  | All tests until the last development fork |
 
 ## Obtaining the Most Recent Release Artifacts

pytest-framework.ini

@@ -10,8 +10,10 @@ markers =
 addopts = 
     -p pytester
     -p pytest_plugins.eels_resolver
-    --ignore=src/pytest_plugins/consume/
+    --ignore=src/pytest_plugins/consume/test_cache.py
+    --ignore=src/pytest_plugins/consume/direct/
     --ignore=src/pytest_plugins/consume/direct/test_via_direct.py
+    --ignore=src/pytest_plugins/consume/hive_simulators/
     --ignore=src/pytest_plugins/consume/hive_simulators/engine/test_via_engine.py
     --ignore=src/pytest_plugins/consume/hive_simulators/rlp/test_via_rlp.py
     --ignore=src/pytest_plugins/execute/test_recover.py

src/cli/pytest_commands/consume.py

@@ -57,6 +57,12 @@ def get_command_paths(command_name: str, is_hive: bool) -> List[Path]:
     return command_paths
 
 
+@click.group(context_settings={"help_option_names": ["-h", "--help"]})
+def consume() -> None:
+    """Consume command to aid client consumption of test fixtures."""
+    pass
+
+
 def consume_command(is_hive: bool = False) -> Callable[[Callable[..., Any]], click.Command]:
     """Generate a consume sub-command."""
 
@@ -93,16 +99,6 @@ def decorator(func: Callable[..., Any]) -> click.Command:
     return decorator
 
 
-@click.group(
-    context_settings={
-        "help_option_names": ["-h", "--help"],
-    }
-)
-def consume() -> None:
-    """Consume command to aid client consumption of test fixtures."""
-    pass
-
-
 @consume_command(is_hive=False)
 def direct() -> None:
     """Clients consume directly via the `blocktest` interface."""
@@ -125,3 +121,15 @@ def engine() -> None:
 def hive() -> None:
     """Client consumes via all available hive methods (rlp, engine)."""
     pass
+
+
+@consume.command(
+    context_settings={"ignore_unknown_options": True},
+)
+@common_click_options
+def cache(pytest_args: List[str], **kwargs) -> None:
+    """Consume command to cache test fixtures."""
+    args = handle_consume_command_flags(pytest_args, is_hive=False)
+    args += ["src/pytest_plugins/consume/test_cache.py"]
+    args += ["--cache-only"]
+    sys.exit(pytest.main(args))

src/pytest_plugins/consume/consume.py

@@ -1,12 +1,13 @@
 """A pytest plugin providing common functionality for consuming test fixtures."""
 
-import os
 import sys
 import tarfile
+from io import BytesIO
 from pathlib import Path
-from typing import Literal, Union
+from typing import List, Literal, Union
 from urllib.parse import urlparse
 
+import platformdirs
 import pytest
 import requests
 import rich
@@ -15,9 +16,13 @@
 from ethereum_test_fixtures.consume import TestCases
 from ethereum_test_tools.utility.versioning import get_current_commit_hash_or_tag
 
-cached_downloads_directory = Path("./cached_downloads")
+from .releases import ReleaseTag, get_release_url
 
-JsonSource = Union[Path, Literal["stdin"]]
+CACHED_DOWNLOADS_DIRECTORY = (
+    Path(platformdirs.user_cache_dir("ethereum-execution-spec-tests")) / "cached_downloads"
+)
+
+FixturesSource = Union[Path, Literal["stdin"]]
 
 
 def default_input_directory() -> str:
@@ -33,7 +38,7 @@ def default_html_report_file_path() -> str:
     Filepath (default) to store the generated HTML test report. Defined as a
     function to allow for easier testing.
     """
-    return ".meta/report_consume.html"
+    return "./report_consume.html"
 
 
 def is_url(string: str) -> bool:
@@ -57,11 +62,7 @@ def download_and_extract(url: str, base_directory: Path) -> Path:
     response = requests.get(url)
     response.raise_for_status()
 
-    archive_path = extract_to / filename
-    with open(archive_path, "wb") as file:
-        file.write(response.content)
-
-    with tarfile.open(archive_path, "r:gz") as tar:
+    with tarfile.open(fileobj=BytesIO(response.content), mode="r:gz") as tar:  # noqa: SC200
         tar.extractall(path=extract_to)
 
     return extract_to / "fixtures"
@@ -74,12 +75,13 @@ def pytest_addoption(parser):  # noqa: D103
     consume_group.addoption(
         "--input",
         action="store",
-        dest="fixture_source",
-        default=default_input_directory(),
+        dest="fixtures_source",
+        default=None,
         help=(
             "Specify the JSON test fixtures source. Can be a local directory, a URL pointing to a "
-            " fixtures.tar.gz archive, or one of the special keywords: 'stdin', "
-            "'latest-stable', 'latest-develop'. "
+            " fixtures.tar.gz archive, a release name and version in the form of `NAME@v1.2.3` "
+            "(`stable` and `develop` are valid release names, and `latest` is a valid version), "
+            "or the special keyword 'stdin'. "
             f"Defaults to the following local directory: '{default_input_directory()}'."
         ),
     )
@@ -100,6 +102,23 @@ def pytest_addoption(parser):  # noqa: D103
             "The --html flag can be used to specify a different path."
         ),
     )
+    consume_group.addoption(
+        "--cache-folder",
+        action="store",
+        dest="fixture_cache_folder",
+        default=CACHED_DOWNLOADS_DIRECTORY,
+        help=(
+            "Specify the path where the downloaded fixtures should be cached. "
+            f"Defaults to the following directory: '{CACHED_DOWNLOADS_DIRECTORY}'."
+        ),
+    )
+    consume_group.addoption(
+        "--cache-only",
+        action="store_true",
+        dest="cache_only",
+        default=False,
+        help=("Do not run any tests, only cache the fixtures. "),
+    )
 
 
 @pytest.hookimpl(tryfirst=True)
@@ -112,48 +131,49 @@ def pytest_configure(config):  # noqa: D103
     called before the pytest-html plugin's pytest_configure to ensure that
     it uses the modified `htmlpath` option.
     """
-    input_flag = any(arg.startswith("--input") for arg in config.invocation_params.args)
-    input_source = config.getoption("fixture_source")
+    fixtures_source = config.getoption("fixtures_source")
+    config.fixture_source_flags = ["--input", fixtures_source]
 
-    if input_flag and input_source == "stdin":
+    if fixtures_source is None:
+        config.fixture_source_flags = []
+        fixtures_source = default_input_directory()
+    elif fixtures_source == "stdin":
         config.test_cases = TestCases.from_stream(sys.stdin)
+        config.fixtures_real_source = "stdin"
+        config.fixtures_source = "stdin"
         return
+    elif ReleaseTag.is_release_string(fixtures_source):
+        fixtures_source = get_release_url(fixtures_source)
 
-    latest_base_url = "https://github.com/ethereum/execution-spec-tests/releases/latest/download"
-    if input_source == "latest-stable-release" or input_source == "latest-stable":
-        input_source = f"{latest_base_url}/fixtures_stable.tar.gz"
-    if input_source == "latest-develop-release" or input_source == "latest-develop":
-        input_source = f"{latest_base_url}/fixtures_develop.tar.gz"
-
-    if is_url(input_source):
+    config.fixtures_real_source = fixtures_source
+    if is_url(fixtures_source):
+        cached_downloads_directory = Path(config.getoption("fixture_cache_folder"))
         cached_downloads_directory.mkdir(parents=True, exist_ok=True)
-        input_source = download_and_extract(input_source, cached_downloads_directory)
-        config.option.fixture_source = input_source
+        fixtures_source = download_and_extract(fixtures_source, cached_downloads_directory)
 
-    input_source = Path(input_source)
-    if not input_source.exists():
-        pytest.exit(f"Specified fixture directory '{input_source}' does not exist.")
-    if not any(input_source.glob("**/*.json")):
+    fixtures_source = Path(fixtures_source)
+    config.fixtures_source = fixtures_source
+    if not fixtures_source.exists():
+        pytest.exit(f"Specified fixture directory '{fixtures_source}' does not exist.")
+    if not any(fixtures_source.glob("**/*.json")):
         pytest.exit(
-            f"Specified fixture directory '{input_source}' does not contain any JSON files."
+            f"Specified fixture directory '{fixtures_source}' does not contain any JSON files."
         )
 
-    index_file = input_source / ".meta" / "index.json"
+    index_file = fixtures_source / ".meta" / "index.json"
     index_file.parent.mkdir(parents=True, exist_ok=True)
     if not index_file.exists():
         rich.print(f"Generating index file [bold cyan]{index_file}[/]...")
         generate_fixtures_index(
-            input_source, quiet_mode=False, force_flag=False, disable_infer_format=False
+            fixtures_source, quiet_mode=False, force_flag=False, disable_infer_format=False
         )
     config.test_cases = TestCases.from_index_file(index_file)
 
     if config.option.collectonly:
         return
     if not config.getoption("disable_html") and config.getoption("htmlpath") is None:
         # generate an html report by default, unless explicitly disabled
-        config.option.htmlpath = os.path.join(
-            config.getoption("fixture_source"), default_html_report_file_path()
-        )
+        config.option.htmlpath = Path(default_html_report_file_path())
 
 
 def pytest_html_report_title(report):
@@ -163,20 +183,29 @@ def pytest_html_report_title(report):
 
 def pytest_report_header(config):  # noqa: D103
     consume_version = f"consume commit: {get_current_commit_hash_or_tag()}"
-    input_source = f"fixtures: {config.getoption('fixture_source')}"
-    return [consume_version, input_source]
+    fixtures_real_source = f"fixtures: {config.fixtures_real_source}"
+    return [consume_version, fixtures_real_source]
 
 
-@pytest.fixture(scope="function")
-def fixture_source(request) -> JsonSource:  # noqa: D103
-    return request.config.getoption("fixture_source")
+@pytest.fixture(scope="session")
+def fixture_source_flags(request) -> List[str]:
+    """Return the input flags used to specify the JSON test fixtures source."""
+    return request.config.fixture_source_flags
+
+
+@pytest.fixture(scope="session")
+def fixtures_source(request) -> FixturesSource:  # noqa: D103
+    return request.config.fixtures_source
 
 
 def pytest_generate_tests(metafunc):
     """
     Generate test cases for every test fixture in all the JSON fixture files
     within the specified fixtures directory, or read from stdin if the directory is 'stdin'.
     """
+    if metafunc.config.getoption("cache_only"):
+        return
+
     fork = metafunc.config.getoption("single_fork")
     metafunc.parametrize(
         "test_case",

src/pytest_plugins/consume/direct/conftest.py

@@ -17,6 +17,8 @@
 from ethereum_test_fixtures.consume import TestCaseIndexFile, TestCaseStream
 from ethereum_test_fixtures.file import Fixtures
 
+from ..consume import FixturesSource
+
 
 def pytest_addoption(parser):  # noqa: D103
     consume_group = parser.getgroup(
@@ -96,13 +98,13 @@ def test_dump_dir(
 
 
 @pytest.fixture
-def fixture_path(test_case: TestCaseIndexFile | TestCaseStream, fixture_source):
+def fixture_path(test_case: TestCaseIndexFile | TestCaseStream, fixtures_source: FixturesSource):
     """
     Path to the current JSON fixture file.
 
     If the fixture source is stdin, the fixture is written to a temporary json file.
     """
-    if fixture_source == "stdin":
+    if fixtures_source == "stdin":
         assert isinstance(test_case, TestCaseStream)
         temp_dir = tempfile.TemporaryDirectory()
         fixture_path = Path(temp_dir.name) / f"{test_case.id.replace('/','_')}.json"
@@ -113,7 +115,7 @@ def fixture_path(test_case: TestCaseIndexFile | TestCaseStream, fixture_source):
         temp_dir.cleanup()
     else:
         assert isinstance(test_case, TestCaseIndexFile)
-        yield fixture_source / test_case.json_path
+        yield fixtures_source / test_case.json_path
 
 
 @pytest.fixture(scope="function")