From 046d172dd66310656dbb15617f3ce1cbbfc4ff6b Mon Sep 17 00:00:00 2001 From: jnicoulaud-ledger <102984500+jnicoulaud-ledger@users.noreply.github.com> Date: Mon, 18 Nov 2024 16:49:22 +0100 Subject: [PATCH] feat(BACK-7982): add `erc7730 list`/`erc7730 format` commands (#147) * feat(BACK-7982): refactor json methods to make it reusable * feat(BACK-7982): add `list` command/package, make lint use it * feat(BACK-7982): add `format` command * feat(BACK-7982): add command to main * feat(BACK-7982): add tests * feat(BACK-7982): add doc --- docs/pages/usage_cli.md | 33 ++++++++++++++ src/erc7730/common/client.py | 2 + src/erc7730/common/json.py | 29 ++++++++++-- src/erc7730/common/pydantic.py | 10 ++-- src/erc7730/format/__init__.py | 1 + src/erc7730/format/format.py | 83 ++++++++++++++++++++++++++++++++++ src/erc7730/lint/lint.py | 18 +------- src/erc7730/list/__init__.py | 1 + src/erc7730/list/list.py | 62 +++++++++++++++++++++++++ src/erc7730/main.py | 42 ++++++++++++++--- tests/test_main.py | 26 ++++++++++- 11 files changed, 274 insertions(+), 33 deletions(-) create mode 100644 src/erc7730/format/__init__.py create mode 100644 src/erc7730/format/format.py create mode 100644 src/erc7730/list/__init__.py create mode 100644 src/erc7730/list/list.py diff --git a/docs/pages/usage_cli.md b/docs/pages/usage_cli.md index 2b2f477..efa1930 100644 --- a/docs/pages/usage_cli.md +++ b/docs/pages/usage_cli.md @@ -30,6 +30,20 @@ You can validate your setup by running the `erc7730` command: ## Commands +### `erc7730 list` + +The `list` command recursively lists descriptors files in directory: + +```shell +$ erc7730 list +ercs/calldata-erc721-nfts.json +ercs/eip712-erc2612-permit.json +ercs/calldata-erc20-tokens.json +registry/1inch/calldata-AggregationRouterV5.json +registry/1inch/eip712-1inch-aggregation-router.json +... +``` + ### `erc7730 lint` The `lint` command runs validations on descriptors and outputs warnings and errors to the console: @@ -111,3 +125,22 @@ $ erc7730 schema # print JSON schema of input form (ERC-7730 $ erc7730 schema resolved # print JSON schema of resolved form $ erc7730 resolve .json # convert descriptor from input to resolved form ``` + +### `erc7730 format` + +The `format` command recursively finds and formats all descriptor files, starting from current directory by default: + +```shell +$ erc7730 format +📝 formatting 294 descriptor files… + +➡️ formatting registry/uniswap/eip712-uniswap-permit2.json… +no issue found ✔️ + +➡️ formatting registry/tether/calldata-usdt.json… +no issue found ✔️ + +... + +formatted 294 descriptor files, no errors occurred ✅ +``` diff --git a/src/erc7730/common/client.py b/src/erc7730/common/client.py index 0e2f0f1..449c279 100644 --- a/src/erc7730/common/client.py +++ b/src/erc7730/common/client.py @@ -62,6 +62,8 @@ def get_contract_abis(chain_id: int, contract_address: Address) -> list[ABI]: except Exception as e: if "Contract source code not verified" in str(e): raise Exception("contract source is not available on Etherscan") from e + if "Max calls per sec rate limit reached" in str(e): + raise Exception("Etherscan rate limit exceeded, please retry") from e raise e diff --git a/src/erc7730/common/json.py b/src/erc7730/common/json.py index e359874..04d6828 100644 --- a/src/erc7730/common/json.py +++ b/src/erc7730/common/json.py @@ -1,4 +1,5 @@ import json +import os from collections.abc import Iterator from json import JSONEncoder from pathlib import Path @@ -35,9 +36,7 @@ def read_json_with_includes(path: Path) -> Any: - circular includes are not detected and will result in a stack overflow. - "includes" key can only be used at root level of an object. """ - result: Any - with open(path) as f: - result = json.load(f) + result: dict[str, Any] = dict_from_json_file(path) if isinstance(result, dict) and (includes := result.pop("includes", None)) is not None: if isinstance(includes, list): parent = read_jsons_with_includes(paths=[path.parent / p for p in includes]) @@ -68,6 +67,30 @@ def _merge_dicts(d1: dict[str, Any], d2: dict[str, Any]) -> dict[str, Any]: return {**d2, **merged} +def dict_from_json_str(value: str) -> dict[str, Any]: + """Deserialize a dict from a JSON string.""" + return json.loads(value) + + +def dict_from_json_file(path: Path) -> dict[str, Any]: + """Deserialize a dict from a JSON file.""" + with open(path, "rb") as f: + return json.load(f) + + +def dict_to_json_str(values: dict[str, Any]) -> str: + """Serialize a dict into a JSON string.""" + return json.dumps(values, indent=2, cls=CompactJSONEncoder) + + +def dict_to_json_file(path: Path, values: dict[str, Any]) -> None: + """Serialize a dict into a JSON file, creating parent directories as needed.""" + os.makedirs(path.parent, exist_ok=True) + with open(path, "w") as f: + f.write(dict_to_json_str(values)) + f.write("\n") + + class CompactJSONEncoder(JSONEncoder): """A JSON Encoder that puts small containers on single lines.""" diff --git a/src/erc7730/common/pydantic.py b/src/erc7730/common/pydantic.py index 9b53304..295acd5 100644 --- a/src/erc7730/common/pydantic.py +++ b/src/erc7730/common/pydantic.py @@ -1,4 +1,3 @@ -import json import os from collections.abc import Callable from dataclasses import dataclass @@ -8,7 +7,7 @@ from pydantic import BaseModel, ValidationInfo, WrapValidator from pydantic_core import PydanticCustomError -from erc7730.common.json import CompactJSONEncoder, read_json_with_includes +from erc7730.common.json import dict_to_json_file, dict_to_json_str, read_json_with_includes _BaseModel = TypeVar("_BaseModel", bound=BaseModel) @@ -40,15 +39,12 @@ def model_to_json_dict(obj: _BaseModel) -> dict[str, Any]: def model_to_json_str(obj: _BaseModel) -> str: """Serialize a pydantic model into a JSON string.""" - return json.dumps(model_to_json_dict(obj), indent=2, cls=CompactJSONEncoder) + return dict_to_json_str(model_to_json_dict(obj)) def model_to_json_file(path: Path, model: _BaseModel) -> None: """Write a model to a JSON file, creating parent directories as needed.""" - os.makedirs(path.parent, exist_ok=True) - with open(path, "w") as f: - f.write(model_to_json_str(model)) - f.write("\n") + dict_to_json_file(path, model_to_json_dict(model)) @dataclass(frozen=True) diff --git a/src/erc7730/format/__init__.py b/src/erc7730/format/__init__.py new file mode 100644 index 0000000..750becb --- /dev/null +++ b/src/erc7730/format/__init__.py @@ -0,0 +1 @@ +"""Package implementing formatting commands to normalize descriptor files.""" diff --git a/src/erc7730/format/format.py b/src/erc7730/format/format.py new file mode 100644 index 0000000..e012901 --- /dev/null +++ b/src/erc7730/format/format.py @@ -0,0 +1,83 @@ +import os +from concurrent.futures.thread import ThreadPoolExecutor +from pathlib import Path + +from rich import print + +from erc7730.common.json import dict_from_json_file, dict_to_json_file +from erc7730.common.output import ( + AddFileOutputAdder, + BufferAdder, + ConsoleOutputAdder, + DropFileOutputAdder, + ExceptionsToOutput, + OutputAdder, +) +from erc7730.list.list import get_erc7730_files + + +def format_all_and_print_errors(paths: list[Path]) -> bool: + """ + Format all ERC-7730 descriptor files at given paths and print errors. + + :param paths: paths to apply formatter on + :return: true if not errors occurred + """ + out = DropFileOutputAdder(delegate=ConsoleOutputAdder()) + + count = format_all(paths, out) + + if out.has_errors: + print(f"[bold][red]formatted {count} descriptor files, some errors occurred ❌[/red][/bold]") + return False + + if out.has_warnings: + print(f"[bold][yellow]formatted {count} descriptor files, some warnings occurred ⚠️[/yellow][/bold]") + return True + + print(f"[bold][green]formatted {count} descriptor files, no errors occurred ✅[/green][/bold]") + return True + + +def format_all(paths: list[Path], out: OutputAdder) -> int: + """ + Format all ERC-7730 descriptor files at given paths. + + Paths can be files or directories, in which case all descriptor files in the directory are recursively formatted. + + :param paths: paths to apply formatter on + :param out: output adder + :return: number of files formatted + """ + files = list(get_erc7730_files(*paths, out=out)) + + if len(files) <= 1 or not (root_path := os.path.commonpath(files)): + root_path = None + + def label(f: Path) -> Path | None: + return f.relative_to(root_path) if root_path is not None else None + + if len(files) > 1: + print(f"📝 formatting {len(files)} descriptor files…\n") + + with ThreadPoolExecutor() as executor: + for future in (executor.submit(format_file, file, out, label(file)) for file in files): + future.result() + + return len(files) + + +def format_file(path: Path, out: OutputAdder, show_as: Path | None = None) -> None: + """ + Format a single ERC-7730 descriptor file. + + :param path: ERC-7730 descriptor file path + :param show_as: if provided, print this label instead of the file path + :param out: error handler + """ + + label = path if show_as is None else show_as + file_out = AddFileOutputAdder(delegate=out, file=path) + + with BufferAdder(file_out, prolog=f"➡️ formatting [bold]{label}[/bold]…", epilog="") as out, ExceptionsToOutput(out): + dict_to_json_file(path, dict_from_json_file(path)) diff --git a/src/erc7730/lint/lint.py b/src/erc7730/lint/lint.py index a2fe84d..24b1739 100644 --- a/src/erc7730/lint/lint.py +++ b/src/erc7730/lint/lint.py @@ -1,11 +1,9 @@ import os -from collections.abc import Generator from concurrent.futures.thread import ThreadPoolExecutor from pathlib import Path from rich import print -from erc7730 import ERC_7730_REGISTRY_CALLDATA_PREFIX, ERC_7730_REGISTRY_EIP712_PREFIX from erc7730.common.output import ( AddFileOutputAdder, BufferAdder, @@ -21,6 +19,7 @@ from erc7730.lint.lint_transaction_type_classifier import ClassifyTransactionTypeLinter from erc7730.lint.lint_validate_abi import ValidateABILinter from erc7730.lint.lint_validate_display_fields import ValidateDisplayFieldsLinter +from erc7730.list.list import get_erc7730_files from erc7730.model.input.descriptor import InputERC7730Descriptor @@ -59,20 +58,7 @@ def lint_all(paths: list[Path], out: OutputAdder) -> int: ] ) - def get_descriptor_files() -> Generator[Path, None, None]: - for path in paths: - if path.is_file(): - yield path - elif path.is_dir(): - for file in path.rglob("*.json"): - if file.name.startswith(ERC_7730_REGISTRY_CALLDATA_PREFIX) or file.name.startswith( - ERC_7730_REGISTRY_EIP712_PREFIX - ): - yield file - else: - raise ValueError(f"Invalid path: {path}") - - files = list(get_descriptor_files()) + files = list(get_erc7730_files(*paths, out=out)) if len(files) <= 1 or not (root_path := os.path.commonpath(files)): root_path = None diff --git a/src/erc7730/list/__init__.py b/src/erc7730/list/__init__.py new file mode 100644 index 0000000..b71d77f --- /dev/null +++ b/src/erc7730/list/__init__.py @@ -0,0 +1 @@ +"""Package implementing listing commands to easily find descriptor files.""" diff --git a/src/erc7730/list/list.py b/src/erc7730/list/list.py new file mode 100644 index 0000000..79b62d1 --- /dev/null +++ b/src/erc7730/list/list.py @@ -0,0 +1,62 @@ +from collections.abc import Generator +from pathlib import Path + +from rich import print + +from erc7730 import ERC_7730_REGISTRY_CALLDATA_PREFIX, ERC_7730_REGISTRY_EIP712_PREFIX +from erc7730.common.output import ( + ConsoleOutputAdder, + OutputAdder, +) + + +def list_all(paths: list[Path]) -> bool: + """ + List all ERC-7730 descriptor files at given paths. + + Paths can be files or directories, in which case all descriptor files in the directory are recursively listed. + + :param paths: paths to search for descriptor files + :return: true if no error occurred + """ + out = ConsoleOutputAdder() + + for file in get_erc7730_files(*paths, out=out): + print(file) + + return not out.has_errors + + +def get_erc7730_files(*paths: Path, out: OutputAdder) -> Generator[Path, None, None]: + """ + List all ERC-7730 descriptor files at given paths. + + Paths can be files or directories, in which case all descriptor files in the directory are recursively listed. + + :param paths: paths to search for descriptor files + :param out: error handler + """ + for path in paths: + if path.is_file(): + if is_erc7730_file(path): + yield path + else: + out.error(title="Invalid path", message=f"{path} is not an ERC-7730 descriptor file") + elif path.is_dir(): + for file in path.rglob("*.json"): + if is_erc7730_file(file): + yield file + else: + out.error(title="Invalid path", message=f"{path} is not a file or directory") + + +def is_erc7730_file(path: Path) -> bool: + """ + Check if a file is an ERC-7730 descriptor file. + + :param path: file path + :return: true if the file is an ERC-7730 descriptor file + """ + return path.is_file() and ( + path.name.startswith(ERC_7730_REGISTRY_CALLDATA_PREFIX) or path.name.startswith(ERC_7730_REGISTRY_EIP712_PREFIX) + ) diff --git a/src/erc7730/main.py b/src/erc7730/main.py index f41d286..c63836a 100644 --- a/src/erc7730/main.py +++ b/src/erc7730/main.py @@ -13,8 +13,10 @@ from erc7730.convert.ledger.eip712.convert_eip712_to_erc7730 import EIP712toERC7730Converter from erc7730.convert.ledger.eip712.convert_erc7730_to_eip712 import ERC7730toEIP712Converter from erc7730.convert.resolved.convert_erc7730_input_to_resolved import ERC7730InputToResolved +from erc7730.format.format import format_all_and_print_errors from erc7730.generate.generate import generate_descriptor from erc7730.lint.lint import lint_all_and_print_errors +from erc7730.list.list import list_all from erc7730.model import ERC7730ModelType from erc7730.model.base import Model from erc7730.model.input.descriptor import InputERC7730Descriptor @@ -46,7 +48,7 @@ Print ERC-7730 descriptor JSON schema. """, ) -def schema( +def command_schema( model_type: Annotated[ERC7730ModelType, Argument(help="The descriptor form ")] = ERC7730ModelType.INPUT, ) -> None: descriptor_type: type[Model] @@ -68,7 +70,7 @@ def schema( Validate descriptor files. """, ) -def lint( +def command_lint( paths: Annotated[list[Path], Argument(help="The files or directory paths to lint")], gha: Annotated[bool, Option(help="Enable Github annotations output")] = False, ) -> None: @@ -76,6 +78,34 @@ def lint( raise Exit(1) +@app.command( + name="list", + short_help="List descriptor files.", + help=""" + Recursively list all descriptor files, starting from current directory by default. + """, +) +def command_list( + paths: Annotated[list[Path] | None, Argument(help="The files or directory paths to search")] = None, +) -> None: + if not list_all(paths or [Path.cwd()]): + raise Exit(1) + + +@app.command( + name="format", + short_help="Format descriptor files.", + help=""" + Recursively find and format all descriptor files, starting from current directory by default. + """, +) +def command_format( + paths: Annotated[list[Path] | None, Argument(help="The files or directory paths to search")] = None, +) -> None: + if not format_all_and_print_errors(paths or [Path.cwd()]): + raise Exit(1) + + @app.command( name="resolve", short_help="Convert descriptor to resolved form.", @@ -92,7 +122,7 @@ def lint( See `erc7730 schema resolved` for the resolved descriptor schema. """, ) -def resolve( +def command_resolve( input_path: Annotated[Path, Argument(help="The input ERC-7730 file path")], ) -> None: input_descriptor = InputERC7730Descriptor.load(input_path) @@ -108,7 +138,7 @@ def resolve( Fetches ABI or schema files and generates a minimal descriptor. """, ) -def generate( +def command_generate( chain_id: Annotated[int, Option(help="The EIP-155 chain id")], address: Annotated[Address, Option(help="The contract address")], abi: Annotated[Path | None, Option(help="Path to a JSON ABI file (to generate a calldata descriptor)")] = None, @@ -136,7 +166,7 @@ def generate( Convert a legacy EIP-712 descriptor file to an ERC-7730 file. """, ) -def convert_eip712_to_erc7730( +def command_convert_eip712_to_erc7730( input_eip712_path: Annotated[Path, Argument(help="The input EIP-712 file path")], output_erc7730_path: Annotated[Path, Argument(help="The output ERC-7730 file path")], ) -> None: @@ -157,7 +187,7 @@ def convert_eip712_to_erc7730( Convert an ERC-7730 file to a legacy EIP-712 descriptor file (if applicable). """, ) -def convert_erc7730_to_eip712( +def command_convert_erc7730_to_eip712( input_erc7730_path: Annotated[Path, Argument(help="The input ERC-7730 file path")], output_eip712_path: Annotated[Path, Argument(help="The output EIP-712 file path")], ) -> None: diff --git a/tests/test_main.py b/tests/test_main.py index 748f162..a93a3d0 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,5 +1,6 @@ import json from pathlib import Path +from shutil import copytree import pytest from typer.testing import CliRunner @@ -7,7 +8,12 @@ from erc7730.main import app from erc7730.model import ERC7730ModelType from tests.cases import path_id -from tests.files import ERC7730_DESCRIPTORS, ERC7730_EIP712_DESCRIPTORS, LEGACY_EIP712_DESCRIPTORS +from tests.files import ( + ERC7730_DESCRIPTORS, + ERC7730_EIP712_DESCRIPTORS, + ERC7730_REGISTRY_ROOT, + LEGACY_EIP712_DESCRIPTORS, +) runner = CliRunner() @@ -28,6 +34,24 @@ def test_schema(model_type: ERC7730ModelType) -> None: assert json.loads(out) is not None +def test_list() -> None: + result = runner.invoke(app, ["list", str(ERC7730_REGISTRY_ROOT)]) + out = "".join(result.stdout.splitlines()) + assert "calldata-" in out + assert "eip712-" in out + assert ".json" in out + + +def test_format(tmp_path: Path) -> None: + copytree(ERC7730_REGISTRY_ROOT, tmp_path / "registry") + result = runner.invoke(app, ["format", str(tmp_path)]) + out = "".join(result.stdout.splitlines()) + assert "calldata-" in out + assert "eip712-" in out + assert ".json" in out + assert "no errors occurred ✅" in out + + @pytest.mark.parametrize("input_file", ERC7730_DESCRIPTORS, ids=path_id) def test_lint_registry_files(input_file: Path) -> None: result = runner.invoke(app, ["lint", str(input_file)])