diff --git a/.gitignore b/.gitignore index 187c0e0..88d606f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -# Created by https://www.toptal.com/developers/gitignore/api/node -# Edit at https://www.toptal.com/developers/gitignore?templates=node +# Created by https://www.toptal.com/developers/gitignore/api/node,python +# Edit at https://www.toptal.com/developers/gitignore?templates=node,python ### Node ### # Logs @@ -58,6 +58,9 @@ web_modules/ # Optional eslint cache .eslintcache +# Optional stylelint cache +.stylelintcache + # Microbundle cache .rpt2_cache/ .rts2_cache_cjs/ @@ -73,10 +76,12 @@ web_modules/ # Yarn Integrity file .yarn-integrity -# dotenv environment variables file +# dotenv environment variable files .env -.env.test -.env.production +.env.development.local +.env.test.local +.env.production.local +.env.local # parcel-bundler cache (https://parceljs.org/) .cache @@ -99,6 +104,12 @@ dist # vuepress build output .vuepress/dist +# vuepress v2.x temp and cache directory +.temp + +# Docusaurus cache and generated files +.docusaurus + # Serverless directories .serverless/ @@ -126,9 +137,177 @@ dist .webpack/ # Optional stylelint cache -.stylelintcache # SvelteKit build / generate output .svelte-kit -# End of https://www.toptal.com/developers/gitignore/api/node +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +# End of https://www.toptal.com/developers/gitignore/api/node,python diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 783bf7d..c6992dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,6 +18,14 @@ repos: .*/requirements\.txt ) args: [--unique] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.5.5 + hooks: + # lint & attempt to correct failures (e.g. pyupgrade) + - id: ruff + args: [--fix] + # compatible replacement for black + - id: ruff-format - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks rev: v2.14.0 hooks: diff --git a/combine-durations/action.py b/combine-durations/action.py index a30f902..cc25c8d 100644 --- a/combine-durations/action.py +++ b/combine-durations/action.py @@ -1,4 +1,5 @@ """Combine test durations from all recent runs.""" + from __future__ import annotations import json diff --git a/template-files/action.py b/template-files/action.py index 0c24496..acf1ca3 100644 --- a/template-files/action.py +++ b/template-files/action.py @@ -1,9 +1,13 @@ """Copy files from external locations as defined in `sync.yml`.""" + from __future__ import annotations import os import sys from argparse import ArgumentParser, ArgumentTypeError, Namespace +from collections import defaultdict +from contextlib import contextmanager +from enum import Enum, nonmember from pathlib import Path from typing import TYPE_CHECKING @@ -11,24 +15,106 @@ from github import Auth, Github, UnknownObjectException from github.Repository import Repository from jinja2 import Environment, FileSystemLoader +from jinja2.exceptions import TemplateNotFound, TemplateError from jsonschema import validate from rich.console import Console +from rich.table import Table +from rich.padding import Padding +from rich import box if TYPE_CHECKING: - from typing import Any, Literal + from typing import Any, Callable, Iterator + +INDENT = 4 +console = Console(color_system="standard", width=100_000_000, record=True) + -print = Console(color_system="standard", soft_wrap=True).print -perror = Console( - color_system="standard", - soft_wrap=True, - stderr=True, - style="bold red", -).print +def print(renderable, *, indent: int = 0, **kwargs) -> None: + if indent: + renderable = Padding.indent(renderable, indent) + console.print(renderable, **kwargs) + + +def perror(renderable, **kwargs) -> None: + kwargs.setdefault("style", "bold red") + try: + console.stderr = True + print(renderable, **kwargs) + finally: + console.stderr = False class ActionError(Exception): pass + +class TemplateState(Enum): + UNUSED = "unused" + MISSING = "missing" + USED = "used" + CONTEXT = "context" + WIDTH = nonmember(12) + + def __rich__(self) -> str: + if self == self.UNUSED: + return f"[yellow]⚠️ ({self.value})[/yellow]" + elif self == self.MISSING: + return f"[red]❌ ({self.value})[/red]" + elif self == self.USED: + return f"[green]✅ ({self.value})[/green]" + elif self == self.CONTEXT: + return f"[blue]📑 ({self.value})[/blue]" + else: + raise ValueError("Invalid TemplateState") + + +class StubLoader(FileSystemLoader): + current: tuple[str, str, str] | None + stubs: dict[str, TemplateState] + templates: dict[tuple[str, str, str], dict[str, TemplateState]] + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.templates = defaultdict(dict) + self.stubs = dict.fromkeys(self.list_templates(), TemplateState.UNUSED) + + def get_source( + self, + environment: Environment, + stub: str, + ) -> tuple[str, str, Callable[[], bool]]: + # assume template is used, track for current file and globally + if self.current: + self.templates[self.current][stub] = TemplateState.USED + self.stubs[stub] = TemplateState.USED + + try: + # delegate to FileSystemLoader + return super().get_source(environment, stub) + except TemplateNotFound: + # TemplateNotFound: template does not exist, mark it as missing + if self.current: + self.templates[self.current][stub] = TemplateState.MISSING + self.stubs[stub] = TemplateState.MISSING + raise + + @contextmanager + def get_stubs( + self, + file: str, + src: str, + dst: str, + ) -> Iterator[dict[str, TemplateState]]: + try: + # set current file + self.current = (file, src, dst) + + yield self.templates[self.current] + finally: + # clear current file + self.current = None + + def validate_file(value: str) -> Path | None: try: path = Path(value).expanduser().resolve() @@ -114,23 +200,92 @@ def parse_config(file: str | dict) -> tuple[str | None, Path, bool, dict[str, An elif isinstance(file, dict): src = file.get("src", None) if (tmp := file.get("dst", src)) is None: - perror(f"❌ Invalid file definition ({file}), expected dst") + perror(f"* ❌ Invalid file definition (`{file}`), expected `dst`") raise ActionError dst = Path(tmp) remove = file.get("remove", False) context = file.get("with", {}) else: - perror(f"❌ Invalid file definition ({file}), expected str or dict") + perror(f"* ❌ Invalid file definition (`{file}`), expected `str` or `dict`") raise ActionError # to template a file we need a source file if not remove and src is None: - perror(f"❌ Invalid file definition ({file}), expected src") + perror(f"* ❌ Invalid file definition (`{file}`), expected `src`") raise ActionError return src, dst, remove, context +def remove_file(dst: Path) -> int: + try: + dst.unlink() + except FileNotFoundError: + # FileNotFoundError: dst does not exist + print(f"* ⚠️ `{dst}` already removed", indent=INDENT) + except PermissionError as err: + # PermissionError: not possible to remove dst + perror(f"* ❌ Failed to remove `{dst}`: {err}", indent=INDENT) + return 1 + else: + print(f"* ❎ `{dst}` removed") + return 0 # no errors + + +def template_file( + env: Environment, + current_repo: Repository, + upstream_name: str, + upstream_repo: Repository, + src: str | None, + dst: Path, + context: dict[str, Any], +) -> int: + # fetch src file + try: + content = upstream_repo.get_contents(src).decoded_content.decode() + except UnknownObjectException as err: + perror(f"* ❌ Failed to fetch `{src}`: {err}", indent=INDENT) + return 1 + + # standard context with source and destination details + standard_context = { + # the current repository from which this GHA is being run, + # where the new files will be written + "repo": current_repo, + "dst": current_repo, + "destination": current_repo, + "current": current_repo, + # source (should be rarely, if ever, used in templating) + "src": upstream_repo, + "source": upstream_repo, + } + + with env.loader.get_stubs(upstream_name, src, dst) as stubs: + try: + template = env.from_string(content) + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_text(template.render(**{**context, **standard_context})) + except TemplateError as err: + perror(f"* ❌ Failed to template `{src}`: {err}", indent=INDENT) + return 1 + + print(f"* ✅ `{src}` → `{dst}`", indent=INDENT) + + # display stubs & context for this file + if stubs or context: + table = Table.grid(padding=1) + table.add_column() + table.add_column(max_width=TemplateState.WIDTH) + table.add_column() + for stub, state in stubs.items(): + table.add_row("*", state, f"`{stub}`") + for key, value in context.items(): + table.add_row("*", TemplateState.CONTEXT, f"`{key}={value}`") + print(table, indent=INDENT * 2) + return 0 # no errors + + def iterate_config( config: dict, gh: Github, @@ -143,9 +298,11 @@ def iterate_config( try: upstream_repo = gh.get_repo(upstream_name) except UnknownObjectException as err: - perror(f"❌ Failed to fetch {upstream_name}: {err}") + perror(f"* ❌ Failed to fetch `{upstream_name}`: {err}") errors += 1 continue + else: + print(f"* 🔄 Fetching files from `{upstream_name}`") for file in files: try: @@ -155,64 +312,56 @@ def iterate_config( errors += 1 continue - # remove dst file if remove: - try: - dst.unlink() - except FileNotFoundError: - # FileNotFoundError: dst does not exist - print(f"⚠️ {dst} has already been removed") - except PermissionError as err: - # PermissionError: not possible to remove dst - perror(f"❌ Failed to remove {dst}: {err}") - errors += 1 - continue - else: - print(f"✅ Removed {dst}") + errors += remove_file(dst) else: - # fetch src file - try: - content = upstream_repo.get_contents(src).decoded_content.decode() - except UnknownObjectException as err: - perror(f"❌ Failed to fetch {src} from {upstream_name}: {err}") - errors += 1 - continue - else: - # inject stuff about the source and destination - context.update({ - # the current repository from which this GHA is being run, - # where the new files will be written - "repo": current_repo, - "dst": current_repo, - "destination": current_repo, - "current": current_repo, - # source (should be rarely, if ever, used in templating) - "src": upstream_repo, - "source": upstream_repo, - }) - - template = env.from_string(content) - dst.parent.mkdir(parents=True, exist_ok=True) - dst.write_text(template.render(**context)) - - print(f"✅ Templated {upstream_name}/{src} as {dst}") + errors += template_file( + env, current_repo, upstream_name, upstream_repo, src, dst, context + ) return errors -def main(): - errors = 0 +def dump_summary(): + # dump summary to GitHub Actions summary + summary = os.getenv("GITHUB_STEP_SUMMARY") + output = os.getenv("GITHUB_OUTPUT") + if summary or output: + html = console.export_text() + if summary: + Path(summary).write_text(f"### Templating Audit\n{html}") + if output: + with Path(output).open("a") as fh: + fh.write( + # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-an-output-parameter + # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#multiline-strings + f"summary<\n" + f"Templating Audit\n" + f"\n" + f"{html}\n" + f"\n" + f"\n" + f"GITHUB_OUTPUT_summary\n" + ) + +def main(): args = parse_args() if not args.config: print("⚠️ No configuration file found, nothing to update") + dump_summary() sys.exit(0) + errors = 0 config = read_config(args) - # initialize Jinja environment and GitHub client + # initialize stub loader + loader = StubLoader(args.stubs) + + # initialize Jinja environment env = Environment( - loader=FileSystemLoader(args.stubs), + loader=loader, # {{ }} is used in MermaidJS # ${{ }} is used in GitHub Actions # { } is used in Python @@ -225,6 +374,8 @@ def main(): comment_end_string="#]", keep_trailing_newline=True, ) + + # initialize GitHub client gh = Github(auth=Auth.Token(os.environ["GITHUB_TOKEN"])) # get current repository @@ -232,14 +383,24 @@ def main(): try: current_repo = gh.get_repo(current_name) except UnknownObjectException as err: - perror(f"❌ Failed to fetch {current_name}: {err}") + perror(f"❌ Failed to fetch `{current_name}`: {err}") errors += 1 if not errors: errors += iterate_config(config, gh, env, current_repo) + # provide audit of stub usage + table = Table(box=box.MARKDOWN) + table.add_column("Stub") + table.add_column("State", max_width=TemplateState.WIDTH) + for stub, state in loader.stubs.items(): + table.add_row(f"`{stub}`", state) + print(table) + if errors: perror(f"Got {errors} error(s)") + + dump_summary() sys.exit(errors) diff --git a/template-files/action.yml b/template-files/action.yml index b2cc91c..286232f 100644 --- a/template-files/action.yml +++ b/template-files/action.yml @@ -18,6 +18,10 @@ inputs: A token with ability to comment, label, and modify the commit status (`pull_request: write` and `statuses: write` for fine-grained PAT; `repo` for classic PAT) default: ${{ github.token }} +outputs: + summary: + description: Summary of the files that were templated/copied. + value: ${{ steps.template.outputs.summary }} runs: using: composite @@ -32,11 +36,19 @@ runs: with: python-version: '3.11' - - name: install dependencies + - name: Install Dependencies shell: bash run: pip install --quiet -r ${{ github.action_path }}/requirements.txt - - name: sync & template files + - name: Pip List + shell: bash + run: | + echo ::group::Pip List + pip list + echo ::endgroup:: + + - name: Template Files + id: template shell: bash run: python ${{ github.action_path }}/action.py --config ${{ inputs.config }} --stubs ${{ inputs.stubs }} env: