From 77de4d7d2879572a938e53a2bf4a68503eec59c8 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 6 Aug 2024 11:03:08 -0500 Subject: [PATCH 01/12] Add ruff --- .pre-commit-config.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 783bf7d..c6992dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,6 +18,14 @@ repos: .*/requirements\.txt ) args: [--unique] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.5.5 + hooks: + # lint & attempt to correct failures (e.g. pyupgrade) + - id: ruff + args: [--fix] + # compatible replacement for black + - id: ruff-format - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks rev: v2.14.0 hooks: From 9acf41fb63ea918442e3848fd3f5eab03bbd240c Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 6 Aug 2024 11:03:31 -0500 Subject: [PATCH 02/12] Add templating summary --- template-files/action.py | 218 +++++++++++++++++++++++++++++++++----- template-files/action.yml | 13 ++- 2 files changed, 202 insertions(+), 29 deletions(-) diff --git a/template-files/action.py b/template-files/action.py index 0c24496..bf19287 100644 --- a/template-files/action.py +++ b/template-files/action.py @@ -1,9 +1,13 @@ """Copy files from external locations as defined in `sync.yml`.""" + from __future__ import annotations import os import sys from argparse import ArgumentParser, ArgumentTypeError, Namespace +from collections import defaultdict +from contextlib import contextmanager +from enum import Enum from pathlib import Path from typing import TYPE_CHECKING @@ -11,24 +15,27 @@ from github import Auth, Github, UnknownObjectException from github.Repository import Repository from jinja2 import Environment, FileSystemLoader +from jinja2.exceptions import TemplateNotFound, TemplateError from jsonschema import validate from rich.console import Console +from rich.table import Table +from rich.padding import Padding if TYPE_CHECKING: - from typing import Any, Literal + from typing import Any, Callable, Iterator -print = Console(color_system="standard", soft_wrap=True).print -perror = Console( - color_system="standard", - soft_wrap=True, - stderr=True, - style="bold red", -).print +stdout_console = Console(color_system="standard", soft_wrap=True) +stderr_console = Console( + color_system="standard", soft_wrap=True, stderr=True, style="bold red" +) +print = stdout_console.print +perror = stderr_console.print class ActionError(Exception): pass + def validate_file(value: str) -> Path | None: try: path = Path(value).expanduser().resolve() @@ -179,27 +186,154 @@ def iterate_config( continue else: # inject stuff about the source and destination - context.update({ - # the current repository from which this GHA is being run, - # where the new files will be written - "repo": current_repo, - "dst": current_repo, - "destination": current_repo, - "current": current_repo, - # source (should be rarely, if ever, used in templating) - "src": upstream_repo, - "source": upstream_repo, - }) - - template = env.from_string(content) - dst.parent.mkdir(parents=True, exist_ok=True) - dst.write_text(template.render(**context)) - - print(f"✅ Templated {upstream_name}/{src} as {dst}") + context.update( + { + # the current repository from which this GHA is being run, + # where the new files will be written + "repo": current_repo, + "dst": current_repo, + "destination": current_repo, + "current": current_repo, + # source (should be rarely, if ever, used in templating) + "src": upstream_repo, + "source": upstream_repo, + } + ) + + with env.loader.get_stubs(upstream_name, src, dst) as stubs: + try: + template = env.from_string(content) + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_text(template.render(**context)) + except TemplateError as err: + perror( + f"❌ Failed to template {upstream_name}::{src}: {err}" + ) + errors += 1 + continue + + print(f"✅ {upstream_name}::{src} → {dst}") + + # display stubs for this file + if stubs: + table = Table.grid(padding=1) + table.add_column() + table.add_column() + for stub, state in stubs.items(): + table.add_row(stub, state) + print(Padding(table, (0, 0, 0, 3))) return errors +class TemplateState(Enum): + UNUSED = "unused" + MISSING = "missing" + USED = "used" + + def __rich__(self) -> str: + if self == self.UNUSED: + return f"[yellow]{self.value}[/yellow]" + elif self == self.MISSING: + return f"[red]{self.value}[/red]" + elif self == self.USED: + return f"[green]{self.value}[/green]" + else: + raise ValueError("Invalid TemplateState") + + @property + def icon(self) -> str: + if self == self.UNUSED: + return "⚠️" + elif self == self.MISSING: + return "❌" + elif self == self.USED: + return "✅" + else: + raise ValueError("Invalid TemplateState") + + +StubToStateType = dict[str, TemplateState] + + +TEMPALTE_STATE_LEN = max( + len(state.value) for state in TemplateState.__members__.values() +) + + +class StubLoader(FileSystemLoader): + current: tuple[str, str] | None + templates: dict[tuple[str, str] | None, StubToStateType] + + @property + def stub_to_state(self) -> StubToStateType: + return self.templates[None] + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.templates = defaultdict(dict) + self.templates[None] = dict.fromkeys( + self.list_templates(), TemplateState.UNUSED + ) + + def get_source( + self, + environment: Environment, + stub: str, + ) -> tuple[str, str, Callable[[], bool]]: + # assume template is used, track for current file and globally + self.templates[self.current][stub] = TemplateState.USED + self.templates[None][stub] = TemplateState.USED + + try: + # delegate to FileSystemLoader + return super().get_source(environment, stub) + except TemplateNotFound: + # TemplateNotFound: template does not exist, mark it as missing + self.templates[self.current][stub] = TemplateState.MISSING + self.templates[None][stub] = TemplateState.MISSING + raise + + @contextmanager + def get_stubs(self, file: str, src: str, dst: str) -> Iterator[StubToStateType]: + try: + # set current file + self.current = (f"{file}::{src}", f"{dst}") + + yield self.templates[self.current] + finally: + # clear current file + self.current = None + + +def format_html(loader: StubLoader) -> Iterator[str]: + yield "
" + yield "Templating Audit" + yield "" + + # filter out None keys (global stubs) + for (src, dst), stubs in filter(lambda key: key[0], loader.templates.items()): + yield f"* {src} → {dst}" + if stubs: + for stub, state in stubs.items(): + yield f" * {stub} {state.icon} ({state.value})" + yield "" + + yield "" + yield "" + for stub, state in loader.stub_to_state.items(): + yield ( + f"" + f"" + f"" + f"" + ) + yield "
StubState
{stub}{state.icon} ({state.value})
" + yield "" + + yield "
" + + def main(): errors = 0 @@ -210,9 +344,12 @@ def main(): config = read_config(args) - # initialize Jinja environment and GitHub client + # initialize stub loader + loader = StubLoader(args.stubs) + + # initialize Jinja environment env = Environment( - loader=FileSystemLoader(args.stubs), + loader=loader, # {{ }} is used in MermaidJS # ${{ }} is used in GitHub Actions # { } is used in Python @@ -225,6 +362,8 @@ def main(): comment_end_string="#]", keep_trailing_newline=True, ) + + # initialize GitHub client gh = Github(auth=Auth.Token(os.environ["GITHUB_TOKEN"])) # get current repository @@ -238,6 +377,31 @@ def main(): if not errors: errors += iterate_config(config, gh, env, current_repo) + # provide audit of stub usage + table = Table() + table.add_column("Stub") + table.add_column("State") + for stub, state in loader.stub_to_state.items(): + table.add_row(stub, state) + print(table) + + # dump summary to GitHub Actions summary + summary = os.getenv("GITHUB_STEP_SUMMARY") + output = os.getenv("GITHUB_OUTPUT") + if summary or output: + html = "\n".join(format_html(loader)) + if summary: + Path(summary).write_text(html) + if output: + with Path(output).open("a") as fh: + fh.write( + # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-an-output-parameter + # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#multiline-strings + f"summary< Date: Tue, 6 Aug 2024 12:47:18 -0500 Subject: [PATCH 03/12] console(record=True) & indentations --- template-files/action.py | 182 ++++++++++++++++++--------------------- 1 file changed, 84 insertions(+), 98 deletions(-) diff --git a/template-files/action.py b/template-files/action.py index bf19287..d9d6b66 100644 --- a/template-files/action.py +++ b/template-files/action.py @@ -20,16 +20,30 @@ from rich.console import Console from rich.table import Table from rich.padding import Padding +from rich import box if TYPE_CHECKING: from typing import Any, Callable, Iterator -stdout_console = Console(color_system="standard", soft_wrap=True) -stderr_console = Console( - color_system="standard", soft_wrap=True, stderr=True, style="bold red" -) -print = stdout_console.print -perror = stderr_console.print +INDENT = 3 + +console = Console(color_system="standard", record=True) + + +def print(renderable, *, indent: int = 0, **kwargs) -> None: + kwargs.setdefault("crop", False) + if indent: + renderable = Padding.indent(renderable, indent) + console.print(renderable, **kwargs) + + +def perror(*args, **kwargs) -> None: + kwargs.setdefault("style", "bold red") + try: + console.stderr = True + print(*args, **kwargs) + finally: + console.stderr = False class ActionError(Exception): @@ -121,18 +135,18 @@ def parse_config(file: str | dict) -> tuple[str | None, Path, bool, dict[str, An elif isinstance(file, dict): src = file.get("src", None) if (tmp := file.get("dst", src)) is None: - perror(f"❌ Invalid file definition ({file}), expected dst") + perror(f"* ❌ Invalid file definition (`{file}`), expected `dst`") raise ActionError dst = Path(tmp) remove = file.get("remove", False) context = file.get("with", {}) else: - perror(f"❌ Invalid file definition ({file}), expected str or dict") + perror(f"* ❌ Invalid file definition (`{file}`), expected `str` or `dict`") raise ActionError # to template a file we need a source file if not remove and src is None: - perror(f"❌ Invalid file definition ({file}), expected src") + perror(f"* ❌ Invalid file definition (`{file}`), expected `src`") raise ActionError return src, dst, remove, context @@ -150,9 +164,11 @@ def iterate_config( try: upstream_repo = gh.get_repo(upstream_name) except UnknownObjectException as err: - perror(f"❌ Failed to fetch {upstream_name}: {err}") + perror(f"* ❌ Failed to fetch `{upstream_name}`: {err}") errors += 1 continue + else: + print(f"* 🔄 Fetching files from `{upstream_name}`") for file in files: try: @@ -168,20 +184,20 @@ def iterate_config( dst.unlink() except FileNotFoundError: # FileNotFoundError: dst does not exist - print(f"⚠️ {dst} has already been removed") + print(f"* ⚠️ `{dst}` already removed", indent=INDENT) except PermissionError as err: # PermissionError: not possible to remove dst - perror(f"❌ Failed to remove {dst}: {err}") + perror(f"* ❌ Failed to remove `{dst}`: {err}", indent=INDENT) errors += 1 continue else: - print(f"✅ Removed {dst}") + print(f"* ❎ `{dst}` removed") else: # fetch src file try: content = upstream_repo.get_contents(src).decoded_content.decode() except UnknownObjectException as err: - perror(f"❌ Failed to fetch {src} from {upstream_name}: {err}") + perror(f"* ❌ Failed to fetch `{src}`: {err}", indent=INDENT) errors += 1 continue else: @@ -207,21 +223,22 @@ def iterate_config( dst.write_text(template.render(**context)) except TemplateError as err: perror( - f"❌ Failed to template {upstream_name}::{src}: {err}" + f"* ❌ Failed to template `{src}`: {err}", indent=INDENT ) errors += 1 continue - - print(f"✅ {upstream_name}::{src} → {dst}") - - # display stubs for this file - if stubs: - table = Table.grid(padding=1) - table.add_column() - table.add_column() - for stub, state in stubs.items(): - table.add_row(stub, state) - print(Padding(table, (0, 0, 0, 3))) + else: + print(f"* ✅ `{src}` → `{dst}`", indent=INDENT) + + # display stubs for this file + if stubs: + table = Table.grid(padding=1) + table.add_column() + table.add_column(max_width=STATE_WIDTH) + table.add_column() + for stub, state in stubs.items(): + table.add_row("*", state, f"`{stub}`") + print(table, indent=INDENT * 2) return errors @@ -233,48 +250,30 @@ class TemplateState(Enum): def __rich__(self) -> str: if self == self.UNUSED: - return f"[yellow]{self.value}[/yellow]" + return f"[yellow]⚠️ ({self.value})[/yellow]" elif self == self.MISSING: - return f"[red]{self.value}[/red]" + return f"[red]❌ ({self.value})[/red]" elif self == self.USED: - return f"[green]{self.value}[/green]" + return f"[green]✅ ({self.value})[/green]" else: raise ValueError("Invalid TemplateState") - @property - def icon(self) -> str: - if self == self.UNUSED: - return "⚠️" - elif self == self.MISSING: - return "❌" - elif self == self.USED: - return "✅" - else: - raise ValueError("Invalid TemplateState") +STATE_WIDTH = 12 -StubToStateType = dict[str, TemplateState] - -TEMPALTE_STATE_LEN = max( - len(state.value) for state in TemplateState.__members__.values() -) +StubToStateType = dict[str, TemplateState] class StubLoader(FileSystemLoader): - current: tuple[str, str] | None - templates: dict[tuple[str, str] | None, StubToStateType] - - @property - def stub_to_state(self) -> StubToStateType: - return self.templates[None] + current: tuple[str, str, str] | None + stubs: StubToStateType + templates: dict[tuple[str, str, str], StubToStateType] def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.templates = defaultdict(dict) - self.templates[None] = dict.fromkeys( - self.list_templates(), TemplateState.UNUSED - ) + self.stubs = dict.fromkeys(self.list_templates(), TemplateState.UNUSED) def get_source( self, @@ -282,23 +281,25 @@ def get_source( stub: str, ) -> tuple[str, str, Callable[[], bool]]: # assume template is used, track for current file and globally - self.templates[self.current][stub] = TemplateState.USED - self.templates[None][stub] = TemplateState.USED + if self.current: + self.templates[self.current][stub] = TemplateState.USED + self.stubs[stub] = TemplateState.USED try: # delegate to FileSystemLoader return super().get_source(environment, stub) except TemplateNotFound: # TemplateNotFound: template does not exist, mark it as missing - self.templates[self.current][stub] = TemplateState.MISSING - self.templates[None][stub] = TemplateState.MISSING + if self.current: + self.templates[self.current][stub] = TemplateState.MISSING + self.stubs[stub] = TemplateState.MISSING raise @contextmanager def get_stubs(self, file: str, src: str, dst: str) -> Iterator[StubToStateType]: try: # set current file - self.current = (f"{file}::{src}", f"{dst}") + self.current = (file, src, dst) yield self.templates[self.current] finally: @@ -306,41 +307,13 @@ def get_stubs(self, file: str, src: str, dst: str) -> Iterator[StubToStateType]: self.current = None -def format_html(loader: StubLoader) -> Iterator[str]: - yield "
" - yield "Templating Audit" - yield "" - - # filter out None keys (global stubs) - for (src, dst), stubs in filter(lambda key: key[0], loader.templates.items()): - yield f"* {src} → {dst}" - if stubs: - for stub, state in stubs.items(): - yield f" * {stub} {state.icon} ({state.value})" - yield "" - - yield "" - yield "" - for stub, state in loader.stub_to_state.items(): - yield ( - f"" - f"" - f"" - f"" - ) - yield "
StubState
{stub}{state.icon} ({state.value})
" - yield "" - - yield "
" - - def main(): - errors = 0 - args = parse_args() if not args.config: print("⚠️ No configuration file found, nothing to update") + dump_summary(0) sys.exit(0) + errors = 0 config = read_config(args) @@ -371,25 +344,42 @@ def main(): try: current_repo = gh.get_repo(current_name) except UnknownObjectException as err: - perror(f"❌ Failed to fetch {current_name}: {err}") + perror(f"❌ Failed to fetch `{current_name}`: {err}") errors += 1 if not errors: errors += iterate_config(config, gh, env, current_repo) # provide audit of stub usage - table = Table() + table = Table(box=box.MARKDOWN) table.add_column("Stub") - table.add_column("State") - for stub, state in loader.stub_to_state.items(): - table.add_row(stub, state) + table.add_column("State", max_width=STATE_WIDTH) + for stub, state in loader.stubs.items(): + table.add_row(f"`{stub}`", state) print(table) + if errors: + perror(f"Got {errors} error(s)") + + dump_summary() + sys.exit(errors) + + +def dump_summary(): # dump summary to GitHub Actions summary summary = os.getenv("GITHUB_STEP_SUMMARY") output = os.getenv("GITHUB_OUTPUT") if summary or output: - html = "\n".join(format_html(loader)) + html = console.export_html( + code_format=( + "
\n" + "Templating Audit\n" + "\n" + "{code}\n" + "\n" + "
" + ) + ) if summary: Path(summary).write_text(html) if output: @@ -402,10 +392,6 @@ def main(): f"GITHUB_OUTPUT_summary\n" ) - if errors: - perror(f"Got {errors} error(s)") - sys.exit(errors) - if __name__ == "__main__": main() From 4e0fcfcdc6caeea40be58d72d281216e98707286 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 6 Aug 2024 12:49:22 -0500 Subject: [PATCH 04/12] Set width instead --- template-files/action.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/template-files/action.py b/template-files/action.py index d9d6b66..33b844c 100644 --- a/template-files/action.py +++ b/template-files/action.py @@ -27,11 +27,10 @@ INDENT = 3 -console = Console(color_system="standard", record=True) +console = Console(color_system="standard", width=100_000_000, record=True) def print(renderable, *, indent: int = 0, **kwargs) -> None: - kwargs.setdefault("crop", False) if indent: renderable = Padding.indent(renderable, indent) console.print(renderable, **kwargs) From 484314697f2e22324943980f44576c0e1107e2c6 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 6 Aug 2024 12:52:18 -0500 Subject: [PATCH 05/12] Formatting --- template-files/action.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/template-files/action.py b/template-files/action.py index 33b844c..fcde190 100644 --- a/template-files/action.py +++ b/template-files/action.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from typing import Any, Callable, Iterator -INDENT = 3 +INDENT = 4 console = Console(color_system="standard", width=100_000_000, record=True) @@ -369,25 +369,21 @@ def dump_summary(): summary = os.getenv("GITHUB_STEP_SUMMARY") output = os.getenv("GITHUB_OUTPUT") if summary or output: - html = console.export_html( - code_format=( - "
\n" - "Templating Audit\n" - "\n" - "{code}\n" - "\n" - "
" - ) - ) + html = console.export_html(code_format="{code}") if summary: - Path(summary).write_text(html) + Path(summary).write_text(f"### Templating Audit\n{html}") if output: with Path(output).open("a") as fh: fh.write( # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-an-output-parameter # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#multiline-strings f"summary<\n" + f"Templating Audit\n" + f"\n" f"{html}\n" + f"\n" + f"\n" f"GITHUB_OUTPUT_summary\n" ) From c3a880ace18aa739a813c9517bec373bf03405d9 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 6 Aug 2024 13:01:14 -0500 Subject: [PATCH 06/12] export_text --- template-files/action.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/template-files/action.py b/template-files/action.py index fcde190..6861439 100644 --- a/template-files/action.py +++ b/template-files/action.py @@ -36,11 +36,11 @@ def print(renderable, *, indent: int = 0, **kwargs) -> None: console.print(renderable, **kwargs) -def perror(*args, **kwargs) -> None: +def perror(renderable, **kwargs) -> None: kwargs.setdefault("style", "bold red") try: console.stderr = True - print(*args, **kwargs) + print(renderable, **kwargs) finally: console.stderr = False @@ -310,7 +310,7 @@ def main(): args = parse_args() if not args.config: print("⚠️ No configuration file found, nothing to update") - dump_summary(0) + dump_summary() sys.exit(0) errors = 0 @@ -369,7 +369,7 @@ def dump_summary(): summary = os.getenv("GITHUB_STEP_SUMMARY") output = os.getenv("GITHUB_OUTPUT") if summary or output: - html = console.export_html(code_format="{code}") + html = console.export_text() if summary: Path(summary).write_text(f"### Templating Audit\n{html}") if output: From 75103ef7bbc2801fa935af12e639aa4da82b5b14 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 6 Aug 2024 13:27:22 -0500 Subject: [PATCH 07/12] group pip list --- template-files/action.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/template-files/action.yml b/template-files/action.yml index 727136f..286232f 100644 --- a/template-files/action.yml +++ b/template-files/action.yml @@ -42,7 +42,10 @@ runs: - name: Pip List shell: bash - run: pip list + run: | + echo ::group::Pip List + pip list + echo ::endgroup:: - name: Template Files id: template From 2a7bcd39a7f51c4047e278a00ace8fada84d91d7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2024 18:36:31 +0000 Subject: [PATCH 08/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- combine-durations/action.py | 1 + 1 file changed, 1 insertion(+) diff --git a/combine-durations/action.py b/combine-durations/action.py index a30f902..cc25c8d 100644 --- a/combine-durations/action.py +++ b/combine-durations/action.py @@ -1,4 +1,5 @@ """Combine test durations from all recent runs.""" + from __future__ import annotations import json From c6f4ff3b0e284544091c44ea774028e95da34428 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 6 Aug 2024 13:48:58 -0500 Subject: [PATCH 09/12] Lower complexity --- template-files/action.py | 132 ++++++++++++++++++++++----------------- 1 file changed, 73 insertions(+), 59 deletions(-) diff --git a/template-files/action.py b/template-files/action.py index 6861439..64ea0ba 100644 --- a/template-files/action.py +++ b/template-files/action.py @@ -151,6 +151,75 @@ def parse_config(file: str | dict) -> tuple[str | None, Path, bool, dict[str, An return src, dst, remove, context +def remove_file(dst: Path) -> int: + try: + dst.unlink() + except FileNotFoundError: + # FileNotFoundError: dst does not exist + print(f"* ⚠️ `{dst}` already removed", indent=INDENT) + except PermissionError as err: + # PermissionError: not possible to remove dst + perror(f"* ❌ Failed to remove `{dst}`: {err}", indent=INDENT) + return 1 + else: + print(f"* ❎ `{dst}` removed") + return 0 # no errors + + +def template_file( + env: Environment, + current_repo: Repository, + upstream_name: str, + upstream_repo: Repository, + src: str | None, + dst: Path, + context: dict[str, Any], +) -> int: + # fetch src file + try: + content = upstream_repo.get_contents(src).decoded_content.decode() + except UnknownObjectException as err: + perror(f"* ❌ Failed to fetch `{src}`: {err}", indent=INDENT) + return 1 + + # inject stuff about the source and destination + context.update( + { + # the current repository from which this GHA is being run, + # where the new files will be written + "repo": current_repo, + "dst": current_repo, + "destination": current_repo, + "current": current_repo, + # source (should be rarely, if ever, used in templating) + "src": upstream_repo, + "source": upstream_repo, + } + ) + + with env.loader.get_stubs(upstream_name, src, dst) as stubs: + try: + template = env.from_string(content) + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_text(template.render(**context)) + except TemplateError as err: + perror(f"* ❌ Failed to template `{src}`: {err}", indent=INDENT) + return 1 + + print(f"* ✅ `{src}` → `{dst}`", indent=INDENT) + + # display stubs for this file + if stubs: + table = Table.grid(padding=1) + table.add_column() + table.add_column(max_width=STATE_WIDTH) + table.add_column() + for stub, state in stubs.items(): + table.add_row("*", state, f"`{stub}`") + print(table, indent=INDENT * 2) + return 0 # no errors + + def iterate_config( config: dict, gh: Github, @@ -177,67 +246,12 @@ def iterate_config( errors += 1 continue - # remove dst file if remove: - try: - dst.unlink() - except FileNotFoundError: - # FileNotFoundError: dst does not exist - print(f"* ⚠️ `{dst}` already removed", indent=INDENT) - except PermissionError as err: - # PermissionError: not possible to remove dst - perror(f"* ❌ Failed to remove `{dst}`: {err}", indent=INDENT) - errors += 1 - continue - else: - print(f"* ❎ `{dst}` removed") + errors += remove_file(dst) else: - # fetch src file - try: - content = upstream_repo.get_contents(src).decoded_content.decode() - except UnknownObjectException as err: - perror(f"* ❌ Failed to fetch `{src}`: {err}", indent=INDENT) - errors += 1 - continue - else: - # inject stuff about the source and destination - context.update( - { - # the current repository from which this GHA is being run, - # where the new files will be written - "repo": current_repo, - "dst": current_repo, - "destination": current_repo, - "current": current_repo, - # source (should be rarely, if ever, used in templating) - "src": upstream_repo, - "source": upstream_repo, - } - ) - - with env.loader.get_stubs(upstream_name, src, dst) as stubs: - try: - template = env.from_string(content) - dst.parent.mkdir(parents=True, exist_ok=True) - dst.write_text(template.render(**context)) - except TemplateError as err: - perror( - f"* ❌ Failed to template `{src}`: {err}", indent=INDENT - ) - errors += 1 - continue - else: - print(f"* ✅ `{src}` → `{dst}`", indent=INDENT) - - # display stubs for this file - if stubs: - table = Table.grid(padding=1) - table.add_column() - table.add_column(max_width=STATE_WIDTH) - table.add_column() - for stub, state in stubs.items(): - table.add_row("*", state, f"`{stub}`") - print(table, indent=INDENT * 2) + errors += template_file( + env, current_repo, upstream_name, upstream_repo, src, dst, context + ) return errors From 0f7e209b2945af1a6081adc736756f549ea4a546 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 6 Aug 2024 13:55:26 -0500 Subject: [PATCH 10/12] Update .gitignore --- .gitignore | 193 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 186 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 187c0e0..88d606f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -# Created by https://www.toptal.com/developers/gitignore/api/node -# Edit at https://www.toptal.com/developers/gitignore?templates=node +# Created by https://www.toptal.com/developers/gitignore/api/node,python +# Edit at https://www.toptal.com/developers/gitignore?templates=node,python ### Node ### # Logs @@ -58,6 +58,9 @@ web_modules/ # Optional eslint cache .eslintcache +# Optional stylelint cache +.stylelintcache + # Microbundle cache .rpt2_cache/ .rts2_cache_cjs/ @@ -73,10 +76,12 @@ web_modules/ # Yarn Integrity file .yarn-integrity -# dotenv environment variables file +# dotenv environment variable files .env -.env.test -.env.production +.env.development.local +.env.test.local +.env.production.local +.env.local # parcel-bundler cache (https://parceljs.org/) .cache @@ -99,6 +104,12 @@ dist # vuepress build output .vuepress/dist +# vuepress v2.x temp and cache directory +.temp + +# Docusaurus cache and generated files +.docusaurus + # Serverless directories .serverless/ @@ -126,9 +137,177 @@ dist .webpack/ # Optional stylelint cache -.stylelintcache # SvelteKit build / generate output .svelte-kit -# End of https://www.toptal.com/developers/gitignore/api/node +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +# End of https://www.toptal.com/developers/gitignore/api/node,python From d66207738df46724c145da6e5db8cba91580c948 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 6 Aug 2024 14:02:51 -0500 Subject: [PATCH 11/12] Reorder --- template-files/action.py | 179 +++++++++++++++++++-------------------- 1 file changed, 89 insertions(+), 90 deletions(-) diff --git a/template-files/action.py b/template-files/action.py index 64ea0ba..9ce63c1 100644 --- a/template-files/action.py +++ b/template-files/action.py @@ -7,7 +7,7 @@ from argparse import ArgumentParser, ArgumentTypeError, Namespace from collections import defaultdict from contextlib import contextmanager -from enum import Enum +from enum import Enum, nonmember from pathlib import Path from typing import TYPE_CHECKING @@ -26,7 +26,6 @@ from typing import Any, Callable, Iterator INDENT = 4 - console = Console(color_system="standard", width=100_000_000, record=True) @@ -49,6 +48,70 @@ class ActionError(Exception): pass +class TemplateState(Enum): + UNUSED = "unused" + MISSING = "missing" + USED = "used" + WIDTH = nonmember(12) + + def __rich__(self) -> str: + if self == self.UNUSED: + return f"[yellow]⚠️ ({self.value})[/yellow]" + elif self == self.MISSING: + return f"[red]❌ ({self.value})[/red]" + elif self == self.USED: + return f"[green]✅ ({self.value})[/green]" + else: + raise ValueError("Invalid TemplateState") + + +class StubLoader(FileSystemLoader): + current: tuple[str, str, str] | None + stubs: dict[str, TemplateState] + templates: dict[tuple[str, str, str], dict[str, TemplateState]] + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.templates = defaultdict(dict) + self.stubs = dict.fromkeys(self.list_templates(), TemplateState.UNUSED) + + def get_source( + self, + environment: Environment, + stub: str, + ) -> tuple[str, str, Callable[[], bool]]: + # assume template is used, track for current file and globally + if self.current: + self.templates[self.current][stub] = TemplateState.USED + self.stubs[stub] = TemplateState.USED + + try: + # delegate to FileSystemLoader + return super().get_source(environment, stub) + except TemplateNotFound: + # TemplateNotFound: template does not exist, mark it as missing + if self.current: + self.templates[self.current][stub] = TemplateState.MISSING + self.stubs[stub] = TemplateState.MISSING + raise + + @contextmanager + def get_stubs( + self, + file: str, + src: str, + dst: str, + ) -> Iterator[dict[str, TemplateState]]: + try: + # set current file + self.current = (file, src, dst) + + yield self.templates[self.current] + finally: + # clear current file + self.current = None + + def validate_file(value: str) -> Path | None: try: path = Path(value).expanduser().resolve() @@ -212,7 +275,7 @@ def template_file( if stubs: table = Table.grid(padding=1) table.add_column() - table.add_column(max_width=STATE_WIDTH) + table.add_column(max_width=TemplateState.WIDTH) table.add_column() for stub, state in stubs.items(): table.add_row("*", state, f"`{stub}`") @@ -256,68 +319,28 @@ def iterate_config( return errors -class TemplateState(Enum): - UNUSED = "unused" - MISSING = "missing" - USED = "used" - - def __rich__(self) -> str: - if self == self.UNUSED: - return f"[yellow]⚠️ ({self.value})[/yellow]" - elif self == self.MISSING: - return f"[red]❌ ({self.value})[/red]" - elif self == self.USED: - return f"[green]✅ ({self.value})[/green]" - else: - raise ValueError("Invalid TemplateState") - - -STATE_WIDTH = 12 - - -StubToStateType = dict[str, TemplateState] - - -class StubLoader(FileSystemLoader): - current: tuple[str, str, str] | None - stubs: StubToStateType - templates: dict[tuple[str, str, str], StubToStateType] - - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.templates = defaultdict(dict) - self.stubs = dict.fromkeys(self.list_templates(), TemplateState.UNUSED) - - def get_source( - self, - environment: Environment, - stub: str, - ) -> tuple[str, str, Callable[[], bool]]: - # assume template is used, track for current file and globally - if self.current: - self.templates[self.current][stub] = TemplateState.USED - self.stubs[stub] = TemplateState.USED - - try: - # delegate to FileSystemLoader - return super().get_source(environment, stub) - except TemplateNotFound: - # TemplateNotFound: template does not exist, mark it as missing - if self.current: - self.templates[self.current][stub] = TemplateState.MISSING - self.stubs[stub] = TemplateState.MISSING - raise - - @contextmanager - def get_stubs(self, file: str, src: str, dst: str) -> Iterator[StubToStateType]: - try: - # set current file - self.current = (file, src, dst) - - yield self.templates[self.current] - finally: - # clear current file - self.current = None +def dump_summary(): + # dump summary to GitHub Actions summary + summary = os.getenv("GITHUB_STEP_SUMMARY") + output = os.getenv("GITHUB_OUTPUT") + if summary or output: + html = console.export_text() + if summary: + Path(summary).write_text(f"### Templating Audit\n{html}") + if output: + with Path(output).open("a") as fh: + fh.write( + # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-an-output-parameter + # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#multiline-strings + f"summary<\n" + f"Templating Audit\n" + f"\n" + f"{html}\n" + f"\n" + f"\n" + f"GITHUB_OUTPUT_summary\n" + ) def main(): @@ -366,7 +389,7 @@ def main(): # provide audit of stub usage table = Table(box=box.MARKDOWN) table.add_column("Stub") - table.add_column("State", max_width=STATE_WIDTH) + table.add_column("State", max_width=TemplateState.WIDTH) for stub, state in loader.stubs.items(): table.add_row(f"`{stub}`", state) print(table) @@ -378,29 +401,5 @@ def main(): sys.exit(errors) -def dump_summary(): - # dump summary to GitHub Actions summary - summary = os.getenv("GITHUB_STEP_SUMMARY") - output = os.getenv("GITHUB_OUTPUT") - if summary or output: - html = console.export_text() - if summary: - Path(summary).write_text(f"### Templating Audit\n{html}") - if output: - with Path(output).open("a") as fh: - fh.write( - # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-an-output-parameter - # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#multiline-strings - f"summary<\n" - f"Templating Audit\n" - f"\n" - f"{html}\n" - f"\n" - f"\n" - f"GITHUB_OUTPUT_summary\n" - ) - - if __name__ == "__main__": main() From 7300a2b8710a8040e06d3f37ba05ed2e8ac43801 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 6 Aug 2024 14:21:05 -0500 Subject: [PATCH 12/12] Display context in summary --- template-files/action.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/template-files/action.py b/template-files/action.py index 9ce63c1..acf1ca3 100644 --- a/template-files/action.py +++ b/template-files/action.py @@ -52,6 +52,7 @@ class TemplateState(Enum): UNUSED = "unused" MISSING = "missing" USED = "used" + CONTEXT = "context" WIDTH = nonmember(12) def __rich__(self) -> str: @@ -61,6 +62,8 @@ def __rich__(self) -> str: return f"[red]❌ ({self.value})[/red]" elif self == self.USED: return f"[green]✅ ({self.value})[/green]" + elif self == self.CONTEXT: + return f"[blue]📑 ({self.value})[/blue]" else: raise ValueError("Invalid TemplateState") @@ -245,40 +248,40 @@ def template_file( perror(f"* ❌ Failed to fetch `{src}`: {err}", indent=INDENT) return 1 - # inject stuff about the source and destination - context.update( - { - # the current repository from which this GHA is being run, - # where the new files will be written - "repo": current_repo, - "dst": current_repo, - "destination": current_repo, - "current": current_repo, - # source (should be rarely, if ever, used in templating) - "src": upstream_repo, - "source": upstream_repo, - } - ) + # standard context with source and destination details + standard_context = { + # the current repository from which this GHA is being run, + # where the new files will be written + "repo": current_repo, + "dst": current_repo, + "destination": current_repo, + "current": current_repo, + # source (should be rarely, if ever, used in templating) + "src": upstream_repo, + "source": upstream_repo, + } with env.loader.get_stubs(upstream_name, src, dst) as stubs: try: template = env.from_string(content) dst.parent.mkdir(parents=True, exist_ok=True) - dst.write_text(template.render(**context)) + dst.write_text(template.render(**{**context, **standard_context})) except TemplateError as err: perror(f"* ❌ Failed to template `{src}`: {err}", indent=INDENT) return 1 print(f"* ✅ `{src}` → `{dst}`", indent=INDENT) - # display stubs for this file - if stubs: + # display stubs & context for this file + if stubs or context: table = Table.grid(padding=1) table.add_column() table.add_column(max_width=TemplateState.WIDTH) table.add_column() for stub, state in stubs.items(): table.add_row("*", state, f"`{stub}`") + for key, value in context.items(): + table.add_row("*", TemplateState.CONTEXT, f"`{key}={value}`") print(table, indent=INDENT * 2) return 0 # no errors