From 37bb6d4828d1e9589a3ec9f0a1a24306b347f0fe Mon Sep 17 00:00:00 2001 From: Axel H Date: Mon, 6 Jun 2022 01:21:20 +0200 Subject: [PATCH] feat(scripts): added composite tasks support --- docs/docs/usage/scripts.md | 38 +++++- news/1117.feature.md | 1 + pdm/cli/commands/run.py | 55 ++++++--- tests/cli/test_run.py | 237 +++++++++++++++++++++++++++++++++++++ tests/conftest.py | 1 + 5 files changed, 317 insertions(+), 15 deletions(-) create mode 100644 news/1117.feature.md diff --git a/docs/docs/usage/scripts.md b/docs/docs/usage/scripts.md index 8a092f6bf0..67d0b0d088 100644 --- a/docs/docs/usage/scripts.md +++ b/docs/docs/usage/scripts.md @@ -35,7 +35,7 @@ $ pdm run start -h 0.0.0.0 Flask server started at http://0.0.0.0:54321 ``` -PDM supports 3 types of scripts: +PDM supports 4 types of scripts: ### `cmd` @@ -85,6 +85,32 @@ The function can be supplied with literal arguments: foobar = {call = "foo_package.bar_module:main('dev')"} ``` +### `composite` + +This script kind execute other defined scripts: + +```toml +[tool.pdm.scripts] +lint = "flake8" +test = "pytest" +all = {composite = ["lint", "test"]} +``` + +Running `pdm run all` will run `lint` first and then `test` if `lint` succeeded. + +You can also provide arguments to the called scripts: + +```toml +[tool.pdm.scripts] +lint = "flake8" +test = "pytest" +all = {composite = ["lint mypackage/", "test -v tests/"]} +``` + +!!! note + Argument passed on the command line are given to each called task. + + ### `env` All environment variables set in the current shell can be seen by `pdm run` and will be expanded when executed. @@ -98,6 +124,9 @@ start.env = {FOO = "bar", FLASK_ENV = "development"} Note how we use [TOML's syntax](https://github.com/toml-lang/toml) to define a composite dictionary. +!!! note + Environment variables specified on a composite task level will override those defined by called tasks. + ### `env_file` You can also store all environment variables in a dotenv file and let PDM read it: @@ -108,6 +137,9 @@ start.cmd = "flask run -p 54321" start.env_file = ".env" ``` +!!! note + A dotenv file specified on a composite task level will override those defined by called tasks. + ### `site_packages` To make sure the running environment is properly isolated from the outer Python interpreter, @@ -183,3 +215,7 @@ Under certain situations PDM will look for some special hook scripts for executi If there exists an `install` scripts under `[tool.pdm.scripts]` table, `pre_install` scripts can be triggered by both `pdm install` and `pdm run install`. So it is recommended to not use the preserved names. + +!!! note + Composite tasks can also have pre and post scripts. + Called tasks will run their own pre and post scripts. diff --git a/news/1117.feature.md b/news/1117.feature.md new file mode 100644 index 0000000000..6b715dad63 --- /dev/null +++ b/news/1117.feature.md @@ -0,0 +1 @@ +Add a `composite` script kind allowing to run multiple defined scripts in a single command as well as reusing scripts but overriding `env` or `env_file`. diff --git a/pdm/cli/commands/run.py b/pdm/cli/commands/run.py index 6fad92026a..c172559f7d 100644 --- a/pdm/cli/commands/run.py +++ b/pdm/cli/commands/run.py @@ -25,6 +25,19 @@ class TaskOptions(TypedDict, total=False): site_packages: bool +def exec_opts(*options: TaskOptions | None) -> dict[str, Any]: + return dict( + env={k: v for opts in options if opts for k, v in opts.get("env", {}).items()}, + **{ + k: v + for opts in options + if opts + for k, v in opts.items() + if k not in ("env", "help") + }, + ) + + class Task(NamedTuple): kind: str name: str @@ -38,7 +51,7 @@ def __str__(self) -> str: class TaskRunner: """The task runner for pdm project""" - TYPES = ["cmd", "shell", "call"] + TYPES = ["cmd", "shell", "call", "composite"] OPTIONS = ["env", "env_file", "help", "site_packages"] def __init__(self, project: Project) -> None: @@ -156,9 +169,10 @@ def _run_process( pass return process.returncode - def _run_task(self, task: Task, args: Sequence[str] = ()) -> int: + def _run_task( + self, task: Task, args: Sequence[str] = (), opts: TaskOptions | None = None + ) -> int: kind, _, value, options = task - options.pop("help", None) shell = False if kind == "cmd": if not isinstance(value, list): @@ -184,38 +198,51 @@ def _run_task(self, task: Task, args: Sequence[str] = ()) -> int: f"import sys, {module} as {short_name};" f"sys.exit({short_name}.{func})", ] + list(args) - if "env" in self.global_options: - options["env"] = {**self.global_options["env"], **options.get("env", {})} - options["env_file"] = options.get( - "env_file", self.global_options.get("env_file") - ) + elif kind == "composite": + assert isinstance(value, list) + self.project.core.ui.echo( f"Running {task}: [green]{str(args)}[/]", err=True, verbosity=termui.Verbosity.DETAIL, ) + if kind == "composite": + for script in value: + splitted = shlex.split(script) + cmd = splitted[0] + subargs = splitted[1:] + args # type: ignore + code = self.run(cmd, subargs, options) + if code != 0: + return code + return code return self._run_process( - args, chdir=True, shell=shell, **options # type: ignore + args, + chdir=True, + shell=shell, + **exec_opts(self.global_options, options, opts), ) - def run(self, command: str, args: Sequence[str]) -> int: + def run( + self, command: str, args: Sequence[str], opts: TaskOptions | None = None + ) -> int: task = self._get_task(command) if task is not None: pre_task = self._get_task(f"pre_{command}") if pre_task is not None: - code = self._run_task(pre_task) + code = self._run_task(pre_task, opts=opts) if code != 0: return code - code = self._run_task(task, args) + code = self._run_task(task, args, opts=opts) if code != 0: return code post_task = self._get_task(f"post_{command}") if post_task is not None: - code = self._run_task(post_task) + code = self._run_task(post_task, opts=opts) return code else: return self._run_process( - [command] + args, **self.global_options # type: ignore + [command] + args, # type: ignore + **exec_opts(self.global_options, opts), ) def show_list(self) -> None: diff --git a/tests/cli/test_run.py b/tests/cli/test_run.py index 03f3bdc522..bc99d61697 100644 --- a/tests/cli/test_run.py +++ b/tests/cli/test_run.py @@ -5,10 +5,42 @@ from pathlib import Path from tempfile import TemporaryDirectory +import pytest + from pdm.cli.actions import PEP582_PATH from pdm.utils import cd +@pytest.fixture +def _vars(project): + (project.root / "vars.py").write_text( + textwrap.dedent( + """ + import os + import sys + name = sys.argv[1] + vars = " ".join([f"{v}={os.getenv(v)}" for v in sys.argv[2:]]) + print(f"{name} CALLED with {vars}" if vars else f"{name} CALLED") + """ + ) + ) + + +@pytest.fixture +def _args(project): + (project.root / "args.py").write_text( + textwrap.dedent( + """ + import os + import sys + name = sys.argv[1] + args = ", ".join(sys.argv[2:]) + print(f"{name} CALLED with {args}" if args else f"{name} CALLED") + """ + ) + ) + + def test_pep582_launcher_for_python_interpreter(project, local_finder, invoke): project.root.joinpath("main.py").write_text( "import first;print(first.first([0, False, 1, 2]))\n" @@ -350,8 +382,213 @@ def test_pre_and_post_scripts(project, invoke, capfd): "post_test": "python -c \"print('POST test CALLED')\"", } project.write_pyproject() + capfd.readouterr() invoke(["run", "test"], strict=True, obj=project) out, _ = capfd.readouterr() assert "PRE test CALLED" in out assert "IN test CALLED" in out assert "POST test CALLED" in out + + +def test_run_composite(project, invoke, capfd): + project.tool_settings["scripts"] = { + "first": "echo 'First CALLED'", + "second": {"shell": "echo 'Second CALLED'"}, + "test": {"composite": ["first", "second"]}, + } + project.write_pyproject() + capfd.readouterr() + invoke(["run", "test"], strict=True, obj=project) + out, _ = capfd.readouterr() + assert "First CALLED" in out + assert "Second CALLED" in out + + +def test_composite_stops_on_first_failure(project, invoke, capfd): + project.tool_settings["scripts"] = { + "first": "echo 'First CALLED'", + "fail": "false", + "second": "echo 'Second CALLED'", + "test": {"composite": ["first", "fail", "second"]}, + } + project.write_pyproject() + capfd.readouterr() + result = invoke(["run", "test"], obj=project) + assert result.exit_code == 1 + out, _ = capfd.readouterr() + assert "First CALLED" in out + assert "Second CALLED" not in out + + +def test_composite_inherit_env(project, invoke, capfd, _vars): + project.tool_settings["scripts"] = { + "first": { + "cmd": "python vars.py First VAR", + "env": {"VAR": "42"}, + }, + "second": { + "cmd": "python vars.py Second VAR", + "env": {"VAR": "42"}, + }, + "test": {"composite": ["first", "second"], "env": {"VAR": "overriden"}}, + } + project.write_pyproject() + capfd.readouterr() + invoke(["run", "test"], strict=True, obj=project) + out, _ = capfd.readouterr() + assert "First CALLED with VAR=overriden" in out + assert "Second CALLED with VAR=overriden" in out + + +def test_composite_fail_on_first_missing_task(project, invoke, capfd): + project.tool_settings["scripts"] = { + "first": "echo 'First CALLED'", + "second": "echo 'Second CALLED'", + "test": {"composite": ["first", "fail", "second"]}, + } + project.write_pyproject() + capfd.readouterr() + result = invoke(["run", "test"], obj=project) + assert result.exit_code == 1 + out, _ = capfd.readouterr() + assert "First CALLED" in out + assert "Second CALLED" not in out + + +def test_composite_runs_all_hooks(project, invoke, capfd): + project.tool_settings["scripts"] = { + "test": {"composite": ["first", "second"]}, + "pre_test": "echo 'Pre-Test CALLED'", + "post_test": "echo 'Post-Test CALLED'", + "first": "echo 'First CALLED'", + "pre_first": "echo 'Pre-First CALLED'", + "second": "echo 'Second CALLED'", + "post_second": "echo 'Post-Second CALLED'", + } + project.write_pyproject() + capfd.readouterr() + invoke(["run", "test"], strict=True, obj=project) + out, _ = capfd.readouterr() + assert "Pre-Test CALLED" in out + assert "Pre-First CALLED" in out + assert "First CALLED" in out + assert "Second CALLED" in out + assert "Post-Second CALLED" in out + assert "Post-Test CALLED" in out + + +def test_composite_pass_parameters_to_subtasks(project, invoke, capfd, _args): + project.tool_settings["scripts"] = { + "test": {"composite": ["first", "second"]}, + "pre_test": "python args.py Pre-Test", + "post_test": "python args.py Post-Test", + "first": "python args.py First", + "pre_first": "python args.py Pre-First", + "second": "python args.py Second", + "post_second": "python args.py Post-Second", + } + project.write_pyproject() + capfd.readouterr() + invoke(["run", "test", "param=value"], strict=True, obj=project) + out, _ = capfd.readouterr() + assert "Pre-Test CALLED" in out + assert "Pre-First CALLED" in out + assert "First CALLED with param=value" in out + assert "Second CALLED with param=value" in out + assert "Post-Second CALLED" in out + assert "Post-Test CALLED" in out + + +def test_composite_can_pass_parameters(project, invoke, capfd, _args): + project.tool_settings["scripts"] = { + "test": {"composite": ["first param=first", "second param=second"]}, + "pre_test": "python args.py Pre-Test", + "post_test": "python args.py Post-Test", + "first": "python args.py First", + "pre_first": "python args.py Pre-First", + "second": "python args.py Second", + "post_second": "python args.py Post-Second", + } + project.write_pyproject() + capfd.readouterr() + invoke(["run", "test"], strict=True, obj=project) + out, _ = capfd.readouterr() + assert "Pre-Test CALLED" in out + assert "Pre-First CALLED" in out + assert "First CALLED with param=first" in out + assert "Second CALLED with param=second" in out + assert "Post-Second CALLED" in out + assert "Post-Test CALLED" in out + + +def test_composite_hooks_inherit_env(project, invoke, capfd, _vars): + project.tool_settings["scripts"] = { + "pre_task": {"cmd": "python vars.py Pre-Task VAR", "env": {"VAR": "42"}}, + "task": "echo 'Task CALLED'", + "post_task": {"cmd": "python vars.py Post-Task VAR", "env": {"VAR": "42"}}, + "test": {"composite": ["task"], "env": {"VAR": "overriden"}}, + } + project.write_pyproject() + capfd.readouterr() + invoke(["run", "test"], strict=True, obj=project) + out, _ = capfd.readouterr() + assert "Pre-Task CALLED with VAR=overriden" in out + assert "Task CALLED" in out + assert "Post-Task CALLED with VAR=overriden" in out + + +def test_composite_inherit_env_in_cascade(project, invoke, capfd, _vars): + project.tool_settings["scripts"] = { + "_": {"env": {"FOO": "BAR", "TIK": "TOK"}}, + "pre_task": { + "cmd": "python vars.py Pre-Task VAR FOO TIK", + "env": {"VAR": "42", "FOO": "foobar"}, + }, + "task": { + "cmd": "python vars.py Task VAR FOO TIK", + "env": {"VAR": "42", "FOO": "foobar"}, + }, + "post_task": { + "cmd": "python vars.py Post-Task VAR FOO TIK", + "env": {"VAR": "42", "FOO": "foobar"}, + }, + "test": {"composite": ["task"], "env": {"VAR": "overriden"}}, + } + project.write_pyproject() + capfd.readouterr() + invoke(["run", "test"], strict=True, obj=project) + out, _ = capfd.readouterr() + assert "Pre-Task CALLED with VAR=overriden FOO=foobar TIK=TOK" in out + assert "Task CALLED with VAR=overriden FOO=foobar TIK=TOK" in out + assert "Post-Task CALLED with VAR=overriden FOO=foobar TIK=TOK" in out + + +def test_composite_inherit_dotfile(project, invoke, capfd, _vars): + (project.root / ".env").write_text("VAR=42") + (project.root / "override.env").write_text("VAR=overriden") + project.tool_settings["scripts"] = { + "pre_task": {"cmd": "python vars.py Pre-Task VAR", "env_file": ".env"}, + "task": {"cmd": "python vars.py Task VAR", "env_file": ".env"}, + "post_task": {"cmd": "python vars.py Post-Task VAR", "env_file": ".env"}, + "test": {"composite": ["task"], "env_file": "override.env"}, + } + project.write_pyproject() + capfd.readouterr() + invoke(["run", "test"], strict=True, obj=project) + out, _ = capfd.readouterr() + assert "Pre-Task CALLED with VAR=overriden" in out + assert "Task CALLED with VAR=overriden" in out + assert "Post-Task CALLED with VAR=overriden" in out + + +def test_composite_can_have_commands(project, invoke, capfd): + project.tool_settings["scripts"] = { + "task": "echo 'Task CALLED'", + "test": {"composite": ["task", "echo 'Command CALLED'"]}, + } + project.write_pyproject() + capfd.readouterr() + invoke(["run", "test"], strict=True, obj=project) + out, _ = capfd.readouterr() + assert "Task CALLED" in out + assert "Command CALLED" in out diff --git a/tests/conftest.py b/tests/conftest.py index 56cf044b16..bf199abfcc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -381,6 +381,7 @@ def invoke(core): runner = CliRunner(mix_stderr=False) def caller(args, strict=False, **kwargs): + __tracebackhide__ = True result = runner.invoke( core, args, catch_exceptions=not strict, prog_name="pdm", **kwargs )