From 1211dc39be8e0b39728b8cb721b99a30576fc7e0 Mon Sep 17 00:00:00 2001 From: Axel H Date: Mon, 6 Jun 2022 01:21:20 +0200 Subject: [PATCH] feat(scripts): added composite tasks support --- docs/docs/usage/scripts.md | 17 ++++- news/1117.feature.md | 1 + pdm/cli/commands/run.py | 54 ++++++++++++- tests/cli/test_run.py | 152 +++++++++++++++++++++++++++++++++++++ 4 files changed, 222 insertions(+), 2 deletions(-) create mode 100644 news/1117.feature.md diff --git a/docs/docs/usage/scripts.md b/docs/docs/usage/scripts.md index 8a092f6bf0..7da614e7f5 100644 --- a/docs/docs/usage/scripts.md +++ b/docs/docs/usage/scripts.md @@ -35,7 +35,7 @@ $ pdm run start -h 0.0.0.0 Flask server started at http://0.0.0.0:54321 ``` -PDM supports 3 types of scripts: +PDM supports 4 types of scripts: ### `cmd` @@ -85,6 +85,17 @@ The function can be supplied with literal arguments: foobar = {call = "foo_package.bar_module:main('dev')"} ``` +### `composite` + +This script kind execute other defined scripts: + +```toml +[tool.pdm.scripts] +lint = "flake8" +test = "pytest" +all = {composite = ["lint", "test"]} +``` + ### `env` All environment variables set in the current shell can be seen by `pdm run` and will be expanded when executed. @@ -98,6 +109,8 @@ start.env = {FOO = "bar", FLASK_ENV = "development"} Note how we use [TOML's syntax](https://github.com/toml-lang/toml) to define a composite dictionary. +Note that environment variables specified on composite level will override those defined by called tasks. + ### `env_file` You can also store all environment variables in a dotenv file and let PDM read it: @@ -108,6 +121,8 @@ start.cmd = "flask run -p 54321" start.env_file = ".env" ``` +Note that a dotenv file specified on composite level will override those defined by called tasks. + ### `site_packages` To make sure the running environment is properly isolated from the outer Python interpreter, diff --git a/news/1117.feature.md b/news/1117.feature.md new file mode 100644 index 0000000000..6b715dad63 --- /dev/null +++ b/news/1117.feature.md @@ -0,0 +1 @@ +Add a `composite` script kind allowing to run multiple defined scripts in a single command as well as reusing scripts but overriding `env` or `env_file`. diff --git a/pdm/cli/commands/run.py b/pdm/cli/commands/run.py index be90d710d2..59d787deff 100644 --- a/pdm/cli/commands/run.py +++ b/pdm/cli/commands/run.py @@ -34,11 +34,24 @@ class Task(NamedTuple): def __str__(self) -> str: return f"" + def clone_with(self, env: Mapping[str, str], env_file: str | None = None) -> Task: + """Clone the task with an updated env and|or env file""" + return Task( + kind=self.kind, + name=self.name, + args=self.args, + options={ + **self.options, # type: ignore + "env": {**self.options.get("env", {}), **env}, + "env_file": env_file or self.options.get("env_file"), + }, + ) + class TaskRunner: """The task runner for pdm project""" - TYPES = ["cmd", "shell", "call"] + TYPES = ["cmd", "shell", "call", "composite"] OPTIONS = ["env", "env_file", "help", "site_packages"] def __init__(self, project: Project) -> None: @@ -186,6 +199,9 @@ def _run_task(self, task: Task, args: Sequence[str] = ()) -> int: f"import sys, {module} as {short_name};" f"sys.exit({short_name}.{func})", ] + list(args) + elif kind == "composite": + assert isinstance(value, list) + if "env" in self.global_options: options["env"] = {**self.global_options["env"], **options.get("env", {})} options["env_file"] = options.get( @@ -196,10 +212,46 @@ def _run_task(self, task: Task, args: Sequence[str] = ()) -> int: err=True, verbosity=termui.DETAIL, ) + if kind == "composite": + env = options.get("env", {}) + env_file = options["env_file"] + for script in value: + if script is None: + return 1 + code = self._run_subtask(script, env, env_file) + if code != 0: + return code + return code return self._run_process( args, chdir=True, shell=shell, **options # type: ignore ) + def _run_subtask( + self, command: str, env: Mapping[str, str], env_file: str | None = None + ) -> int: + task = self._get_task(command) + + if task is not None: + pre_task = self._get_task(f"pre_{command}") + if pre_task is not None: + code = self._run_task(pre_task.clone_with(env, env_file)) + if code != 0: + return code + code = self._run_task(task.clone_with(env, env_file)) + if code != 0: + return code + post_task = self._get_task(f"post_{command}") + if post_task is not None: + code = self._run_task(post_task.clone_with(env, env_file)) + return code + else: + return self._run_process( + shlex.split(command), + **self.global_options, + env=env, + env_file=env_file, + ) # type: ignore + def run(self, command: str, args: Sequence[str]) -> int: task = self._get_task(command) if task is not None: diff --git a/tests/cli/test_run.py b/tests/cli/test_run.py index a8d1641e4e..ca0613ef45 100644 --- a/tests/cli/test_run.py +++ b/tests/cli/test_run.py @@ -5,6 +5,8 @@ from pathlib import Path from tempfile import TemporaryDirectory +import pytest + from pdm.cli.actions import PEP582_PATH from pdm.utils import cd @@ -330,8 +332,158 @@ def test_pre_and_post_scripts(project, invoke, capfd): "post_test": "python -c \"print('POST test CALLED')\"", } project.write_pyproject() + capfd.readouterr() invoke(["run", "test"], strict=True, obj=project) out, _ = capfd.readouterr() assert "PRE test CALLED" in out assert "IN test CALLED" in out assert "POST test CALLED" in out + + +@pytest.fixture +def _var(project): + (project.root / "var.py").write_text( + textwrap.dedent( + """ + import os + import sys + name = sys.argv[1] + var = os.getenv('VAR') + print(f"{name} CALLED with {var}") + """ + ) + ) + + +def test_run_composite(project, invoke, capfd): + project.tool_settings["scripts"] = { + "first": "echo 'First CALLED'", + "second": {"shell": "echo 'Second CALLED'"}, + "test": {"composite": ["first", "second"]}, + } + project.write_pyproject() + capfd.readouterr() + invoke(["run", "test"], strict=True, obj=project) + out, _ = capfd.readouterr() + assert "First CALLED" in out + assert "Second CALLED" in out + + +def test_composite_stops_on_first_failure(project, invoke, capfd): + project.tool_settings["scripts"] = { + "first": "echo 'First CALLED'", + "fail": "false", + "second": "echo 'Second CALLED'", + "test": {"composite": ["first", "fail", "second"]}, + } + project.write_pyproject() + capfd.readouterr() + result = invoke(["run", "test"], obj=project) + assert result.exit_code == 1 + out, _ = capfd.readouterr() + assert "First CALLED" in out + assert "Second CALLED" not in out + + +def test_composite_inherit_env(project, invoke, capfd, _var): + project.tool_settings["scripts"] = { + "first": { + "cmd": "python var.py First", + "env": {"VAR": "42"}, + }, + "second": { + "cmd": "python var.py Second", + "env": {"VAR": "42"}, + }, + "test": {"composite": ["first", "second"], "env": {"VAR": "overriden"}}, + } + project.write_pyproject() + capfd.readouterr() + invoke(["run", "test"], strict=True, obj=project) + out, _ = capfd.readouterr() + assert "First CALLED with overriden" in out + assert "Second CALLED with overriden" in out + + +def test_composite_fail_on_first_missing_task(project, invoke, capfd): + project.tool_settings["scripts"] = { + "first": "echo 'First CALLED'", + "second": "echo 'Second CALLED'", + "test": {"composite": ["first", "fail", "second"]}, + } + project.write_pyproject() + capfd.readouterr() + result = invoke(["run", "test"], obj=project) + assert result.exit_code == 1 + out, _ = capfd.readouterr() + assert "First CALLED" in out + assert "Second CALLED" not in out + + +def test_composite_runs_all_hooks(project, invoke, capfd): + project.tool_settings["scripts"] = { + "test": {"composite": ["first", "second"]}, + "pre_test": "echo 'Pre-Test CALLED'", + "post_test": "echo 'Post-Test CALLED'", + "first": "echo 'First CALLED'", + "pre_first": "echo 'Pre-First CALLED'", + "second": "echo 'Second CALLED'", + "post_second": "echo 'Post-Second CALLED'", + } + project.write_pyproject() + capfd.readouterr() + invoke(["run", "test"], strict=True, obj=project) + out, _ = capfd.readouterr() + assert "Pre-Test CALLED" in out + assert "Pre-First CALLED" in out + assert "First CALLED" in out + assert "Second CALLED" in out + assert "Post-Second CALLED" in out + assert "Post-Test CALLED" in out + + +def test_composite_hooks_inherit_env(project, invoke, capfd, _var): + project.tool_settings["scripts"] = { + "pre_task": {"cmd": "python var.py Pre-Task", "env": {"VAR": "42"}}, + "task": "echo 'Task CALLED'", + "post_task": {"cmd": "python var.py Post-Task", "env": {"VAR": "42"}}, + "test": {"composite": ["task"], "env": {"VAR": "overriden"}}, + } + project.write_pyproject() + capfd.readouterr() + invoke(["run", "test"], strict=True, obj=project) + out, _ = capfd.readouterr() + assert "Pre-Task CALLED with overriden" in out + assert "Task CALLED" in out + assert "Post-Task CALLED with overriden" in out + + +def test_composite_inherit_dotfile(project, invoke, capfd, _var): + (project.root / ".env").write_text("VAR=42") + (project.root / "override.env").write_text("VAR=overriden") + project.tool_settings["scripts"] = { + "pre_task": {"cmd": "python var.py Pre-Task", "env_file": ".env"}, + "task": {"cmd": "python var.py Task", "env_file": ".env"}, + "post_task": {"cmd": "python var.py Post-Task", "env_file": ".env"}, + "test": {"composite": ["task"], "env_file": "override.env"}, + } + project.write_pyproject() + capfd.readouterr() + invoke(["run", "test"], strict=True, obj=project) + out, _ = capfd.readouterr() + assert "Pre-Task CALLED with overriden" in out + assert "Task CALLED with overriden" in out + assert "Post-Task CALLED with overriden" in out + + +def test_composite_can_have_commands(project, invoke, capfd): + project.tool_settings["scripts"] = { + "task": "echo 'Task CALLED'", + "test": {"composite": ["task", "echo 'Command CALLED'"]}, + } + project.write_pyproject() + capfd.readouterr() + invoke(["run", "test"], strict=True, obj=project) + out, _ = capfd.readouterr() + assert "Task CALLED" in out + assert "Command CALLED" in out