From acde5e5ea9b6c2267686c94b800b6cf8eedd5e8f Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 21 Mar 2023 17:06:50 +0100 Subject: [PATCH 1/2] Implement a new pre-commit hook `update-pyproject` The hook runs `ci-cd update-deps` to update dependencies in a `pyproject.toml` file. A new documentation page about this hook as been added. --- .github/workflows/_local_cd_release.yml | 1 + .pre-commit-hooks.yaml | 19 ++++++ ci_cd/tasks/update_deps.py | 26 +++++---- docs/hooks/docs_update_pyproject.md | 77 +++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 10 deletions(-) create mode 100644 docs/hooks/docs_update_pyproject.md diff --git a/.github/workflows/_local_cd_release.yml b/.github/workflows/_local_cd_release.yml index 2932f1ea..be27ebec 100644 --- a/.github/workflows/_local_cd_release.yml +++ b/.github/workflows/_local_cd_release.yml @@ -27,6 +27,7 @@ jobs: docs/index.md,^\*\*Current version to use:\*\* \`v[0-9]+.*\`$,**Current version to use:** \`v{version}\` docs/hooks/docs_api_reference.md,rev: v[0-9]+.*$,rev: v{version} docs/hooks/docs_landing_page.md,rev: v[0-9]+.*$,rev: v{version} + docs/hooks/docs_update_pyproject.md,rev: v[0-9]+.*$,rev: v{version} docs/workflows/cd_release.md,uses: SINTEF/ci-cd/\.github/workflows/cd_release\.yml@v[0-9]+.*$,uses: SINTEF/ci-cd/.github/workflows/cd_release.yml@v{version} docs/workflows/ci_automerge_prs.md,uses: SINTEF/ci-cd/\.github/workflows/ci_automerge_prs\.yml@v[0-9]+.*$,uses: SINTEF/ci-cd/.github/workflows/ci_automerge_prs.yml@v{version} docs/workflows/ci_cd_updated_default_branch.md,uses: SINTEF/ci-cd/\.github/workflows/ci_cd_updated_default_branch\.yml@v[0-9]+.*$,uses: SINTEF/ci-cd/.github/workflows/ci_cd_updated_default_branch.yml@v{version} diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 71dd7b34..9d909d88 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -39,3 +39,22 @@ args: - '--replacement' - '(LICENSE),(LICENSE.md)' + +- id: update-pyproject + name: Update dependencies in pyproject.toml. + entry: "ci-cd update-deps --pre-commit" + language: python + files: "" + exclude: ^$ + types: [] + types_or: [] + exclude_types: [] + always_run: true + fail_fast: false + verbose: false + pass_filenames: false + require_serial: false + description: "Update dependencies in pyproject.toml according to the latest version on PyPI." + language_version: default + minimum_pre_commit_version: "2.16.0" + args: [] diff --git a/ci_cd/tasks/update_deps.py b/ci_cd/tasks/update_deps.py index 7548c33a..a02d6830 100644 --- a/ci_cd/tasks/update_deps.py +++ b/ci_cd/tasks/update_deps.py @@ -17,7 +17,7 @@ from invoke import task from ci_cd.exceptions import CICDException, InputError, InputParserError -from ci_cd.utils import update_file +from ci_cd.utils import Emoji, update_file if TYPE_CHECKING: # pragma: no cover from typing import Literal @@ -90,7 +90,10 @@ def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-s try: ignore_rules = parse_ignore_entries(ignore, ignore_separator) except InputError as exc: - sys.exit(f"Error: Could not parse ignore options.\nException: {exc}") + sys.exit( + f"{Emoji.CROSS_MARK.value} Error: Could not parse ignore options.\n" + f"Exception: {exc}" + ) LOGGER.debug("Parsed ignore rules: %s", ignore_rules) if pre_commit and root_repo_path == ".": @@ -101,8 +104,8 @@ def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-s pyproject_path = Path(root_repo_path).resolve() / "pyproject.toml" if not pyproject_path.exists(): sys.exit( - "Error: Could not find the Python package repository's 'pyproject.toml' " - f"file at: {pyproject_path}" + f"{Emoji.CROSS_MARK.value} Error: Could not find the Python package " + f"repository's 'pyproject.toml' file at: {pyproject_path}" ) pyproject = tomlkit.loads(pyproject_path.read_bytes()) @@ -145,7 +148,7 @@ def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-s ) LOGGER.warning(msg) if fail_fast: - sys.exit(msg) + sys.exit(f"{Emoji.CROSS_MARK.value} {msg}") print(msg) error = True continue @@ -194,7 +197,7 @@ def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-s ) LOGGER.warning(msg) if fail_fast: - sys.exit(msg) + sys.exit(f"{Emoji.CROSS_MARK.value} {msg}") print(msg) already_handled_packages.add(version_spec.package) error = True @@ -209,7 +212,7 @@ def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-s ) LOGGER.warning(msg) if fail_fast: - sys.exit(msg) + sys.exit(f"{Emoji.CROSS_MARK.value} {msg}") print(msg) already_handled_packages.add(version_spec.package) error = True @@ -273,11 +276,14 @@ def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-s ] = f"{version_spec.operator}{updated_version}" if error: - sys.exit("Errors occurred! See printed statements above.") + sys.exit( + f"{Emoji.CROSS_MARK.value} Errors occurred! See printed statements above." + ) if updated_packages: print( - "Successfully updated the following dependencies:\n" + f"{Emoji.PARTY_POPPER.value} Successfully updated the following " + "dependencies:\n" + "\n".join( f" {package} ({version}" f"{version_spec.extra_operator_version if version_spec.extra_operator_version else ''}" # pylint: disable=line-too-long @@ -287,7 +293,7 @@ def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-s + "\n" ) else: - print("No dependency updates available.") + print(f"{Emoji.CHECK_MARK.value} No dependency updates available.") def parse_ignore_entries( diff --git a/docs/hooks/docs_update_pyproject.md b/docs/hooks/docs_update_pyproject.md new file mode 100644 index 00000000..aaa0041c --- /dev/null +++ b/docs/hooks/docs_update_pyproject.md @@ -0,0 +1,77 @@ +# Update dependencies in `pyproject.toml` + +**pre-commit hook _id_:** `update-pyproject` + +Run this hook to update the dependencies in your `pyproject.toml` file. + +The hook utilizes `pip index versions` to determine the latest version available for all required and optional dependencies listed in your `pyproject.toml` file. +It checks this based on the Python version listed as the minimum supported Python version by the package (defined through the `requires-python` key in your `pyproject.toml` file). + +## Ignoring dependencies + +To ignore or configure how specific dependencies should be updated, the `--ignore` argument option can be utilized. +This is done by specifying a line per dependency that contains `--ignore-separator`-separated (defaults to ellipsis (`...`)) key/value-pairs of: + +| **Key** | **Description** | +|:---:|:--- | +| `dependency-name` | Ignore updates for dependencies with matching names, optionally using `*` to match zero or more characters. | +| `versions` | Ignore specific versions or ranges of versions. Examples: `~=1.0.5`, `>= 1.0.5,<2`, `>=0.1.1`. | +| `update-types` | Ignore types of updates, such as [SemVer](https://semver.org) `major`, `minor`, `patch` updates on version updates (for example: `version-update:semver-patch` will ignore patch updates). This can be combined with `dependency-name=*` to ignore particular `update-types` for all dependencies. | + +!!! note "Supported `update-types` values" + Currently, only `version-update:semver-major`, `version-update:semver-minor`, and `version-update:semver-patch` are supported options for `update-types`. + +The `--ignore` option is essentially similar to [the `ignore` option of Dependabot](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#ignore). +If `versions` and `update-types` are used together, they will both be respected jointly. + +Here are some examples of different values that may be given for the `--ignore` option that accomplishes different things: + +- _Value_: `dependency-name=Sphinx...versions=>=4.5.0` + _Accomplishes_: For Sphinx, ignore all updates for/from version 4.5.0 and up / keep the minimum version for Sphinx at 4.5.0. + +- _Value_: `dependency-name=pydantic...update-types=version-update:semver-patch` + _Accomplishes_: For pydantic, ignore all patch updates. + +- _Value_: `dependency-name=numpy` + _Accomplishes_: For NumPy, ignore any and all updates. + +[Below](#usage-example) is a usage example, where some of the example values above are implemented. + +## Expectations + +It is **required** that the root `pyproject.toml` exists. + +A minimum Python version for the Python package should be specified in the `pyproject.toml` file through the `requires-python` key. + +An active internet connection and for PyPI not to be down. + +## Options + +Any of these options can be given through the `args` key when defining the hook. + +| **Name** | **Description** | **Required** | **Default** | **Type** | +|:--- |:--- |:---:|:---:|:---:| +| `--root-repo-path` | A resolvable path to the root directory of the repository folder, where the `pyproject.toml` file can be found. | No | `.` | _string_ | +| `--fail-fast` | Fail immediately if an error occurs. Otherwise, print and ignore all non-critical errors. | No | `False` | _boolean_ | +| `--ignore` | Ignore-rules based on the `ignore` config option of Dependabot.

It should be of the format: `key=value...key=value`, i.e., an ellipsis (`...`) separator and then equal-sign-separated key/value-pairs.
Alternatively, the `--ignore-separator` can be set to something else to overwrite the ellipsis.

The only supported keys are: `dependency-name`, `versions`, and `update-types`.

Can be supplied multiple times per `dependency-name`. | No | | _string_ | +| `--ignore-separator` | Value to use instead of ellipsis (`...`) as a separator in `--ignore` key/value-pairs. | No | | _string_ | + +## Usage example + +The following is an example of how an addition of the _Update dependencies in `pyproject.toml`_ hook into a `.pre-commit-config.yaml` file may look. +It is meant to be complete as is. + +```yaml +repos: + - repo: https://github.com/SINTEF/ci-cd + rev: v2.2.1 + hooks: + - id: update-pyproject + args: + - --fail-fast + - --ignore-separator=// + - --ignore + - dependency-name=Sphinx//versions=>=4.5.0 + - --ignore + - dependency-name=numpy +``` From 49e829d3dc4f678d87f0d895c092a62a4e8c7150 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 21 Mar 2023 17:09:57 +0100 Subject: [PATCH 2/2] Add new hook docs page to overview page Also update the name of the file to match the hook id. --- .github/workflows/_local_cd_release.yml | 2 +- docs/hooks/index.md | 1 + docs/hooks/{docs_update_pyproject.md => update_pyproject.md} | 0 3 files changed, 2 insertions(+), 1 deletion(-) rename docs/hooks/{docs_update_pyproject.md => update_pyproject.md} (100%) diff --git a/.github/workflows/_local_cd_release.yml b/.github/workflows/_local_cd_release.yml index be27ebec..9122ff60 100644 --- a/.github/workflows/_local_cd_release.yml +++ b/.github/workflows/_local_cd_release.yml @@ -27,7 +27,7 @@ jobs: docs/index.md,^\*\*Current version to use:\*\* \`v[0-9]+.*\`$,**Current version to use:** \`v{version}\` docs/hooks/docs_api_reference.md,rev: v[0-9]+.*$,rev: v{version} docs/hooks/docs_landing_page.md,rev: v[0-9]+.*$,rev: v{version} - docs/hooks/docs_update_pyproject.md,rev: v[0-9]+.*$,rev: v{version} + docs/hooks/update_pyproject.md,rev: v[0-9]+.*$,rev: v{version} docs/workflows/cd_release.md,uses: SINTEF/ci-cd/\.github/workflows/cd_release\.yml@v[0-9]+.*$,uses: SINTEF/ci-cd/.github/workflows/cd_release.yml@v{version} docs/workflows/ci_automerge_prs.md,uses: SINTEF/ci-cd/\.github/workflows/ci_automerge_prs\.yml@v[0-9]+.*$,uses: SINTEF/ci-cd/.github/workflows/ci_automerge_prs.yml@v{version} docs/workflows/ci_cd_updated_default_branch.md,uses: SINTEF/ci-cd/\.github/workflows/ci_cd_updated_default_branch\.yml@v[0-9]+.*$,uses: SINTEF/ci-cd/.github/workflows/ci_cd_updated_default_branch.yml@v{version} diff --git a/docs/hooks/index.md b/docs/hooks/index.md index 88ca0b0b..0b84ffc2 100644 --- a/docs/hooks/index.md +++ b/docs/hooks/index.md @@ -10,3 +10,4 @@ This section contains all the available pre-commit hooks: - [Update API Reference in Documentation](./docs_api_reference.md) - [Update Landing Page (index.md) for Documentation](./docs_landing_page.md) +- [Update dependencies in `pyproject.toml`](./update_pyproject.md) diff --git a/docs/hooks/docs_update_pyproject.md b/docs/hooks/update_pyproject.md similarity index 100% rename from docs/hooks/docs_update_pyproject.md rename to docs/hooks/update_pyproject.md