From cd41192370007a027ab10344491e0a712061c892 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 9 Oct 2023 13:32:17 +0200 Subject: [PATCH 01/12] Update to ruff (instead of pylint (and isort)) Remove pylint as a dependency. Remove all "# pylint: disable=..." statements. Remove pylint and isort as pre-commit hooks. Add ruff as a pre-commit hook. Avoid running pylint in the CI. Add a 'ruff' CI job. Add (back) disable E501 statements. E501: Line too long. Add it for either lines with string-formatted type (which has to be on one line) or an f-string with a single variable (i.e., a single-line f-string part with an implemented logic for a variable that results in the line being too long). --- .github/utils/run_hooks.py | 2 +- .github/workflows/_local_ci_tests.yml | 20 ++++++--- .../_local_ci_update_dependencies.yml | 1 - .pre-commit-config.yaml | 45 ++++--------------- ci_cd/tasks/api_reference_docs.py | 9 ++-- ci_cd/tasks/docs_index.py | 2 +- ci_cd/tasks/setver.py | 8 ++-- ci_cd/tasks/update_deps.py | 29 ++++++------ pyproject.toml | 9 +--- tests/tasks/test_api_reference_docs.py | 6 ++- tests/tasks/test_update_deps.py | 13 +++--- tests/test_utils.py | 6 +-- 12 files changed, 61 insertions(+), 89 deletions(-) diff --git a/.github/utils/run_hooks.py b/.github/utils/run_hooks.py index ab0fc690..e7ae10b2 100755 --- a/.github/utils/run_hooks.py +++ b/.github/utils/run_hooks.py @@ -60,5 +60,5 @@ def main(hook: str, options: list[str]) -> None: hook=sys.argv[1], options=sys.argv[2:] if len(sys.argv) > 2 else [], ) - except Exception as exc: # pylint: disable=broad-except + except Exception as exc: sys.exit(str(exc)) diff --git a/.github/workflows/_local_ci_tests.yml b/.github/workflows/_local_ci_tests.yml index 6c9194d8..401fb221 100644 --- a/.github/workflows/_local_ci_tests.yml +++ b/.github/workflows/_local_ci_tests.yml @@ -15,13 +15,9 @@ jobs: # general install_extras: "[dev,docs,testing]" - # pre-commit - skip_pre-commit_hooks: pylint,pylint-tests - - # pylint - pylint_runs: | - --rcfile=pyproject.toml ci_cd - --rcfile=pyproject.toml --disable=import-outside-toplevel,redefined-outer-name tests + # pylint & safety + run_pylint: false + run_safety: true # build dist build_libs: flit @@ -33,6 +29,16 @@ jobs: package_dirs: ci_cd debug: false + ruff: + name: ruff + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - uses: chartboost/ruff-action@v1 + pytest: name: pytest runs-on: ${{ matrix.os }} diff --git a/.github/workflows/_local_ci_update_dependencies.yml b/.github/workflows/_local_ci_update_dependencies.yml index 8e8b40a2..e5601e76 100644 --- a/.github/workflows/_local_ci_update_dependencies.yml +++ b/.github/workflows/_local_ci_update_dependencies.yml @@ -20,6 +20,5 @@ jobs: extra_to_dos: "- [ ] Make sure the PR is **squash** merged, with a sensible commit message." update_pre-commit: true install_extras: "[dev]" - skip_pre-commit_hooks: "pylint,pylint-tests" secrets: PAT: ${{ secrets.RELEASE_PAT }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 66d1d611..22a3b157 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,13 +18,15 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] - # isort is a tool to sort and group import statements in Python files - # It works on files in-place - - repo: https://github.com/timothycrosley/isort - rev: 5.12.0 + # ruff is a Python linter, incl. import sorter and formatter + # It works partly on files in-place + # More information can be found in its documentation: + # https://docs.astral.sh/ruff/ + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.292 hooks: - - id: isort - args: ["--profile", "black", "--filter-files", "--skip-gitignore"] + - id: ruff + args: [ --fix, --exit-non-zero-on-fix ] # Black is a code style and formatter # It works on files in-place @@ -59,34 +61,3 @@ repos: - id: docs-api-reference args: - "--package-dir=ci_cd" - - - repo: local - hooks: - # pylint is a Python linter - # It is run through the local environment to ensure external packages can be - # imported without issue. - # For more information about pylint see its documentation at: - # https://pylint.pycqa.org/en/latest/ - - id: pylint - name: pylint - entry: pylint - args: ["--rcfile=pyproject.toml"] - language: python - types: [python] - require_serial: true - exclude: ^tests/.*$ - # pylint is a Python linter - # It is run through the local environment to ensure external packages can be - # imported without issue. - # For more information about pylint see its documentation at: - # https://pylint.pycqa.org/en/latest/ - - id: pylint-tests - name: pylint - tests - entry: pylint - args: - - "--rcfile=pyproject.toml" - - "--disable=import-outside-toplevel,redefined-outer-name" - language: python - types: [python] - require_serial: true - files: ^tests/.*$ diff --git a/ci_cd/tasks/api_reference_docs.py b/ci_cd/tasks/api_reference_docs.py index 27152d31..ab426dd9 100644 --- a/ci_cd/tasks/api_reference_docs.py +++ b/ci_cd/tasks/api_reference_docs.py @@ -3,7 +3,6 @@ Create Python API reference in the documentation. This is specifically to be used with the MkDocs and mkdocstrings framework. """ -# pylint: disable=duplicate-code import logging import os import re @@ -95,7 +94,7 @@ "special_option", ], ) -def create_api_reference_docs( # pylint: disable=too-many-locals,too-many-branches,too-many-statements,line-too-long +def create_api_reference_docs( context, package_dir, pre_clean=False, @@ -310,7 +309,11 @@ def write_file(full_path: Path, content: str) -> None: f"{py_path_root}/{filename.stem}" if str(relpath) == "." or (str(relpath) == package.name and not single_package) - else f"{py_path_root}/{relpath if single_package else relpath.relative_to(package.name)}/{filename.stem}" + else ( + f"{py_path_root}/" + f"{relpath if single_package else relpath.relative_to(package.name)}/" # noqa: E501 + f"{filename.stem}" + ) ) # Replace OS specific path separators with forward slashes before diff --git a/ci_cd/tasks/docs_index.py b/ci_cd/tasks/docs_index.py index df8ae09f..474afe0d 100644 --- a/ci_cd/tasks/docs_index.py +++ b/ci_cd/tasks/docs_index.py @@ -38,7 +38,7 @@ }, iterable=["replacement"], ) -def create_docs_index( # pylint: disable=too-many-locals +def create_docs_index( context, pre_commit=False, root_repo_path=".", diff --git a/ci_cd/tasks/setver.py b/ci_cd/tasks/setver.py index 69f356f3..7f7993d7 100644 --- a/ci_cd/tasks/setver.py +++ b/ci_cd/tasks/setver.py @@ -48,7 +48,7 @@ }, iterable=["code_base_update"], ) -def setver( # pylint: disable=too-many-locals +def setver( _, package_dir, version, @@ -64,7 +64,7 @@ def setver( # pylint: disable=too-many-locals version: str = version # type: ignore[no-redef] root_repo_path: str = root_repo_path # type: ignore[no-redef] code_base_update: list[str] = code_base_update # type: ignore[no-redef] - code_base_update_separator: str = code_base_update_separator # type: ignore[no-redef] # pylint: disable=line-too-long + code_base_update_separator: str = code_base_update_separator # type: ignore[no-redef] test: bool = test # type: ignore[no-redef] fail_fast: bool = fail_fast # type: ignore[no-redef] @@ -152,7 +152,7 @@ def setver( # pylint: disable=too-many-locals ) print( "replacement (handled): " - f"{replacement.format(**{'package_dir': package_dir, 'version': semantic_version})}" # pylint: disable=line-too-long + f"{replacement.format(**{'package_dir': package_dir, 'version': semantic_version})}" # noqa: E501 ) try: @@ -174,7 +174,7 @@ def setver( # pylint: disable=too-many-locals f"{Emoji.CROSS_MARK.value} Error: Could not update file {filepath}" f" according to the given input:\n\n pattern: {pattern}\n " "replacement: " - f"{replacement.format(**{'package_dir': package_dir, 'version': semantic_version})}" # pylint: disable=line-too-long + f"{replacement.format(**{'package_dir': package_dir, 'version': semantic_version})}" # noqa: E501 ) print( diff --git a/ci_cd/tasks/update_deps.py b/ci_cd/tasks/update_deps.py index a7152e08..3ea5f528 100644 --- a/ci_cd/tasks/update_deps.py +++ b/ci_cd/tasks/update_deps.py @@ -2,7 +2,6 @@ Update dependencies in a `pyproject.toml` file. """ -# pylint: disable=duplicate-code from __future__ import annotations import logging @@ -55,7 +54,7 @@ }, iterable=["ignore"], ) -def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-statements +def update_deps( context, root_repo_path=".", fail_fast=False, @@ -230,7 +229,7 @@ def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-s # Apply ignore rules if version_spec.package in ignore_rules or "*" in ignore_rules: versions: "list[dict[Literal['operator', 'version'], str]]" = [] - update_types: "dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]" = ( # pylint: disable=line-too-long + update_types: "dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]" = ( # noqa: E501 {} ) @@ -270,15 +269,15 @@ def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-s rf'"{escaped_full_dependency_name} {version_spec.operator}.*"', f'"{version_spec.full_dependency} ' f"{version_spec.operator}{updated_version}" - f'{version_spec.extra_operator_version if version_spec.extra_operator_version else ""}' # pylint: disable=line-too-long - f'{version_spec.environment_marker if version_spec.environment_marker else ""}"', # pylint: disable=line-too-long + f'{version_spec.extra_operator_version if version_spec.extra_operator_version else ""}' # noqa: E501 + f'{version_spec.environment_marker if version_spec.environment_marker else ""}"', # noqa: E501 ), ) already_handled_packages.add(version_spec.package) updated_packages[version_spec.full_dependency] = ( f"{version_spec.operator}{updated_version}" - f"{version_spec.extra_operator_version if version_spec.extra_operator_version else ''}" # pylint: disable=line-too-long - f"{' ' + version_spec.environment_marker if version_spec.environment_marker else ''}" # pylint: disable=line-too-long + f"{version_spec.extra_operator_version if version_spec.extra_operator_version else ''}" # noqa: E501 + f"{' ' + version_spec.environment_marker if version_spec.environment_marker else ''}" # noqa: E501 ) if error: @@ -317,7 +316,7 @@ def parse_ignore_entries( A parsed mapping of dependencies to ignore rules. """ - ignore_entries: 'dict[str, dict[Literal["versions", "update-types"], list[str]]]' = ( + ignore_entries: 'dict[str, dict[Literal["versions", "update-types"], list[str]]]' = ( # noqa: E501 {} ) @@ -331,7 +330,7 @@ def parse_ignore_entries( f"value: --ignore={entry}" ) - ignore_entry: 'dict[Literal["dependency-name", "versions", "update-types"], str]' = ( # pylint: disable=line-too-long + ignore_entry: 'dict[Literal["dependency-name", "versions", "update-types"], str]' = ( # noqa: E501 {} ) for pair in pairs: @@ -351,7 +350,7 @@ def parse_ignore_entries( f"times in the option {entry!r}" ) - ignore_entry[match.group("key")] = match.group("value").strip() # type: ignore[index] # pylint: disable=line-too-long + ignore_entry[match.group("key")] = match.group("value").strip() # type: ignore[index] if "dependency-name" not in ignore_entry: raise InputError( @@ -373,7 +372,7 @@ def parse_ignore_entries( def parse_ignore_rules( rules: "dict[Literal['versions', 'update-types'], list[str]]", -) -> "tuple[list[dict[Literal['operator', 'version'], str]], dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]]": # pylint: disable=line-too-long +) -> "tuple[list[dict[Literal['operator', 'version'], str]], dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]]": # noqa: E501 """Parser for a specific set of ignore rules. Parameters: @@ -388,7 +387,7 @@ def parse_ignore_rules( return [{"operator": ">=", "version": "0"}], {} versions: 'list[dict[Literal["operator", "version"], str]]' = [] - update_types: "dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]" = ( # pylint: disable=line-too-long + update_types: "dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]" = ( # noqa: E501 {} ) @@ -422,7 +421,7 @@ def parse_ignore_rules( "'version-update:semver-patch'.\nUnparseable 'update-types' " f"value: {update_type_entry!r}" ) - update_types["version-update"].append(match.group("semver_part")) # type: ignore[arg-type] # pylint: disable=line-too-long + update_types["version-update"].append(match.group("semver_part")) # type: ignore[arg-type] return versions, update_types @@ -488,7 +487,7 @@ def _ignore_version_rules( def _ignore_semver_rules( current: list[str], latest: list[str], - semver_rules: "dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]", # pylint: disable=line-too-long + semver_rules: "dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]", # noqa: E501 ) -> bool: """If ANY of the semver rules are True, ignore the version.""" if any( @@ -529,7 +528,7 @@ def ignore_version( current: list[str], latest: list[str], version_rules: "list[dict[Literal['operator', 'version'], str]]", - semver_rules: "dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]", # pylint: disable=line-too-long + semver_rules: "dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]", # noqa: E501 ) -> bool: """Determine whether the latest version can be ignored. diff --git a/pyproject.toml b/pyproject.toml index 0cf0e5dc..72209008 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,8 +46,8 @@ testing = [ "pytest-cov ~=4.1", ] dev = [ + "ci-cd[docs,testing]", "pre-commit ~=2.21", - "pylint ~=2.13", ] [project.urls] @@ -69,13 +69,6 @@ show_error_codes = true allow_redefinition = true check_untyped_defs = true -[tool.pylint.messages_control] -max-line-length = 90 -disable = [] -max-args = 15 -max-branches = 15 -max-returns = 10 - [tool.pytest.ini_options] minversion = "7.0" filterwarnings = [ diff --git a/tests/tasks/test_api_reference_docs.py b/tests/tasks/test_api_reference_docs.py index 90502d05..b1129b7c 100644 --- a/tests/tasks/test_api_reference_docs.py +++ b/tests/tasks/test_api_reference_docs.py @@ -1,5 +1,4 @@ """Test `ci_cd.tasks.api_reference_docs`.""" -# pylint: disable=too-many-locals from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -659,7 +658,10 @@ def test_larger_multi_packages(tmp_path: "Path") -> None: ) == 'title: "tasks"\n' assert (package_dir / "tasks" / "api_reference_docs.md").read_text( encoding="utf8" - ) == f"# api_reference_docs\n\n::: {package_dir.name}.tasks.api_reference_docs\n" + ) == ( + "# api_reference_docs\n\n::: " + f"{package_dir.name}.tasks.api_reference_docs\n" + ) assert (package_dir / "tasks" / "docs_index.md").read_text( encoding="utf8" ) == f"# docs_index\n\n::: {package_dir.name}.tasks.docs_index\n" diff --git a/tests/tasks/test_update_deps.py b/tests/tasks/test_update_deps.py index 09ecc04b..53512a8a 100644 --- a/tests/tasks/test_update_deps.py +++ b/tests/tasks/test_update_deps.py @@ -1,5 +1,4 @@ """Test `ci_cd.tasks.update_deps()`.""" -# pylint: disable=line-too-long,too-many-lines,too-many-locals from __future__ import annotations from typing import TYPE_CHECKING @@ -261,7 +260,7 @@ def test_update_deps(tmp_path: "Path", caplog: pytest.LogCaptureFixture) -> None def test_parse_ignore_entries( entries: list[str], separator: str, - expected_outcome: 'dict[str, dict[Literal["dependency-name", "versions", "update-types"], str]]', + expected_outcome: 'dict[str, dict[Literal["dependency-name", "versions", "update-types"], str]]', # noqa: E501 ) -> None: """Check the `--ignore` option values are parsed as expected.""" from ci_cd.tasks.update_deps import parse_ignore_entries @@ -326,7 +325,7 @@ def test_parse_ignore_entries( ) def test_parse_ignore_rules( rules: 'dict[Literal["versions", "update-types"], list[str]]', - expected_outcome: "tuple[list[dict[Literal['operator', 'version'], str]], dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]]", + expected_outcome: "tuple[list[dict[Literal['operator', 'version'], str]], dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]]", # noqa: E501 ) -> None: """Check a specific set of ignore rules is parsed as expected.""" from ci_cd.tasks.update_deps import parse_ignore_rules @@ -346,14 +345,14 @@ def test_parse_ignore_rules( def _parametrize_ignore_version() -> ( - "dict[str, tuple[str, str, list[dict[Literal['operator', 'version'], str]], dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]], bool]]" + "dict[str, tuple[str, str, list[dict[Literal['operator', 'version'], str]], dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]], bool]]" # noqa: E501 ): """Utility function for `test_ignore_version()`. The parametrized inputs are created in this function in order to have more meaningful IDs in the runtime overview. """ - test_cases: "list[tuple[str, str, list[dict[Literal['operator', 'version'], str]], dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]], bool]]" = [ + test_cases: "list[tuple[str, str, list[dict[Literal['operator', 'version'], str]], dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]], bool]]" = [ # noqa: E501 ("1.1.1", "2.2.2", [{"operator": ">", "version": "2.2.2"}], {}, False), ("1.1.1", "2.2.2", [{"operator": ">", "version": "2.2"}], {}, True), ("1.1.1", "2.2.2", [{"operator": ">", "version": "2"}], {}, True), @@ -870,7 +869,7 @@ def _parametrize_ignore_version() -> ( ), ("1.1.1", "1.1.2", [], {}, True), ] - res: "dict[str, tuple[str, str, list[dict[Literal['operator', 'version'], str]], dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]], bool]]" = ( + res: "dict[str, tuple[str, str, list[dict[Literal['operator', 'version'], str]], dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]], bool]]" = ( # noqa: E501 {} ) for test_case in test_cases: @@ -905,7 +904,7 @@ def test_ignore_version( current: str, latest: str, version_rules: "list[dict[Literal['operator', 'version'], str]]", - semver_rules: "dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]", + semver_rules: "dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]", # noqa: E501 expected_outcome: bool, ) -> None: """Check the expected ignore rules are resolved correctly.""" diff --git a/tests/test_utils.py b/tests/test_utils.py index 7266ffb6..c2d887b6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -126,9 +126,9 @@ def test_semanticversion_invalid() -> None: ] for input_, exc_msg in invalid_inputs: with pytest.raises(ValueError, match=exc_msg): - SemanticVersion( # pylint: disable=expression-not-assigned - **input_ - ) if isinstance(input_, dict) else SemanticVersion(input_) + SemanticVersion(**input_) if isinstance(input_, dict) else SemanticVersion( + input_ + ) def test_semanticversion_invalid_comparisons() -> None: From 7e50658e9cf2f903973b06a08b7d931cd3bc5160 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 16 Oct 2023 10:23:24 +0200 Subject: [PATCH 02/12] Remove ruff CI job Ruff is already run as part of pre-commit. --- .github/workflows/_local_ci_tests.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/_local_ci_tests.yml b/.github/workflows/_local_ci_tests.yml index 401fb221..65f00631 100644 --- a/.github/workflows/_local_ci_tests.yml +++ b/.github/workflows/_local_ci_tests.yml @@ -29,16 +29,6 @@ jobs: package_dirs: ci_cd debug: false - ruff: - name: ruff - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - uses: chartboost/ruff-action@v1 - pytest: name: pytest runs-on: ${{ matrix.os }} From b47d72a74ddac516176461520e3e77e3672a2de9 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 25 Oct 2023 09:33:36 +0200 Subject: [PATCH 03/12] Fix indentation --- .pre-commit-config.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 22a3b157..4bec8b62 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,8 +25,10 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.0.292 hooks: - - id: ruff - args: [ --fix, --exit-non-zero-on-fix ] + - id: ruff + # Fix what can be fixed in-place and exit with non-zero status if files were + # changed and/or there are rules violations. + args: [--fix, --exit-non-zero-on-fix] # Black is a code style and formatter # It works on files in-place From 47273777ef5e7b7bf9c1983a2bcf6c8fa4ed8c32 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 25 Oct 2023 09:34:05 +0200 Subject: [PATCH 04/12] Use the latest ruff version --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4bec8b62..2315dbbb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: # More information can be found in its documentation: # https://docs.astral.sh/ruff/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.292 + rev: v0.1.2 hooks: - id: ruff # Fix what can be fixed in-place and exit with non-zero status if files were From 05557401261cc481051b08fb2e91d9940ca65e16 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 25 Oct 2023 09:46:29 +0200 Subject: [PATCH 05/12] Add explicit rule set to ruff Extend arguments to ruff CLI, ensuring no unsafe fixes are made and any fixes are shown explicitly in the output. --- .pre-commit-config.yaml | 6 +++++- pyproject.toml | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2315dbbb..dd6bb92b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,11 @@ repos: - id: ruff # Fix what can be fixed in-place and exit with non-zero status if files were # changed and/or there are rules violations. - args: [--fix, --exit-non-zero-on-fix] + args: + - "--fix" + - "--exit-non-zero-on-fix" + - "--show-fixes" + - "--no-unsafe-fixes" # Black is a code style and formatter # It works on files in-place diff --git a/pyproject.toml b/pyproject.toml index 72209008..a8b04264 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,3 +76,25 @@ filterwarnings = [ # Remove when invoke updates to `inspect.signature()` or similar: "ignore:.*inspect.getargspec().*:DeprecationWarning", ] + +[tool.ruff.lint] +select = [ + # pycodestyle + "E", + # Pyflakes + "F", + # pyupgrade + "UP", + # flake8-bugbear + "B", + # flake8-simplify + "SIM", + # flake8-blind-except + "BLE", + # isort + "I", + # pylint + "PL", + # ruff + "RUF", +] From 1d831029ac38eacb497eebe051777d7736946544 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 25 Oct 2023 10:13:46 +0200 Subject: [PATCH 06/12] Split ruff jobs according to (non-)code base Extend rule set for job on code base to include flake8-blind-except and pylint. Ignore self assignment of variable rule. Ignore all too-many-* rules from pylint, because we will extend the recommended number of branches, statements, function args, etc. in this code base. Otherwise, run with a rule set that includes pycodestyle, pyflakes, pyupgrade, flake8-bugbear, flake8-simplify, isort, and ruff. --- .pre-commit-config.yaml | 32 ++++++++++ ci_cd/tasks/update_deps.py | 88 ++++++++++++++------------ pyproject.toml | 4 -- tests/tasks/test_api_reference_docs.py | 2 +- tests/tasks/test_update_deps.py | 55 ++++++++++++---- 5 files changed, 123 insertions(+), 58 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dd6bb92b..e6ad990c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,6 +26,38 @@ repos: rev: v0.1.2 hooks: - id: ruff + name: ruff core code base + exclude: ^(\.github|tests)/.*$ + # Fix what can be fixed in-place and exit with non-zero status if files were + # changed and/or there are rules violations. + args: + - "--fix" + - "--exit-non-zero-on-fix" + - "--show-fixes" + - "--no-unsafe-fixes" + # Extend rule set to includ: + # flake8-blind-except + - "--extend-select=BLE" + # pylint + - "--extend-select=PL" + # Self assignment of variable + - "--extend-ignore=PLW0127" + # too-many-* rules + # Ignore these, as they are not relevant for our code base + # We will, e.g., extend the recommended number of branches, statements, function + # args, etc. + - "--extend-ignore=PLR09" + + # ruff is a Python linter, incl. import sorter and formatter + # It works partly on files in-place + # More information can be found in its documentation: + # https://docs.astral.sh/ruff/ + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.2 + hooks: + - id: ruff + name: ruff non-core code base + exclude: ^ci_cd/.*$ # Fix what can be fixed in-place and exit with non-zero status if files were # changed and/or there are rules violations. args: diff --git a/ci_cd/tasks/update_deps.py b/ci_cd/tasks/update_deps.py index 3ea5f528..1a552336 100644 --- a/ci_cd/tasks/update_deps.py +++ b/ci_cd/tasks/update_deps.py @@ -64,7 +64,7 @@ def update_deps( ): """Update dependencies in specified Python package's `pyproject.toml`.""" if TYPE_CHECKING: # pragma: no cover - context: "Context" = context # type: ignore[no-redef] + context: Context = context # type: ignore[no-redef] root_repo_path: str = root_repo_path # type: ignore[no-redef] fail_fast: bool = fail_fast # type: ignore[no-redef] pre_commit: bool = pre_commit # type: ignore[no-redef] @@ -97,7 +97,7 @@ def update_deps( if pre_commit and root_repo_path == ".": # Use git to determine repo root - result: "Result" = context.run("git rev-parse --show-toplevel", hide=True) + result: Result = context.run("git rev-parse --show-toplevel", hide=True) root_repo_path = result.stdout.strip("\n") pyproject_path = Path(root_repo_path).resolve() / "pyproject.toml" @@ -180,7 +180,7 @@ def update_deps( continue # Check version from PyPI's online package index - out: "Result" = context.run( + out: Result = context.run( f"pip index versions --python-version {py_version} {version_spec.package}", hide=True, ) @@ -228,10 +228,10 @@ def update_deps( # Apply ignore rules if version_spec.package in ignore_rules or "*" in ignore_rules: - versions: "list[dict[Literal['operator', 'version'], str]]" = [] - update_types: "dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]" = ( # noqa: E501 - {} - ) + versions: list[dict[Literal["operator", "version"], str]] = [] + update_types: dict[ + Literal["version-update"], list[Literal["major", "minor", "patch"]] + ] = {} if "*" in ignore_rules: versions, update_types = parse_ignore_rules(ignore_rules["*"]) @@ -301,7 +301,7 @@ def update_deps( def parse_ignore_entries( entries: list[str], separator: str -) -> 'dict[str, dict[Literal["versions", "update-types"], list[str]]]': +) -> dict[str, dict[Literal["versions", "update-types"], list[str]]]: """Parser for the `--ignore` option. The `--ignore` option values are given as key/value-pairs in the form: @@ -316,9 +316,7 @@ def parse_ignore_entries( A parsed mapping of dependencies to ignore rules. """ - ignore_entries: 'dict[str, dict[Literal["versions", "update-types"], list[str]]]' = ( # noqa: E501 - {} - ) + ignore_entries: dict[str, dict[Literal["versions", "update-types"], list[str]]] = {} for entry in entries: pairs = entry.split(separator, maxsplit=2) @@ -330,9 +328,9 @@ def parse_ignore_entries( f"value: --ignore={entry}" ) - ignore_entry: 'dict[Literal["dependency-name", "versions", "update-types"], str]' = ( # noqa: E501 - {} - ) + ignore_entry: dict[ + Literal["dependency-name", "versions", "update-types"], str + ] = {} for pair in pairs: match = re.match( r"^(?Pdependency-name|versions|update-types)=(?P.*)$", @@ -371,8 +369,11 @@ def parse_ignore_entries( def parse_ignore_rules( - rules: "dict[Literal['versions', 'update-types'], list[str]]", -) -> "tuple[list[dict[Literal['operator', 'version'], str]], dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]]": # noqa: E501 + rules: dict[Literal["versions", "update-types"], list[str]], +) -> tuple[ + list[dict[Literal["operator", "version"], str]], + dict[Literal["version-update"], list[Literal["major", "minor", "patch"]]], +]: """Parser for a specific set of ignore rules. Parameters: @@ -386,10 +387,10 @@ def parse_ignore_rules( # Ignore package altogether return [{"operator": ">=", "version": "0"}], {} - versions: 'list[dict[Literal["operator", "version"], str]]' = [] - update_types: "dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]" = ( # noqa: E501 - {} - ) + versions: list[dict[Literal["operator", "version"], str]] = [] + update_types: dict[ + Literal["version-update"], list[Literal["major", "minor", "patch"]] + ] = {} if "versions" in rules: for versions_entry in rules["versions"]: @@ -428,7 +429,7 @@ def parse_ignore_rules( def _ignore_version_rules( latest: list[str], - version_rules: "list[dict[Literal['operator', 'version'], str]]", + version_rules: list[dict[Literal["operator", "version"], str]], ) -> bool: """Determine whether to ignore package based on `versions` input.""" semver_latest = SemanticVersion(".".join(latest)) @@ -451,7 +452,7 @@ def _ignore_version_rules( semver_latest, semver_version_rule ): decision_version_rule = True - elif "~=" == version_rule["operator"]: + elif version_rule["operator"] == "~=": if "." not in version_rule["version"]: raise InputError( "Ignore option value error. For the 'versions' config key, when " @@ -487,7 +488,9 @@ def _ignore_version_rules( def _ignore_semver_rules( current: list[str], latest: list[str], - semver_rules: "dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]", # noqa: E501 + semver_rules: dict[ + Literal["version-update"], list[Literal["major", "minor", "patch"]] + ], ) -> bool: """If ANY of the semver rules are True, ignore the version.""" if any( @@ -498,28 +501,29 @@ def _ignore_semver_rules( f"'patch' (you gave {semver_rules['version-update']!r})." ) - if "major" in semver_rules["version-update"]: - if latest[0] != current[0]: - return True + # Number of version parts in a "minor" and "patch" version + len_minor = 2 + len_patch = 3 - elif "minor" in semver_rules["version-update"]: - if ( - len(latest) >= 2 - and len(current) >= 2 + if ( + ("major" in semver_rules["version-update"] and latest[0] != current[0]) + or ( + "minor" in semver_rules["version-update"] + and len(latest) >= len_minor + and len(current) >= len_minor and latest[1] > current[1] and latest[0] == current[0] - ): - return True - - elif "patch" in semver_rules["version-update"]: - if ( - len(latest) >= 3 - and len(current) >= 3 + ) + or ( + "patch" in semver_rules["version-update"] + and len(latest) >= len_patch + and len(current) >= len_patch and latest[2] > current[2] and latest[0] == current[0] and latest[1] == current[1] - ): - return True + ) + ): + return True return False @@ -527,8 +531,10 @@ def _ignore_semver_rules( def ignore_version( current: list[str], latest: list[str], - version_rules: "list[dict[Literal['operator', 'version'], str]]", - semver_rules: "dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]", # noqa: E501 + version_rules: list[dict[Literal["operator", "version"], str]], + semver_rules: dict[ + Literal["version-update"], list[Literal["major", "minor", "patch"]] + ], ) -> bool: """Determine whether the latest version can be ignored. diff --git a/pyproject.toml b/pyproject.toml index a8b04264..9172784e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,12 +89,8 @@ select = [ "B", # flake8-simplify "SIM", - # flake8-blind-except - "BLE", # isort "I", - # pylint - "PL", # ruff "RUF", ] diff --git a/tests/tasks/test_api_reference_docs.py b/tests/tasks/test_api_reference_docs.py index b1129b7c..b1c6d528 100644 --- a/tests/tasks/test_api_reference_docs.py +++ b/tests/tasks/test_api_reference_docs.py @@ -435,7 +435,7 @@ def test_larger_package(tmp_path: "Path") -> None: package_dir / "module" / "submodule", package_dir / "second_module", ] - for destination in [package_dir] + new_submodules: + for destination in [package_dir, *new_submodules]: shutil.copytree( src=Path(__file__).resolve().parent.parent.parent / "ci_cd", dst=destination, diff --git a/tests/tasks/test_update_deps.py b/tests/tasks/test_update_deps.py index 53512a8a..380a3cbd 100644 --- a/tests/tasks/test_update_deps.py +++ b/tests/tasks/test_update_deps.py @@ -10,7 +10,7 @@ from typing import Literal -def test_update_deps(tmp_path: "Path", caplog: pytest.LogCaptureFixture) -> None: +def test_update_deps(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: """Check update_deps runs with defaults.""" import re @@ -260,7 +260,9 @@ def test_update_deps(tmp_path: "Path", caplog: pytest.LogCaptureFixture) -> None def test_parse_ignore_entries( entries: list[str], separator: str, - expected_outcome: 'dict[str, dict[Literal["dependency-name", "versions", "update-types"], str]]', # noqa: E501 + expected_outcome: dict[ + str, dict[Literal["dependency-name", "versions", "update-types"], str] + ], ) -> None: """Check the `--ignore` option values are parsed as expected.""" from ci_cd.tasks.update_deps import parse_ignore_entries @@ -324,8 +326,11 @@ def test_parse_ignore_entries( ], ) def test_parse_ignore_rules( - rules: 'dict[Literal["versions", "update-types"], list[str]]', - expected_outcome: "tuple[list[dict[Literal['operator', 'version'], str]], dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]]", # noqa: E501 + rules: dict[Literal["versions", "update-types"], list[str]], + expected_outcome: tuple[ + list[dict[Literal["operator", "version"], str]], + dict[Literal["version-update"], list[Literal["major", "minor", "patch"]]], + ], ) -> None: """Check a specific set of ignore rules is parsed as expected.""" from ci_cd.tasks.update_deps import parse_ignore_rules @@ -345,14 +350,31 @@ def test_parse_ignore_rules( def _parametrize_ignore_version() -> ( - "dict[str, tuple[str, str, list[dict[Literal['operator', 'version'], str]], dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]], bool]]" # noqa: E501 + dict[ + str, + tuple[ + str, + str, + list[dict[Literal["operator", "version"], str]], + dict[Literal["version-update"], list[Literal["major", "minor", "patch"]]], + bool, + ], + ] ): """Utility function for `test_ignore_version()`. The parametrized inputs are created in this function in order to have more meaningful IDs in the runtime overview. """ - test_cases: "list[tuple[str, str, list[dict[Literal['operator', 'version'], str]], dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]], bool]]" = [ # noqa: E501 + test_cases: list[ + tuple[ + str, + str, + list[dict[Literal["operator", "version"], str]], + dict[Literal["version-update"], list[Literal["major", "minor", "patch"]]], + bool, + ] + ] = [ ("1.1.1", "2.2.2", [{"operator": ">", "version": "2.2.2"}], {}, False), ("1.1.1", "2.2.2", [{"operator": ">", "version": "2.2"}], {}, True), ("1.1.1", "2.2.2", [{"operator": ">", "version": "2"}], {}, True), @@ -869,9 +891,16 @@ def _parametrize_ignore_version() -> ( ), ("1.1.1", "1.1.2", [], {}, True), ] - res: "dict[str, tuple[str, str, list[dict[Literal['operator', 'version'], str]], dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]], bool]]" = ( # noqa: E501 - {} - ) + res: dict[ + str, + tuple[ + str, + str, + list[dict[Literal["operator", "version"], str]], + dict[Literal["version-update"], list[Literal["major", "minor", "patch"]]], + bool, + ], + ] = {} for test_case in test_cases: if test_case[2] and test_case[3]: operator_version = ",".join( @@ -903,8 +932,10 @@ def _parametrize_ignore_version() -> ( def test_ignore_version( current: str, latest: str, - version_rules: "list[dict[Literal['operator', 'version'], str]]", - semver_rules: "dict[Literal['version-update'], list[Literal['major', 'minor', 'patch']]]", # noqa: E501 + version_rules: list[dict[Literal["operator", "version"], str]], + semver_rules: dict[ + Literal["version-update"], list[Literal["major", "minor", "patch"]] + ], expected_outcome: bool, ) -> None: """Check the expected ignore rules are resolved correctly.""" @@ -1074,7 +1105,7 @@ def test_ignore_version_fails() -> None: ], ) def test_ignore_rules_logic( - tmp_path: "Path", ignore_rules: list[str], expected_result: dict[str, str] + tmp_path: Path, ignore_rules: list[str], expected_result: dict[str, str] ) -> None: """Check the workflow of multiple interconnecting ignore rules are respected.""" import re From 18a8bae4a067ab9b4426decdc6d394a822fdab3a Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 25 Oct 2023 10:24:06 +0200 Subject: [PATCH 07/12] Include flake8-bandit rule set for code base --- .pre-commit-config.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e6ad990c..a8940299 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,9 @@ repos: - "--exit-non-zero-on-fix" - "--show-fixes" - "--no-unsafe-fixes" - # Extend rule set to includ: + # Extend rule set to include: + # flake8-bandit + - "--extend-select=S" # flake8-blind-except - "--extend-select=BLE" # pylint From b454e3a49cfa4ea0981991742c3d9d83d9f3248f Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Fri, 10 Nov 2023 16:26:58 +0100 Subject: [PATCH 08/12] Add pyupgrade pre-commit hook Enforce importing __future__.annotations in all Python files. Update code accordingly. --- .pre-commit-config.yaml | 9 +++++ ci_cd/__init__.py | 2 ++ ci_cd/exceptions.py | 1 + ci_cd/main.py | 2 ++ ci_cd/tasks/__init__.py | 2 ++ ci_cd/tasks/api_reference_docs.py | 6 ++-- ci_cd/tasks/docs_index.py | 8 +++-- ci_cd/tasks/setver.py | 2 ++ ci_cd/utils/__init__.py | 2 ++ ci_cd/utils/console_printing.py | 8 +++-- ci_cd/utils/file_io.py | 5 +-- ci_cd/utils/versions.py | 48 +++++++++++++------------- pyproject.toml | 2 ++ tests/conftest.py | 2 ++ tests/tasks/test_api_reference_docs.py | 14 ++++---- 15 files changed, 73 insertions(+), 40 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3df86d14..3145192b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,6 +18,15 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] + # pyupgrade is a tool for automatically upgrading Python syntax for newer versions of + # the language + # It works on files in-place + - repo: https://github.com/asottile/pyupgrade + rev: v3.15.0 + hooks: + - id: pyupgrade + args: [--py37-plus] + # Black is a code style and formatter # It works on files in-place - repo: https://github.com/ambv/black diff --git a/ci_cd/__init__.py b/ci_cd/__init__.py index 98baf946..8a21d7c2 100644 --- a/ci_cd/__init__.py +++ b/ci_cd/__init__.py @@ -1,4 +1,6 @@ """CI/CD Tools. Tiny package to run invoke tasks as a standalone program.""" +from __future__ import annotations + import logging __version__ = "2.5.3" diff --git a/ci_cd/exceptions.py b/ci_cd/exceptions.py index 1a645495..ae1ac9f0 100644 --- a/ci_cd/exceptions.py +++ b/ci_cd/exceptions.py @@ -1,4 +1,5 @@ """CI/CD-specific exceptions.""" +from __future__ import annotations class CICDException(Exception): diff --git a/ci_cd/main.py b/ci_cd/main.py index 65cedcec..ea224be9 100644 --- a/ci_cd/main.py +++ b/ci_cd/main.py @@ -3,6 +3,8 @@ See [invoke documentation](https://docs.pyinvoke.org/en/stable/concepts/library.html) for more information. """ +from __future__ import annotations + from invoke import Collection, Program from ci_cd import __version__, tasks diff --git a/ci_cd/tasks/__init__.py b/ci_cd/tasks/__init__.py index 78deeeba..53b22d52 100644 --- a/ci_cd/tasks/__init__.py +++ b/ci_cd/tasks/__init__.py @@ -3,6 +3,8 @@ Repository management tasks powered by `invoke`. More information on `invoke` can be found at [pyinvoke.org](http://www.pyinvoke.org/). """ +from __future__ import annotations + from .api_reference_docs import create_api_reference_docs from .docs_index import create_docs_index from .setver import setver diff --git a/ci_cd/tasks/api_reference_docs.py b/ci_cd/tasks/api_reference_docs.py index b7b1992e..6d96a10e 100644 --- a/ci_cd/tasks/api_reference_docs.py +++ b/ci_cd/tasks/api_reference_docs.py @@ -3,6 +3,8 @@ Create Python API reference in the documentation. This is specifically to be used with the MkDocs and mkdocstrings framework. """ +from __future__ import annotations + import logging import os import re @@ -111,7 +113,7 @@ def create_api_reference_docs( ): """Create the Python API Reference in the documentation.""" if TYPE_CHECKING: # pragma: no cover - context: "Context" = context # type: ignore[no-redef] + context: Context = context # type: ignore[no-redef] pre_clean: bool = pre_clean # type: ignore[no-redef] pre_commit: bool = pre_commit # type: ignore[no-redef] root_repo_path: str = root_repo_path # type: ignore[no-redef] @@ -148,7 +150,7 @@ def write_file(full_path: Path, content: str) -> None: if pre_commit: # Ensure git is installed - result: "Result" = context.run("git --version", hide=True) + result: Result = context.run("git --version", hide=True) if result.exited != 0: sys.exit( "Git is not installed. Please install it before running this task." diff --git a/ci_cd/tasks/docs_index.py b/ci_cd/tasks/docs_index.py index 474afe0d..5f2a208c 100644 --- a/ci_cd/tasks/docs_index.py +++ b/ci_cd/tasks/docs_index.py @@ -2,6 +2,8 @@ Create the documentation index (home) page from `README.md`. """ +from __future__ import annotations + import re import sys from pathlib import Path @@ -48,7 +50,7 @@ def create_docs_index( ): """Create the documentation index page from README.md.""" if TYPE_CHECKING: # pragma: no cover - context: "Context" = context # type: ignore[no-redef] + context: Context = context # type: ignore[no-redef] pre_commit: bool = pre_commit # type: ignore[no-redef] root_repo_path: str = root_repo_path # type: ignore[no-redef] replacement_separator: str = replacement_separator # type: ignore[no-redef] @@ -61,7 +63,7 @@ def create_docs_index( if pre_commit and root_repo_path == ".": # Use git to determine repo root - result: "Result" = context.run("git rev-parse --show-toplevel", hide=True) + result: Result = context.run("git rev-parse --show-toplevel", hide=True) root_repo_path = result.stdout.strip("\n") root_repo_path: Path = Path(root_repo_path).resolve() @@ -90,7 +92,7 @@ def create_docs_index( # NOTE: Concerning the weird regular expression, see: # http://manpages.ubuntu.com/manpages/precise/en/man1/git-status.1.html - result: "Result" = context.run( # type: ignore[no-redef] + result: Result = context.run( # type: ignore[no-redef] f'git -C "{root_repo_path}" status --porcelain ' f"{docs_index.relative_to(root_repo_path)}", hide=True, diff --git a/ci_cd/tasks/setver.py b/ci_cd/tasks/setver.py index e231b861..d3fc3f28 100644 --- a/ci_cd/tasks/setver.py +++ b/ci_cd/tasks/setver.py @@ -2,6 +2,8 @@ Set the specified version. """ +from __future__ import annotations + import logging import re import sys diff --git a/ci_cd/utils/__init__.py b/ci_cd/utils/__init__.py index e7f93c04..ea2fd125 100644 --- a/ci_cd/utils/__init__.py +++ b/ci_cd/utils/__init__.py @@ -1,4 +1,6 @@ """Utilities for CI/CD.""" +from __future__ import annotations + from .console_printing import Emoji, error_msg, info_msg, warning_msg from .file_io import update_file from .versions import ( diff --git a/ci_cd/utils/console_printing.py b/ci_cd/utils/console_printing.py index ad15d923..6e5517d2 100644 --- a/ci_cd/utils/console_printing.py +++ b/ci_cd/utils/console_printing.py @@ -1,4 +1,6 @@ """Relevant tools for printing to the console.""" +from __future__ import annotations + import platform from enum import Enum @@ -6,7 +8,7 @@ class Emoji(str, Enum): """Unicode strings for certain emojis.""" - def __new__(cls, value: str) -> "Emoji": + def __new__(cls, value: str) -> Emoji: obj = str.__new__(cls, value) if platform.system() == "Windows": # Windows does not support unicode emojis, so we replace them with @@ -25,7 +27,7 @@ def __new__(cls, value: str) -> "Emoji": class Color(str, Enum): """ANSI escape sequences for colors.""" - def __new__(cls, value: str) -> "Color": + def __new__(cls, value: str) -> Color: obj = str.__new__(cls, value) obj._value_ = value return obj @@ -47,7 +49,7 @@ def write(self, text: str) -> str: class Formatting(str, Enum): """ANSI escape sequences for formatting.""" - def __new__(cls, value: str) -> "Formatting": + def __new__(cls, value: str) -> Formatting: obj = str.__new__(cls, value) obj._value_ = value return obj diff --git a/ci_cd/utils/file_io.py b/ci_cd/utils/file_io.py index b10105da..3b083ae3 100644 --- a/ci_cd/utils/file_io.py +++ b/ci_cd/utils/file_io.py @@ -1,14 +1,15 @@ """Utilities for handling IO operations.""" +from __future__ import annotations + import re from typing import TYPE_CHECKING if TYPE_CHECKING: # pragma: no cover from pathlib import Path - from typing import Optional, Tuple def update_file( - filename: "Path", sub_line: "Tuple[str, str]", strip: "Optional[str]" = None + filename: Path, sub_line: tuple[str, str], strip: str | None = None ) -> None: """Utility function for tasks to read, update, and write files""" if strip is None and filename.suffix == ".md": diff --git a/ci_cd/utils/versions.py b/ci_cd/utils/versions.py index 4309f439..dbe1229f 100644 --- a/ci_cd/utils/versions.py +++ b/ci_cd/utils/versions.py @@ -12,7 +12,7 @@ from ci_cd.exceptions import InputError, InputParserError, UnableToResolve if TYPE_CHECKING: # pragma: no cover - from typing import Any, Literal, Optional, Union + from typing import Any, Literal from packaging.requirements import Requirement @@ -98,7 +98,7 @@ class SemanticVersion(str): @no_type_check def __new__( - cls, version: Optional[str] = None, **kwargs: Union[str, int] + cls, version: str | None = None, **kwargs: str | int ) -> SemanticVersion: return super().__new__( cls, version if version else cls._build_version(**kwargs) @@ -106,13 +106,13 @@ def __new__( def __init__( self, - version: Optional[str] = None, + version: str | None = None, *, - major: Union[str, int] = "", - minor: Optional[Union[str, int]] = None, - patch: Optional[Union[str, int]] = None, - pre_release: Optional[str] = None, - build: Optional[str] = None, + major: str | int = "", + minor: str | int | None = None, + patch: str | int | None = None, + pre_release: str | None = None, + build: str | None = None, ) -> None: if version is not None: if major or minor or patch or pre_release or build: @@ -137,11 +137,11 @@ def __init__( @classmethod def _build_version( cls, - major: Optional[Union[str, int]] = None, - minor: Optional[Union[str, int]] = None, - patch: Optional[Union[str, int]] = None, - pre_release: Optional[str] = None, - build: Optional[str] = None, + major: str | int | None = None, + minor: str | int | None = None, + patch: str | int | None = None, + pre_release: str | None = None, + build: str | None = None, ) -> str: """Build a version from the given parameters.""" if major is None: @@ -187,7 +187,7 @@ def patch(self) -> int: return self._patch @property - def pre_release(self) -> Union[None, str]: + def pre_release(self) -> None | str: """The pre-release part of the version This is the part supplied after a minus (`-`), but before a plus (`+`). @@ -195,7 +195,7 @@ def pre_release(self) -> Union[None, str]: return self._pre_release @property - def build(self) -> Union[None, str]: + def build(self) -> None | str: """The build metadata part of the version. This is the part supplied at the end of the version, after a plus (`+`). @@ -308,7 +308,7 @@ def next_version(self, version_part: str) -> SemanticVersion: return self.__class__(next_version) def previous_version( - self, version_part: str, max_filler: Optional[Union[str, int]] = 99 + self, version_part: str, max_filler: str | int | None = 99 ) -> SemanticVersion: """Return the previous version for the specified version part. @@ -680,11 +680,11 @@ def ignore_version( def regenerate_requirement( requirement: Requirement, *, - name: Optional[str] = None, - extras: Optional[set[str]] = None, - specifier: Optional[Union[SpecifierSet, str]] = None, - url: Optional[str] = None, - marker: Optional[Union[Marker, str]] = None, + name: str | None = None, + extras: set[str] | None = None, + specifier: SpecifierSet | str | None = None, + url: str | None = None, + marker: Marker | str | None = None, post_name_space: bool = False, ) -> str: """Regenerate a requirement string including the given parameters. @@ -736,7 +736,7 @@ def regenerate_requirement( def update_specifier_set( - latest_version: Union[SemanticVersion, str], current_specifier_set: SpecifierSet + latest_version: SemanticVersion | str, current_specifier_set: SpecifierSet ) -> SpecifierSet: """Update the specifier set to include the latest version.""" logger = logging.getLogger(__name__) @@ -844,7 +844,7 @@ def update_specifier_set( # current specifier set is valid as is and already includes the latest version if updated_specifiers != [""]: # Otherwise, add updated specifier(s) to new specifier set - new_specifier_set |= set(Specifier(_) for _ in updated_specifiers) + new_specifier_set |= {Specifier(_) for _ in updated_specifiers} else: raise UnableToResolve( "Cannot resolve how to update specifier set to include latest version." @@ -880,7 +880,7 @@ def _semi_valid_python_version(version: SemanticVersion) -> bool: def get_min_max_py_version( - requires_python: Union[str, Marker], + requires_python: str | Marker, ) -> str: """Get minimum or maximum Python version from `requires_python`. diff --git a/pyproject.toml b/pyproject.toml index a0767925..643dc566 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,3 +94,5 @@ select = [ # ruff "RUF", ] +# Import __future__.annotations for all Python files. +isort.required-imports = ["from __future__ import annotations"] diff --git a/tests/conftest.py b/tests/conftest.py index 22b57de8..c5b86ba9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,6 @@ """Test fixtures.""" +from __future__ import annotations + import pytest diff --git a/tests/tasks/test_api_reference_docs.py b/tests/tasks/test_api_reference_docs.py index 33d0377d..bcd3193b 100644 --- a/tests/tasks/test_api_reference_docs.py +++ b/tests/tasks/test_api_reference_docs.py @@ -1,11 +1,13 @@ """Test `ci_cd.tasks.api_reference_docs`.""" +from __future__ import annotations + from typing import TYPE_CHECKING if TYPE_CHECKING: from pathlib import Path -def test_default_run(tmp_path: "Path") -> None: +def test_default_run(tmp_path: Path) -> None: """Check create_api_reference_docs runs with defaults.""" import os import shutil @@ -93,7 +95,7 @@ def test_default_run(tmp_path: "Path") -> None: ) == "# versions\n\n::: ci_cd.utils.versions\n" -def test_nested_package(tmp_path: "Path") -> None: +def test_nested_package(tmp_path: Path) -> None: """Check create_api_reference_docs generates correct link to sub-nested package directory.""" import os @@ -183,7 +185,7 @@ def test_nested_package(tmp_path: "Path") -> None: ) == "# versions\n\n::: src.ci_cd.ci_cd.utils.versions\n" -def test_special_options(tmp_path: "Path") -> None: +def test_special_options(tmp_path: Path) -> None: """Check create_api_reference_docs generates correct markdown files with `--special-option`.""" import os @@ -308,7 +310,7 @@ def test_special_options(tmp_path: "Path") -> None: ) == "# versions\n\n::: ci_cd.utils.versions\n" -def test_special_options_multiple_packages(tmp_path: "Path") -> None: +def test_special_options_multiple_packages(tmp_path: Path) -> None: """Check create_api_reference_docs generates correct markdown files with `--special-option` for a multi-package repository.""" import os @@ -476,7 +478,7 @@ def test_special_options_multiple_packages(tmp_path: "Path") -> None: ) == f"# versions\n\n::: {package_name}.utils.versions\n" -def test_larger_package(tmp_path: "Path") -> None: +def test_larger_package(tmp_path: Path) -> None: """Check create_api_reference_docs runs with a more 'complete' package.""" import os import shutil @@ -646,7 +648,7 @@ def test_larger_package(tmp_path: "Path") -> None: ) -def test_larger_multi_packages(tmp_path: "Path") -> None: +def test_larger_multi_packages(tmp_path: Path) -> None: """Check create_api_reference_docs runs with a set of more 'complete' packages.""" import os import shutil From 9b4b175e0c5363922c8ca0f53e61791e8d65ba37 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Fri, 10 Nov 2023 17:03:40 +0100 Subject: [PATCH 09/12] Fix constant definitions --- ci_cd/utils/versions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ci_cd/utils/versions.py b/ci_cd/utils/versions.py index dbe1229f..48560c6c 100644 --- a/ci_cd/utils/versions.py +++ b/ci_cd/utils/versions.py @@ -1001,16 +1001,16 @@ def get_min_max_py_version( split_py_version = py_version.split(".") parsed_py_version = SemanticVersion(py_version) + # See the _semi_valid_python_version() function for these values + largest_value_for_a_patch_part = 18 + largest_value_for_a_minor_part = 12 + largest_value_for_any_part = largest_value_for_a_patch_part + while ( not _semi_valid_python_version(parsed_py_version) or py_version not in specifier_set ): if min_or_max == "min": - # See the _semi_valid_python_version() function for these values - largest_value_for_a_patch_part = 18 - largest_value_for_a_minor_part = 12 - largest_value_for_any_part = largest_value_for_a_patch_part - if parsed_py_version.patch >= largest_value_for_a_patch_part: parsed_py_version = parsed_py_version.next_version("minor") elif parsed_py_version.minor >= largest_value_for_a_minor_part: From b087615ce1c1da002416edc6a8879c4a9a4b3f38 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 22 Nov 2023 14:38:59 +0100 Subject: [PATCH 10/12] Update ruff rules to match scientific-python Or rather, base it on scientific-python and then update it to fit this package. Update the code base accordingly. --- .pre-commit-config.yaml | 32 ---------------------- ci_cd/tasks/setver.py | 14 ++++------ ci_cd/utils/console_printing.py | 10 ++++--- ci_cd/utils/versions.py | 12 ++++----- pyproject.toml | 47 ++++++++++++++++++++++++--------- tests/conftest.py | 2 +- tests/tasks/test_update_deps.py | 26 +++++++++--------- tests/utils/test_versions.py | 2 +- 8 files changed, 65 insertions(+), 80 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9ab4bbaa..2ee3e63d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,38 +45,6 @@ repos: rev: v0.1.5 hooks: - id: ruff - name: ruff core code base - exclude: ^(\.github|tests)/.*$ - # Fix what can be fixed in-place and exit with non-zero status if files were - # changed and/or there are rules violations. - args: - - "--fix" - - "--exit-non-zero-on-fix" - - "--show-fixes" - - "--no-unsafe-fixes" - # Extend rule set to include: - # flake8-blind-except - - "--extend-select=BLE" - # pylint - - "--extend-select=PL" - # Self assignment of variable - - "--extend-ignore=PLW0127" - # too-many-* rules - # Ignore these, as they are not relevant for our code base - # We will, e.g., extend the recommended number of branches, statements, function - # args, etc. - - "--extend-ignore=PLR09" - - # ruff is a Python linter, incl. import sorter and formatter - # It works partly on files in-place - # More information can be found in its documentation: - # https://docs.astral.sh/ruff/ - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.5 - hooks: - - id: ruff - name: ruff non-core code base - exclude: ^ci_cd/.*$ # Fix what can be fixed in-place and exit with non-zero status if files were # changed and/or there are rules violations. args: diff --git a/ci_cd/tasks/setver.py b/ci_cd/tasks/setver.py index d3fc3f28..642a7373 100644 --- a/ci_cd/tasks/setver.py +++ b/ci_cd/tasks/setver.py @@ -120,9 +120,7 @@ def setver( ) filepath = Path( - filepath.format( - **{"package_dir": package_dir, "version": semantic_version} - ) + filepath.format(package_dir=package_dir, version=semantic_version) ).resolve() if not filepath.exists(): error_msg = ( @@ -143,9 +141,7 @@ def setver( filepath, pattern, replacement, - replacement.format( - **{"package_dir": package_dir, "version": semantic_version} - ), + replacement.format(package_dir=package_dir, version=semantic_version), ) if test: print( @@ -154,7 +150,7 @@ def setver( ) print( "replacement (handled): " - f"{replacement.format(**{'package_dir': package_dir, 'version': semantic_version})}" # noqa: E501 + f"{replacement.format(package_dir=package_dir, version=semantic_version)}" # noqa: E501 ) try: @@ -163,7 +159,7 @@ def setver( ( pattern, replacement.format( - **{"package_dir": package_dir, "version": semantic_version} + package_dir=package_dir, version=semantic_version ), ), ) @@ -176,7 +172,7 @@ def setver( f"{Emoji.CROSS_MARK.value} Error: Could not update file {filepath}" f" according to the given input:\n\n pattern: {pattern}\n " "replacement: " - f"{replacement.format(**{'package_dir': package_dir, 'version': semantic_version})}" # noqa: E501 + f"{replacement.format(package_dir=package_dir, version=semantic_version)}" # noqa: E501 ) print( diff --git a/ci_cd/utils/console_printing.py b/ci_cd/utils/console_printing.py index 6e5517d2..f2e97f8c 100644 --- a/ci_cd/utils/console_printing.py +++ b/ci_cd/utils/console_printing.py @@ -3,12 +3,16 @@ import platform from enum import Enum +from typing import TYPE_CHECKING + +if TYPE_CHECKING: # pragma: no cover + from typing_extensions import Self class Emoji(str, Enum): """Unicode strings for certain emojis.""" - def __new__(cls, value: str) -> Emoji: + def __new__(cls, value: str) -> Self: obj = str.__new__(cls, value) if platform.system() == "Windows": # Windows does not support unicode emojis, so we replace them with @@ -27,7 +31,7 @@ def __new__(cls, value: str) -> Emoji: class Color(str, Enum): """ANSI escape sequences for colors.""" - def __new__(cls, value: str) -> Color: + def __new__(cls, value: str) -> Self: obj = str.__new__(cls, value) obj._value_ = value return obj @@ -49,7 +53,7 @@ def write(self, text: str) -> str: class Formatting(str, Enum): """ANSI escape sequences for formatting.""" - def __new__(cls, value: str) -> Formatting: + def __new__(cls, value: str) -> Self: obj = str.__new__(cls, value) obj._value_ = value return obj diff --git a/ci_cd/utils/versions.py b/ci_cd/utils/versions.py index a6cd6bcb..8f1e771b 100644 --- a/ci_cd/utils/versions.py +++ b/ci_cd/utils/versions.py @@ -15,7 +15,7 @@ from typing import Any, Dict, List from packaging.requirements import Requirement - from typing_extensions import Literal + from typing_extensions import Literal, Self IgnoreEntry = Dict[Literal["dependency-name", "versions", "update-types"], str] @@ -98,9 +98,7 @@ class SemanticVersion(str): ) @no_type_check - def __new__( - cls, version: str | None = None, **kwargs: str | int - ) -> SemanticVersion: + def __new__(cls, version: str | None = None, **kwargs: str | int) -> Self: return super().__new__( cls, version if version else cls._build_version(**kwargs) ) @@ -257,7 +255,7 @@ def __le__(self, other: Any) -> bool: """Less than or equal to (`<=`) rich comparison.""" return self.__lt__(other) or self.__eq__(other) - def __eq__(self, other: Any) -> bool: + def __eq__(self, other: object) -> bool: """Equal to (`==`) rich comparison.""" other_semver = self._validate_other_type(other) @@ -268,7 +266,7 @@ def __eq__(self, other: Any) -> bool: and self.pre_release == other_semver.pre_release ) - def __ne__(self, other: Any) -> bool: + def __ne__(self, other: object) -> bool: """Not equal to (`!=`) rich comparison.""" return not self.__eq__(other) @@ -872,7 +870,7 @@ def _semi_valid_python_version(version: SemanticVersion) -> bool: f"Invalid Python major version: {version.major}. Expected 1, 2, or 3." ) - if version.minor not in range(0, 12 + 1) or version.patch not in range(0, 18 + 1): + if version.minor not in range(12 + 1) or version.patch not in range(18 + 1): # Either: # Not a valid Python minor version (0, 1, 2, ..., 12) # Not a valid Python patch version (0, 1, 2, ..., 18) diff --git a/pyproject.toml b/pyproject.toml index 66e410a7..358feba2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,19 +84,40 @@ addopts = ["--cov=ci_cd", "--cov-report=term-missing"] filterwarnings = ["error"] [tool.ruff.lint] -select = [ - # pycodestyle - "E", - # Pyflakes - "F", - # flake8-bugbear - "B", - # flake8-simplify - "SIM", - # isort - "I", - # ruff - "RUF", +extend-select = [ + "E", # pycodestyle + "F", # Pyflakes + "B", # flake8-bugbear + "I", # isort + "BLE", # flake8-blind-except + "ARG", # flake8-unused-arguments + "C4", # flake8-comprehensions + "ICN", # flake8-import-conventions + "G", # flake8-logging-format + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PT", # flake8-pytest-style + "PTH", # flake8-use-pathlib + "RET", # flake8-return + "RUF", # Ruff-specific + "SIM", # flake8-simplify + "YTT", # flake8-2020 + "EXE", # flake8-executable + "PYI", # flake8-pyi ] +ignore = [ + "PLR", # Design related pylint codes + "PLW0127", # pylint: Self-assignment of variables +] + # Import __future__.annotations for all Python files. isort.required-imports = ["from __future__ import annotations"] + +[tool.ruff.lint.per-file-ignores] +"tests/**" = [ + "BLE", # flake8-blind-except +] +".github/**" = [ + "BLE", # flake8-blind-except +] diff --git a/tests/conftest.py b/tests/conftest.py index c5b86ba9..0013ea1c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,7 @@ @pytest.fixture(autouse=True) -def clear_loggers() -> None: +def _clear_loggers() -> None: """Remove handlers from all loggers""" import logging diff --git a/tests/tasks/test_update_deps.py b/tests/tasks/test_update_deps.py index 4a9b0079..e8cc5078 100644 --- a/tests/tasks/test_update_deps.py +++ b/tests/tasks/test_update_deps.py @@ -82,19 +82,17 @@ def test_update_deps(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: context = MockContext( run={ - **{ - re.compile(r".*invoke$"): "invoke (1.7.1)\n", - re.compile(r".*tomlkit$"): "tomlkit (1.0.0)", - re.compile(r".*mike$"): "mike (1.1.1)", - re.compile(r".*pytest$"): "pytest (7.1.0)", - re.compile(r".*pytest-cov$"): "pytest-cov (3.1.5)", - re.compile(r".*pre-commit$"): "pre-commit (2.21.5)", - re.compile(r".*pylint$"): "pylint (2.14.2)", - re.compile(r".* A$"): "A (1.2.3)", - re.compile(r".*A.B-C_D$"): "A.B-C_D (1.2.3)", - re.compile(r".*aa$"): "aa (1.2.3)", - re.compile(r".*name$"): "name (1.2.3)", - }, + re.compile(r".*invoke$"): "invoke (1.7.1)\n", + re.compile(r".*tomlkit$"): "tomlkit (1.0.0)", + re.compile(r".*mike$"): "mike (1.1.1)", + re.compile(r".*pytest$"): "pytest (7.1.0)", + re.compile(r".*pytest-cov$"): "pytest-cov (3.1.5)", + re.compile(r".*pre-commit$"): "pre-commit (2.21.5)", + re.compile(r".*pylint$"): "pylint (2.14.2)", + re.compile(r".* A$"): "A (1.2.3)", + re.compile(r".*A.B-C_D$"): "A.B-C_D (1.2.3)", + re.compile(r".*aa$"): "aa (1.2.3)", + re.compile(r".*name$"): "name (1.2.3)", **{re.compile(rf".*name{i}$"): f"name{i} (3.2.1)" for i in range(1, 12)}, } ) @@ -672,7 +670,7 @@ def test_missing_project_package_name(tmp_path: Path) -> None: @pytest.mark.parametrize( - "dependency,optional_dependency,fail_fast", + ("dependency", "optional_dependency", "fail_fast"), [ ("(pytest)", "", False), ("", "(pytest)", False), diff --git a/tests/utils/test_versions.py b/tests/utils/test_versions.py index 6f38ba7c..59899007 100644 --- a/tests/utils/test_versions.py +++ b/tests/utils/test_versions.py @@ -1005,7 +1005,7 @@ def test_ignore_version_fails() -> None: @pytest.mark.parametrize( - ["requires_python", "expected_outcome"], + ("requires_python", "expected_outcome"), [ # Minimum operators # >= From 5c3f49e9925ead87f19feed991e3caac2d673980 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 7 Dec 2023 16:36:21 +0100 Subject: [PATCH 11/12] Remove pylint disable statement --- tests/utils/test_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils/test_versions.py b/tests/utils/test_versions.py index c8fc224c..201642bb 100644 --- a/tests/utils/test_versions.py +++ b/tests/utils/test_versions.py @@ -251,7 +251,7 @@ def test_semanticversion_python_version( if isinstance(version_, Version) or ( isinstance(version_, str) and re.match( - SemanticVersion._semver_regex, # pylint: disable=protected-access + SemanticVersion._semver_regex, version_, ) is None From 182dbb92b7d42c27f9fc32c92a53d9422b5536b8 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 7 Dec 2023 16:46:09 +0100 Subject: [PATCH 12/12] Fix wrong merge --- ci_cd/utils/versions.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ci_cd/utils/versions.py b/ci_cd/utils/versions.py index bbfc1475..f9ac7d87 100644 --- a/ci_cd/utils/versions.py +++ b/ci_cd/utils/versions.py @@ -911,13 +911,13 @@ def update_specifier_set( # Up only the last version segment of the latest version according to # what version segments are defined in the specifier version. if len(split_specifier_version) == PART_TO_LENGTH_MAPPING["major"]: - updated_version = str(latest_version.next_version("major").major) + updated_version += str(latest_version.next_version("major").major) elif len(split_specifier_version) == PART_TO_LENGTH_MAPPING["minor"]: - updated_version = ".".join( + updated_version += ".".join( latest_version.next_version("minor").split(".")[:2] ) elif len(split_specifier_version) == PART_TO_LENGTH_MAPPING["patch"]: - updated_version = latest_version.next_version("patch") + updated_version += latest_version.next_version("patch") else: raise UnableToResolve( "Invalid/unable to handle number of version parts: " @@ -1130,7 +1130,12 @@ def get_min_max_py_version( # See the _semi_valid_python_version() function for these values largest_value_for_a_patch_part = 18 largest_value_for_a_minor_part = 12 - largest_value_for_any_part = largest_value_for_a_patch_part + largest_value_for_a_major_part = 3 + largest_value_for_any_part = max( + largest_value_for_a_patch_part, + largest_value_for_a_minor_part, + largest_value_for_a_major_part, + ) while ( not _semi_valid_python_version(parsed_py_version)