Add experimental tool lockfiles for Black, Isort, Yapf, Coverage.py, Lambdex, and Protobuf MyPy

pants.toml

@@ -11,6 +11,7 @@ backend_packages.add = [
   "pants.backend.python.lint.docformatter",
   "pants.backend.python.lint.flake8",
   "pants.backend.python.lint.isort",
+  "pants.backend.python.lint.yapf",
   "pants.backend.python.typecheck.mypy",
   "pants.backend.python.mixed_interpreter_constraints",
   "pants.backend.shell",
@@ -75,6 +76,16 @@ root_patterns = [
 experimental_lockfile = "3rdparty/python/lockfile.txt"
 interpreter_constraints = [">=3.7,<3.10"]
 
+[python-protobuf.mypy-plugin]
+experimental_lockfile = "src/python/pants/backend/codegen/protobuf/python/mypy_protobuf_lockfile.txt"
+
+[lambdex]
+experimental_lockfile = "src/python/pants/backend/awslambda/python/lambdex_lockfile.txt"
+
+[black]
+# TODO(#12200): Figure out how to get this lockfile to work with multiple interpreters.
+# experimental_lockfile = "src/python/pants/backend/python/lint/black/lockfile.txt"
+
 [docformatter]
 args = ["--wrap-summaries=100", "--wrap-descriptions=100"]
 experimental_lockfile = "src/python/pants/backend/python/lint/docformatter/lockfile.txt"
@@ -87,7 +98,12 @@ extra_requirements.add = [
 ]
 
 [isort]
-version = "isort[pyproject,colors]>=5.5.1,<5.6"
+experimental_lockfile = "src/python/pants/backend/python/lint/isort/lockfile.txt"
+
+[yapf]
+# We only activate this to generate and maintain a lockfile - we don't actually want to use yapf.
+skip = true
+experimental_lockfile = "src/python/pants/backend/python/lint/yapf/lockfile.txt"
 
 [shellcheck]
 args = ["--external-sources"]
@@ -166,6 +182,7 @@ extra_env_vars = [
 
 [coverage-py]
 interpreter_constraints = [">=3.7,<3.10"]
+experimental_lockfile = "src/python/pants/backend/python/subsystems/coverage_py_lockfile.txt"
 
 [sourcefile-validation]
 config = "@build-support/regexes/config.yaml"

src/python/pants/backend/awslambda/python/lambdex.py

@@ -1,15 +1,42 @@
 # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+from pants.backend.experimental.python.lockfile import (
+    PythonToolLockfileRequest,
+    PythonToolLockfileSentinel,
+)
 from pants.backend.python.subsystems.python_tool_base import PythonToolBase
 from pants.backend.python.target_types import ConsoleScript
+from pants.engine.rules import collect_rules, rule
+from pants.engine.unions import UnionRule
+from pants.util.docutil import git_url
 
 
 class Lambdex(PythonToolBase):
     options_scope = "lambdex"
     help = "A tool for turning .pex files into AWS Lambdas (https://github.com/wickman/lambdex)."
 
     default_version = "lambdex==0.1.4"
+    default_main = ConsoleScript("lambdex")
+
     register_interpreter_constraints = True
     default_interpreter_constraints = ["CPython>=3.6"]
-    default_main = ConsoleScript("lambdex")
+
+    register_lockfile = True
+    default_lockfile_resource = ("pants.backend.awslambda.python", "lambdex_lockfile.txt")
+    default_lockfile_url = git_url("src/python/pants/backend/awslambda/python/lambdex_lockfile.txt")
+
+
+class LambdexLockfileSentinel(PythonToolLockfileSentinel):
+    pass
+
+
+@rule
+def setup_lambdex_lockfile(
+    _: LambdexLockfileSentinel, lambdex: Lambdex
+) -> PythonToolLockfileRequest:
+    return PythonToolLockfileRequest.from_tool(lambdex)
+
+
+def rules():
+    return (*collect_rules(), UnionRule(PythonToolLockfileSentinel, LambdexLockfileSentinel))

src/python/pants/backend/awslambda/python/lambdex_lockfile.txt

@@ -0,0 +1,14 @@
+#
+# This file is autogenerated by pip-compile with python 3.6
+# To update, run:
+#
+#    pip-compile --allow-unsafe --generate-hashes --output-file=src/python/pants/backend/awslambda/python/lambdex_lockfile.txt requirements_lambdex.in
+#
+lambdex==0.1.4 \
+    --hash=sha256:189e3f7aa492fd3653a6dcafb7bb493d2fa60a32f6ab8122c2ee53feb97967cc \
+    --hash=sha256:51e4e92a20cf1472e560b7b5a927286308d9e6252261a79d94b8549a3ba39bd3
+    # via -r requirements_lambdex.in
+pex==2.1.43 \
+    --hash=sha256:a2ac38d27730e00ffc9c51e8775cef2e3de9f7337eeaa4d84d0a2efdeb08e712 \
+    --hash=sha256:edc55cfae83d6cab51dde79a1b86f04d4227b5ec85dbed789090451ad9a798f2
+    # via lambdex

src/python/pants/backend/awslambda/python/register.py

@@ -6,13 +6,14 @@
 See https://www.pantsbuild.org/docs/awslambda-python.
 """
 
+from pants.backend.awslambda.python import lambdex
 from pants.backend.awslambda.python import rules as python_rules
 from pants.backend.awslambda.python.target_types import PythonAWSLambda
 from pants.backend.awslambda.python.target_types import rules as target_types_rules
 
 
 def rules():
-    return (*python_rules.rules(), *target_types_rules())
+    return (*python_rules.rules(), *target_types_rules(), *lambdex.rules())
 
 
 def target_types():

src/python/pants/backend/codegen/protobuf/python/mypy_protobuf_lockfile.txt

@@ -0,0 +1,43 @@
+#
+# This file is autogenerated by pip-compile with python 3.6
+# To update, run:
+#
+#    pip-compile --allow-unsafe --generate-hashes --output-file=src/python/pants/backend/codegen/protobuf/python/mypy_protobuf_lockfile.txt requirements_python-protobuf.mypy-plugin.in
+#
+mypy-protobuf==2.4 \
+    --hash=sha256:1986dc8a7994a4e549be5c8ac586aa8247caf92456c67159d71dbe5005809651 \
+    --hash=sha256:77e10c476cdd3ee14535c2357e64deac6b1a69f33eb500d795b064acda48c66f
+    # via -r requirements_python-protobuf.mypy-plugin.in
+protobuf==3.17.3 \
+    --hash=sha256:13ee7be3c2d9a5d2b42a1030976f760f28755fcf5863c55b1460fd205e6cd637 \
+    --hash=sha256:145ce0af55c4259ca74993ddab3479c78af064002ec8227beb3d944405123c71 \
+    --hash=sha256:14c1c9377a7ffbeaccd4722ab0aa900091f52b516ad89c4b0c3bb0a4af903ba5 \
+    --hash=sha256:1556a1049ccec58c7855a78d27e5c6e70e95103b32de9142bae0576e9200a1b0 \
+    --hash=sha256:26010f693b675ff5a1d0e1bdb17689b8b716a18709113288fead438703d45539 \
+    --hash=sha256:2ae692bb6d1992afb6b74348e7bb648a75bb0d3565a3f5eea5bec8f62bd06d87 \
+    --hash=sha256:2bfb815216a9cd9faec52b16fd2bfa68437a44b67c56bee59bc3926522ecb04e \
+    --hash=sha256:4ffbd23640bb7403574f7aff8368e2aeb2ec9a5c6306580be48ac59a6bac8bde \
+    --hash=sha256:59e5cf6b737c3a376932fbfb869043415f7c16a0cf176ab30a5bbc419cd709c1 \
+    --hash=sha256:6902a1e4b7a319ec611a7345ff81b6b004b36b0d2196ce7a748b3493da3d226d \
+    --hash=sha256:6ce4d8bf0321e7b2d4395e253f8002a1a5ffbcfd7bcc0a6ba46712c07d47d0b4 \
+    --hash=sha256:6d847c59963c03fd7a0cd7c488cadfa10cda4fff34d8bc8cba92935a91b7a037 \
+    --hash=sha256:72804ea5eaa9c22a090d2803813e280fb273b62d5ae497aaf3553d141c4fdd7b \
+    --hash=sha256:7a4c97961e9e5b03a56f9a6c82742ed55375c4a25f2692b625d4087d02ed31b9 \
+    --hash=sha256:85d6303e4adade2827e43c2b54114d9a6ea547b671cb63fafd5011dc47d0e13d \
+    --hash=sha256:8727ee027157516e2c311f218ebf2260a18088ffb2d29473e82add217d196b1c \
+    --hash=sha256:99938f2a2d7ca6563c0ade0c5ca8982264c484fdecf418bd68e880a7ab5730b1 \
+    --hash=sha256:9b7a5c1022e0fa0dbde7fd03682d07d14624ad870ae52054849d8960f04bc764 \
+    --hash=sha256:a22b3a0dbac6544dacbafd4c5f6a29e389a50e3b193e2c70dae6bbf7930f651d \
+    --hash=sha256:a38bac25f51c93e4be4092c88b2568b9f407c27217d3dd23c7a57fa522a17554 \
+    --hash=sha256:a981222367fb4210a10a929ad5983ae93bd5a050a0824fc35d6371c07b78caf6 \
+    --hash=sha256:ab6bb0e270c6c58e7ff4345b3a803cc59dbee19ddf77a4719c5b635f1d547aa8 \
+    --hash=sha256:c56c050a947186ba51de4f94ab441d7f04fcd44c56df6e922369cc2e1a92d683 \
+    --hash=sha256:e76d9686e088fece2450dbc7ee905f9be904e427341d289acbe9ad00b78ebd47 \
+    --hash=sha256:ebcb546f10069b56dc2e3da35e003a02076aaa377caf8530fe9789570984a8d2 \
+    --hash=sha256:f0e59430ee953184a703a324b8ec52f571c6c4259d496a19d1cabcdc19dabc62 \
+    --hash=sha256:ffea251f5cd3c0b9b43c7a7a912777e0bc86263436a87c2555242a348817221b
+    # via mypy-protobuf
+six==1.16.0 \
+    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
+    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
+    # via protobuf

src/python/pants/backend/codegen/protobuf/python/python_protobuf_subsystem.py

@@ -4,14 +4,18 @@
 from typing import cast
 
 from pants.backend.codegen.protobuf.target_types import ProtobufDependencies
+from pants.backend.experimental.python.lockfile import (
+    PythonToolLockfileRequest,
+    PythonToolLockfileSentinel,
+)
 from pants.backend.python.subsystems.python_tool_base import PythonToolRequirementsBase
 from pants.engine.addresses import Addresses, UnparsedAddressInputs
 from pants.engine.rules import Get, collect_rules, rule
 from pants.engine.target import InjectDependenciesRequest, InjectedDependencies
 from pants.engine.unions import UnionRule
 from pants.option.custom_types import target_option
 from pants.option.subsystem import Subsystem
-from pants.util.docutil import doc_url
+from pants.util.docutil import doc_url, git_url
 
 
 class PythonProtobufSubsystem(Subsystem):
@@ -59,9 +63,30 @@ class PythonProtobufMypyPlugin(PythonToolRequirementsBase):
     )
 
     default_version = "mypy-protobuf==2.4"
+
     register_interpreter_constraints = True
     default_interpreter_constraints = ["CPython>=3.6"]
 
+    register_lockfile = True
+    default_lockfile_resource = (
+        "pants.backend.codegen.protobuf.python",
+        "mypy_protobuf_lockfile.txt",
+    )
+    default_lockfile_url = git_url(
+        "src/python/pants/backend/codegen/protobuf/python/mypy_protobuf_lockfile.txt"
+    )
+
+
+class MypyProtobufLockfileSentinel(PythonToolLockfileSentinel):
+    pass
+
+
+@rule
+def setup_bandit_lockfile(
+    _: MypyProtobufLockfileSentinel, mypy_protobuf: PythonProtobufMypyPlugin
+) -> PythonToolLockfileRequest:
+    return PythonToolLockfileRequest.from_tool(mypy_protobuf)
+
 
 class InjectPythonProtobufDependencies(InjectDependenciesRequest):
     inject_for = ProtobufDependencies
@@ -79,4 +104,5 @@ def rules():
     return [
         *collect_rules(),
         UnionRule(InjectDependenciesRequest, InjectPythonProtobufDependencies),
+        UnionRule(PythonToolLockfileSentinel, MypyProtobufLockfileSentinel),
     ]

src/python/pants/backend/experimental/python/lockfile.py

@@ -7,7 +7,10 @@
 from dataclasses import dataclass
 from typing import cast
 
-from pants.backend.python.subsystems.python_tool_base import PythonToolBase
+from pants.backend.python.subsystems.python_tool_base import (
+    PythonToolBase,
+    PythonToolRequirementsBase,
+)
 from pants.backend.python.target_types import ConsoleScript, PythonRequirementsField
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
 from pants.backend.python.util_rules.pex import PexRequest, PexRequirements, VenvPex, VenvPexProcess
@@ -188,6 +191,15 @@ class PythonToolLockfileRequest:
     requirements: tuple[str, ...]
     interpreter_constraints: InterpreterConstraints
 
+    @classmethod
+    def from_tool(cls, subsystem: PythonToolRequirementsBase) -> PythonToolLockfileRequest:
+        return cls(
+            tool_name=subsystem.options_scope,
+            lockfile_path=subsystem.lockfile,
+            requirements=subsystem.all_requirements,
+            interpreter_constraints=subsystem.interpreter_constraints,
+        )
+
 
 # TODO(#12314): Unify this goal with `lock` once we figure out how to unify the semantics,
 #  particularly w/ CLI specs. This is a separate goal only to facilitate progress.

src/python/pants/backend/python/goals/coverage_py.py

@@ -12,6 +12,10 @@
 
 import toml
 
+from pants.backend.experimental.python.lockfile import (
+    PythonToolLockfileRequest,
+    PythonToolLockfileSentinel,
+)
 from pants.backend.python.subsystems.python_tool_base import PythonToolBase
 from pants.backend.python.target_types import ConsoleScript
 from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess
@@ -46,6 +50,7 @@
 from pants.engine.unions import UnionRule
 from pants.option.custom_types import file_option
 from pants.source.source_root import AllSourceRoots
+from pants.util.docutil import git_url
 from pants.util.logging import LogLevel
 
 """
@@ -99,9 +104,14 @@ class CoverageSubsystem(PythonToolBase):
 
     default_version = "coverage[toml]>=5.0.3,<5.1"
     default_main = ConsoleScript("coverage")
+
     register_interpreter_constraints = True
     default_interpreter_constraints = ["CPython>=3.6"]
 
+    register_lockfile = True
+    default_lockfile_resource = ("pants.backend.python.subsystems", "coverage_py_lockfile.txt")
+    default_lockfile_url = git_url("src/python/pants/backend/python/goals/coverage_py.py")
+
     @classmethod
     def register_options(cls, register):
         super().register_options(register)
@@ -207,6 +217,17 @@ def global_report(self) -> bool:
         return cast(bool, self.options.global_report)
 
 
+class CoveragePyLockfileSentinel(PythonToolLockfileSentinel):
+    pass
+
+
+@rule
+def setup_coverage_lockfile(
+    _: CoveragePyLockfileSentinel, coverage: CoverageSubsystem
+) -> PythonToolLockfileRequest:
+    return PythonToolLockfileRequest.from_tool(coverage)
+
+
 @dataclass(frozen=True)
 class PytestCoverageData(CoverageData):
     address: Address
@@ -532,4 +553,8 @@ def _get_coverage_report(
 
 
 def rules():
-    return [*collect_rules(), UnionRule(CoverageDataCollection, PytestCoverageDataCollection)]
+    return [
+        *collect_rules(),
+        UnionRule(CoverageDataCollection, PytestCoverageDataCollection),
+        UnionRule(PythonToolLockfileSentinel, CoveragePyLockfileSentinel),
+    ]