From e8654186c11c87e372bd62ab4f82f39ad2f5b208 Mon Sep 17 00:00:00 2001 From: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com> Date: Tue, 12 Oct 2021 22:56:31 -0700 Subject: [PATCH] `python_sources` and `python_tests` targets no longer use dependency inference, only `python_source` and `python_test` targets (#13231) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now our Python code operates on `python_source` and `python_test` targets. This causes some changes. ### Dependency inference Dependency inference no longer runs on `python_sources` and `python_tests`, only `python_source` and `python_test`. That applies to import analysis, conftest.py, and `__init__.py`. This actually speeds up dependency inference! We no longer run `import_parser.py` twice on the same files. Before: ``` ❯ hyperfine -r 10 './pants --no-process-execution-local-cache --no-pantsd dependencies src/python::' Benchmark #1: ./pants --no-process-execution-local-cache --no-pantsd dependencies src/python:: Time (mean ± σ): 7.650 s ± 0.741 s [User: 5.794 s, System: 2.141 s] Range (min … max): 7.250 s … 9.673 s 10 runs ``` After: ``` ❯ hyperfine -r 10 './pants --no-process-execution-local-cache --no-pantsd dependencies src/python::' Benchmark #1: ./pants --no-process-execution-local-cache --no-pantsd dependencies src/python:: Time (mean ± σ): 5.998 s ± 0.279 s [User: 3.524 s, System: 0.903 s] Range (min … max): 5.708 s … 6.688 s 10 runs ``` It means that `./pants dependencies src/py:lib` now only has explicitly declared dependencies. You have to use `--transitive` to get the results of dep inference. Thanks to this change, we can simplify `import_parser.py` to assume there is only one file. ### Lockfile generation When determining the interpreter constraints to use, we now look at generated targets, not target generators. We only run over the `python_source` and `python_test` targets that actually will be used. This makes us future-proof to an `overrides` field adding `skip_tool` to only some of the files. I think it also fixes our dependency resolution for Pylint looking at direct deps? ### `FieldSet.opt_out` no longer worries about file targets Pytest and MyPy used to skip running on non-file targets. That can be removed now. This unblocks allowing you to explicitly define a `python_source`/`python_test` target. --- .../backend/awslambda/python/rules_test.py | 4 +- .../awslambda/python/target_types_test.py | 4 +- .../backend/codegen/protobuf/python/BUILD | 2 +- .../backend/codegen/protobuf/python/rules.py | 4 +- .../python/rules_test.py | 4 +- .../python/target_types_test.py | 8 +- .../backend/project_info/count_loc_test.py | 4 +- .../backend/project_info/dependencies_test.py | 5 +- .../dependency_inference/import_parser.py | 36 +++---- .../import_parser_test.py | 32 +++---- .../dependency_inference/module_mapper.py | 45 +++++---- .../module_mapper_test.py | 8 +- .../python/dependency_inference/rules.py | 10 +- .../python/dependency_inference/rules_test.py | 53 ++++++----- src/python/pants/backend/python/goals/BUILD | 4 +- .../backend/python/goals/publish_test.py | 4 +- .../goals/pytest_runner_integration_test.py | 12 ++- .../python/goals/repl_integration_test.py | 4 +- .../pants/backend/python/goals/setup_py.py | 11 ++- .../backend/python/goals/setup_py_test.py | 5 +- .../pants/backend/python/goals/tailor.py | 16 ++-- .../pants/backend/python/goals/tailor_test.py | 33 ++++--- .../backend/python/lint/autoflake/rules.py | 6 +- .../lint/autoflake/rules_integration_test.py | 4 +- .../python/lint/autoflake/skip_field.py | 13 ++- .../lint/bandit/rules_integration_test.py | 4 +- .../backend/python/lint/bandit/skip_field.py | 13 ++- .../backend/python/lint/bandit/subsystem.py | 12 +-- .../python/lint/bandit/subsystem_test.py | 8 +- .../pants/backend/python/lint/black/rules.py | 6 +- .../lint/black/rules_integration_test.py | 4 +- .../backend/python/lint/black/skip_field.py | 13 ++- .../python/lint/black/subsystem_test.py | 4 +- .../backend/python/lint/docformatter/rules.py | 6 +- .../docformatter/rules_integration_test.py | 4 +- .../python/lint/docformatter/skip_field.py | 13 ++- .../lint/flake8/rules_integration_test.py | 4 +- .../backend/python/lint/flake8/skip_field.py | 13 ++- .../backend/python/lint/flake8/subsystem.py | 12 +-- .../python/lint/flake8/subsystem_test.py | 8 +- .../pants/backend/python/lint/isort/rules.py | 6 +- .../lint/isort/rules_integration_test.py | 4 +- .../backend/python/lint/isort/skip_field.py | 13 ++- .../lint/pylint/rules_integration_test.py | 4 +- .../backend/python/lint/pylint/skip_field.py | 13 ++- .../backend/python/lint/pylint/subsystem.py | 15 ++- .../python/lint/pylint/subsystem_test.py | 63 +++++++------ .../pants/backend/python/lint/python_fmt.py | 6 +- .../lint/python_fmt_integration_test.py | 6 +- .../backend/python/lint/pyupgrade/rules.py | 6 +- .../lint/pyupgrade/rules_integration_test.py | 4 +- .../python/lint/pyupgrade/skip_field.py | 13 ++- .../pants/backend/python/lint/yapf/rules.py | 6 +- .../lint/yapf/rules_integration_test.py | 4 +- .../backend/python/lint/yapf/skip_field.py | 13 ++- .../py_constraints_test.py | 7 +- src/python/pants/backend/python/register.py | 8 +- .../backend/python/subsystems/ipython_test.py | 4 +- .../pants/backend/python/subsystems/pytest.py | 23 ++--- .../backend/python/subsystems/pytest_test.py | 31 ++++--- .../python/subsystems/setuptools_test.py | 4 +- .../pants/backend/python/target_types.py | 93 +++++++++++++------ .../backend/python/target_types_rules.py | 34 ++++--- .../pants/backend/python/target_types_test.py | 11 ++- .../backend/python/typecheck/mypy/rules.py | 9 +- .../typecheck/mypy/rules_integration_test.py | 10 +- .../python/typecheck/mypy/skip_field.py | 13 ++- .../python/typecheck/mypy/subsystem.py | 18 ++-- .../python/typecheck/mypy/subsystem_test.py | 36 ++++--- .../python/util_rules/local_dists_test.py | 4 +- .../util_rules/pex_from_targets_test.py | 4 +- .../python/util_rules/python_sources.py | 8 +- .../python/util_rules/python_sources_test.py | 38 +++----- .../shunit2_test_runner_integration_test.py | 4 +- src/python/pants/core/goals/publish_test.py | 4 +- src/python/pants/core/goals/test_test.py | 4 +- .../pants/engine/internals/engine_test.py | 4 +- src/python/pants/engine/target.py | 5 +- .../pants/vcs/changed_integration_test.py | 1 - 79 files changed, 581 insertions(+), 427 deletions(-) diff --git a/src/python/pants/backend/awslambda/python/rules_test.py b/src/python/pants/backend/awslambda/python/rules_test.py index f9c5eccdd4f..64522a5aa81 100644 --- a/src/python/pants/backend/awslambda/python/rules_test.py +++ b/src/python/pants/backend/awslambda/python/rules_test.py @@ -15,7 +15,7 @@ from pants.backend.awslambda.python.target_types import rules as target_rules from pants.backend.python.subsystems.lambdex import Lambdex from pants.backend.python.subsystems.lambdex import rules as awslambda_python_subsystem_rules -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.backend.python.target_types_rules import rules as python_target_types_rules from pants.core.goals.package import BuiltPackage from pants.core.target_types import FilesGeneratorTarget, RelocatedFiles, ResourcesGeneratorTarget @@ -39,7 +39,7 @@ def rule_runner() -> RuleRunner: ], target_types=[ PythonAWSLambda, - PythonLibrary, + PythonSourcesGeneratorTarget, FilesGeneratorTarget, RelocatedFiles, ResourcesGeneratorTarget, diff --git a/src/python/pants/backend/awslambda/python/target_types_test.py b/src/python/pants/backend/awslambda/python/target_types_test.py index c4b3cd456e4..edc83415b02 100644 --- a/src/python/pants/backend/awslambda/python/target_types_test.py +++ b/src/python/pants/backend/awslambda/python/target_types_test.py @@ -16,7 +16,7 @@ ResolvePythonAwsHandlerRequest, ) from pants.backend.awslambda.python.target_types import rules as target_type_rules -from pants.backend.python.target_types import PythonLibrary, PythonRequirementTarget +from pants.backend.python.target_types import PythonRequirementTarget, PythonSourcesGeneratorTarget from pants.backend.python.target_types_rules import rules as python_target_types_rules from pants.build_graph.address import Address from pants.engine.target import InjectedDependencies, InvalidFieldException @@ -32,7 +32,7 @@ def rule_runner() -> RuleRunner: QueryRule(ResolvedPythonAwsHandler, [ResolvePythonAwsHandlerRequest]), QueryRule(InjectedDependencies, [InjectPythonLambdaHandlerDependency]), ], - target_types=[PythonAWSLambda, PythonRequirementTarget, PythonLibrary], + target_types=[PythonAWSLambda, PythonRequirementTarget, PythonSourcesGeneratorTarget], ) diff --git a/src/python/pants/backend/codegen/protobuf/python/BUILD b/src/python/pants/backend/codegen/protobuf/python/BUILD index 5eece07cb01..f8c36962ef3 100644 --- a/src/python/pants/backend/codegen/protobuf/python/BUILD +++ b/src/python/pants/backend/codegen/protobuf/python/BUILD @@ -11,7 +11,7 @@ python_tests(name="python_protobuf_subsystem_test", sources=["python_protobuf_su python_tests( name="rules_integration_test", sources=["rules_integration_test.py"], - timeout=240, + timeout=330, # We want to make sure the default lockfile for MyPy Protobuf works for both macOS and Linux. tags=["platform_specific_behavior"], ) diff --git a/src/python/pants/backend/codegen/protobuf/python/rules.py b/src/python/pants/backend/codegen/protobuf/python/rules.py index 0e41ebae1e6..39410106bd1 100644 --- a/src/python/pants/backend/codegen/protobuf/python/rules.py +++ b/src/python/pants/backend/codegen/protobuf/python/rules.py @@ -11,7 +11,7 @@ PythonProtobufSubsystem, ) from pants.backend.codegen.protobuf.target_types import ProtobufGrpcToggleField, ProtobufSourceField -from pants.backend.python.target_types import PythonSources +from pants.backend.python.target_types import PythonSourceField from pants.backend.python.util_rules import pex from pants.backend.python.util_rules.pex import PexRequest, PexResolveInfo, VenvPex, VenvPexRequest from pants.backend.python.util_rules.pex_environment import PexEnvironment @@ -43,7 +43,7 @@ class GeneratePythonFromProtobufRequest(GenerateSourcesRequest): input = ProtobufSourceField - output = PythonSources + output = PythonSourceField @rule(desc="Generate Python from Protobuf", level=LogLevel.DEBUG) diff --git a/src/python/pants/backend/google_cloud_function/python/rules_test.py b/src/python/pants/backend/google_cloud_function/python/rules_test.py index 98378ca76da..7e5d3b1d73a 100644 --- a/src/python/pants/backend/google_cloud_function/python/rules_test.py +++ b/src/python/pants/backend/google_cloud_function/python/rules_test.py @@ -19,7 +19,7 @@ from pants.backend.python.subsystems.lambdex import ( rules as python_google_cloud_function_subsystem_rules, ) -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.backend.python.target_types_rules import rules as python_target_types_rules from pants.core.goals.package import BuiltPackage from pants.core.target_types import FilesGeneratorTarget, RelocatedFiles, ResourcesGeneratorTarget @@ -43,7 +43,7 @@ def rule_runner() -> RuleRunner: ], target_types=[ PythonGoogleCloudFunction, - PythonLibrary, + PythonSourcesGeneratorTarget, FilesGeneratorTarget, RelocatedFiles, ResourcesGeneratorTarget, diff --git a/src/python/pants/backend/google_cloud_function/python/target_types_test.py b/src/python/pants/backend/google_cloud_function/python/target_types_test.py index 777d7c9f4bc..b183c529d95 100644 --- a/src/python/pants/backend/google_cloud_function/python/target_types_test.py +++ b/src/python/pants/backend/google_cloud_function/python/target_types_test.py @@ -16,7 +16,7 @@ ResolvePythonGoogleHandlerRequest, ) from pants.backend.google_cloud_function.python.target_types import rules as target_type_rules -from pants.backend.python.target_types import PythonLibrary, PythonRequirementTarget +from pants.backend.python.target_types import PythonRequirementTarget, PythonSourcesGeneratorTarget from pants.backend.python.target_types_rules import rules as python_target_types_rules from pants.build_graph.address import Address from pants.engine.internals.scheduler import ExecutionError @@ -33,7 +33,11 @@ def rule_runner() -> RuleRunner: QueryRule(ResolvedPythonGoogleHandler, [ResolvePythonGoogleHandlerRequest]), QueryRule(InjectedDependencies, [InjectPythonCloudFunctionHandlerDependency]), ], - target_types=[PythonGoogleCloudFunction, PythonRequirementTarget, PythonLibrary], + target_types=[ + PythonGoogleCloudFunction, + PythonRequirementTarget, + PythonSourcesGeneratorTarget, + ], ) diff --git a/src/python/pants/backend/project_info/count_loc_test.py b/src/python/pants/backend/project_info/count_loc_test.py index a432f43d101..e5c2672c3fb 100644 --- a/src/python/pants/backend/project_info/count_loc_test.py +++ b/src/python/pants/backend/project_info/count_loc_test.py @@ -5,7 +5,7 @@ from pants.backend.project_info import count_loc from pants.backend.project_info.count_loc import CountLinesOfCode -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.core.util_rules import external_tool from pants.engine.target import MultipleSourcesField, Target from pants.testutil.rule_runner import GoalRuleResult, RuleRunner @@ -24,7 +24,7 @@ class ElixirTarget(Target): def rule_runner() -> RuleRunner: return RuleRunner( rules=[*count_loc.rules(), *external_tool.rules()], - target_types=[PythonLibrary, ElixirTarget], + target_types=[PythonSourcesGeneratorTarget, ElixirTarget], ) diff --git a/src/python/pants/backend/project_info/dependencies_test.py b/src/python/pants/backend/project_info/dependencies_test.py index b5edb98db57..3a98f37b247 100644 --- a/src/python/pants/backend/project_info/dependencies_test.py +++ b/src/python/pants/backend/project_info/dependencies_test.py @@ -8,7 +8,7 @@ import pytest from pants.backend.project_info.dependencies import Dependencies, DependencyType, rules -from pants.backend.python.target_types import PythonLibrary, PythonRequirementTarget +from pants.backend.python.target_types import PythonRequirementTarget, PythonSourcesGeneratorTarget from pants.engine.target import SpecialCasedDependencies, Target from pants.testutil.rule_runner import RuleRunner @@ -27,7 +27,8 @@ class SpecialDepsTarget(Target): @pytest.fixture def rule_runner() -> RuleRunner: return RuleRunner( - rules=rules(), target_types=[PythonLibrary, PythonRequirementTarget, SpecialDepsTarget] + rules=rules(), + target_types=[PythonSourcesGeneratorTarget, PythonRequirementTarget, SpecialDepsTarget], ) diff --git a/src/python/pants/backend/python/dependency_inference/import_parser.py b/src/python/pants/backend/python/dependency_inference/import_parser.py index efa2aaad77c..29561a0ec46 100644 --- a/src/python/pants/backend/python/dependency_inference/import_parser.py +++ b/src/python/pants/backend/python/dependency_inference/import_parser.py @@ -3,6 +3,7 @@ from dataclasses import dataclass +from pants.backend.python.target_types import PythonSourceField from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.backend.python.util_rules.pex_environment import PythonExecutable from pants.core.util_rules.source_files import SourceFilesRequest @@ -11,7 +12,6 @@ from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests from pants.engine.process import Process, ProcessResult from pants.engine.rules import Get, MultiGet, collect_rules, rule -from pants.engine.target import SourcesField from pants.util.logging import LogLevel # NOTE: Must call .format(min_dots=X) on this string to use it. @@ -113,24 +113,23 @@ def parse_file(filename): return None -if __name__ == "__main__": - imports = set() - - for filename in sys.argv[1:]: - tree = parse_file(filename) - if not tree: - continue +def main(filename): + tree = parse_file(filename) + if not tree: + return - package_parts = os.path.dirname(filename).split(os.path.sep) - visitor = AstVisitor(package_parts) - - visitor.visit(tree) - imports.update(visitor.imports) + package_parts = os.path.dirname(filename).split(os.path.sep) + visitor = AstVisitor(package_parts) + visitor.visit(tree) # We have to be careful to set the encoding explicitly and write raw bytes ourselves. # See below for where we explicitly decode. buffer = sys.stdout if sys.version_info[0:2] == (2, 7) else sys.stdout.buffer - buffer.write("\\n".join(sorted(imports)).encode("utf8")) + buffer.write("\\n".join(sorted(visitor.imports)).encode("utf8")) + + +if __name__ == "__main__": + main(sys.argv[1]) """ @@ -143,7 +142,7 @@ class ParsedPythonImports(DeduplicatedCollection[str]): @dataclass(frozen=True) class ParsePythonImportsRequest: - sources: SourcesField + sources: PythonSourceField interpreter_constraints: InterpreterConstraints string_imports: bool string_imports_min_dots: int @@ -157,6 +156,11 @@ async def parse_python_imports(request: ParsePythonImportsRequest) -> ParsedPyth Get(Digest, CreateDigest([FileContent("__parse_python_imports.py", script)])), Get(StrippedSourceFiles, SourceFilesRequest([request.sources])), ) + + # We operate on PythonSourceField, which should be one file. + assert len(stripped_sources.snapshot.files) == 1 + file = stripped_sources.snapshot.files[0] + input_digest = await Get( Digest, MergeDigests([script_digest, stripped_sources.snapshot.digest]) ) @@ -166,7 +170,7 @@ async def parse_python_imports(request: ParsePythonImportsRequest) -> ParsedPyth argv=[ python_interpreter.path, "./__parse_python_imports.py", - *stripped_sources.snapshot.files, + file, ], input_digest=input_digest, description=f"Determine Python imports for {request.sources.address}", diff --git a/src/python/pants/backend/python/dependency_inference/import_parser_test.py b/src/python/pants/backend/python/dependency_inference/import_parser_test.py index 1dda99b67d7..3fddcae1bf2 100644 --- a/src/python/pants/backend/python/dependency_inference/import_parser_test.py +++ b/src/python/pants/backend/python/dependency_inference/import_parser_test.py @@ -12,7 +12,7 @@ ParsedPythonImports, ParsePythonImportsRequest, ) -from pants.backend.python.target_types import PythonLibrary, PythonSources +from pants.backend.python.target_types import PythonSourceField, PythonSourceTarget from pants.backend.python.util_rules import pex from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.core.util_rules import stripped_source_files @@ -34,13 +34,13 @@ def rule_runner() -> RuleRunner: *pex.rules(), QueryRule(ParsedPythonImports, [ParsePythonImportsRequest]), ], - target_types=[PythonLibrary], + target_types=[PythonSourceTarget], ) def assert_imports_parsed( rule_runner: RuleRunner, - content: str | None, + content: str, *, expected: list[str], filename: str = "project/foo.py", @@ -49,16 +49,18 @@ def assert_imports_parsed( string_imports_min_dots: int = 2, ) -> None: rule_runner.set_options([], env_inherit={"PATH", "PYENV_ROOT", "HOME"}) - files = {"project/BUILD": "python_sources(sources=['**/*.py'])"} - if content is not None: - files[filename] = content - rule_runner.write_files(files) # type: ignore[arg-type] - tgt = rule_runner.get_target(Address("project")) + rule_runner.write_files( + { + "BUILD": f"python_sources(name='t', source={repr(filename)})", + filename: content, + } + ) + tgt = rule_runner.get_target(Address("", target_name="t")) imports = rule_runner.request( ParsedPythonImports, [ ParsePythonImportsRequest( - tgt[PythonSources], + tgt[PythonSourceField], InterpreterConstraints([constraints]), string_imports=string_imports, string_imports_min_dots=string_imports_min_dots, @@ -94,9 +96,6 @@ def test_normal_imports(rule_runner: RuleRunner) -> None: __import__("pkg_resources") """ ) - # We create a second file, in addition to what `assert_imports_parsed` does, to ensure we can - # handle multiple files belonging to the same target. - rule_runner.write_files({"project/f2.py": "import second_import"}) assert_imports_parsed( rule_runner, content, @@ -110,7 +109,6 @@ def test_normal_imports(rule_runner: RuleRunner) -> None: "project.demo.Demo", "project.demo.OriginalName", "project.circular_dep.CircularDep", - "second_import", "subprocess", "subprocess23", "pkg_resources", @@ -195,15 +193,11 @@ def test_imports_from_strings(rule_runner: RuleRunner, min_dots: int) -> None: def test_gracefully_handle_syntax_errors(rule_runner: RuleRunner) -> None: - assert_imports_parsed(rule_runner, content="x =", expected=[]) + assert_imports_parsed(rule_runner, "x =", expected=[]) def test_handle_unicode(rule_runner: RuleRunner) -> None: - assert_imports_parsed(rule_runner, content="x = 'äbç'", expected=[]) - - -def test_gracefully_handle_no_sources(rule_runner: RuleRunner) -> None: - assert_imports_parsed(rule_runner, content=None, expected=[]) + assert_imports_parsed(rule_runner, "x = 'äbç'", expected=[]) @skip_unless_python27_present diff --git a/src/python/pants/backend/python/dependency_inference/module_mapper.py b/src/python/pants/backend/python/dependency_inference/module_mapper.py index d711f4ca643..c3587a0b709 100644 --- a/src/python/pants/backend/python/dependency_inference/module_mapper.py +++ b/src/python/pants/backend/python/dependency_inference/module_mapper.py @@ -20,7 +20,7 @@ PythonRequirementModulesField, PythonRequirementsField, PythonRequirementTypeStubModulesField, - PythonSources, + PythonSourceField, TypeStubsModuleMappingField, ) from pants.base.specs import AddressSpecs, DescendantAddresses @@ -174,32 +174,37 @@ async def map_first_party_python_targets_to_modules( _: FirstPartyPythonTargetsMappingMarker, ) -> FirstPartyPythonMappingImpl: all_expanded_targets = await Get(Targets, AddressSpecs([DescendantAddresses("")])) - python_targets = tuple(tgt for tgt in all_expanded_targets if tgt.has_field(PythonSources)) + python_targets = tuple(tgt for tgt in all_expanded_targets if tgt.has_field(PythonSourceField)) stripped_sources_per_target = await MultiGet( - Get(StrippedSourceFileNames, SourcesPathsRequest(tgt[PythonSources])) + Get(StrippedSourceFileNames, SourcesPathsRequest(tgt[PythonSourceField])) for tgt in python_targets ) modules_to_addresses: DefaultDict[str, list[Address]] = defaultdict(list) modules_with_multiple_implementations: DefaultDict[str, set[Address]] = defaultdict(set) for tgt, stripped_sources in zip(python_targets, stripped_sources_per_target): - for stripped_f in stripped_sources: - module = PythonModule.create_from_stripped_path(PurePath(stripped_f)).module - if module in modules_to_addresses: - # We check if one of the targets is an implementation (.py file) and the other is - # a type stub (.pyi file), which we allow. Otherwise, we have ambiguity. - prior_is_type_stub = len( - modules_to_addresses[module] - ) == 1 and modules_to_addresses[module][0].filename.endswith(".pyi") - current_is_type_stub = tgt.address.filename.endswith(".pyi") - if prior_is_type_stub ^ current_is_type_stub: - modules_to_addresses[module].append(tgt.address) - else: - modules_with_multiple_implementations[module].update( - {*modules_to_addresses[module], tgt.address} - ) - else: - modules_to_addresses[module].append(tgt.address) + # `PythonSourceFile` validates that each target has exactly one file. + assert len(stripped_sources) == 1 + stripped_f = stripped_sources[0] + + module = PythonModule.create_from_stripped_path(PurePath(stripped_f)).module + if module not in modules_to_addresses: + modules_to_addresses[module].append(tgt.address) + continue + + # Else, possible ambiguity. Check if one of the targets is an implementation + # (.py file) and the other is a type stub (.pyi file), which we allow. Otherwise, it's + # ambiguous. + prior_is_type_stub = len(modules_to_addresses[module]) == 1 and modules_to_addresses[ + module + ][0].filename.endswith(".pyi") + current_is_type_stub = tgt.address.filename.endswith(".pyi") + if prior_is_type_stub ^ current_is_type_stub: + modules_to_addresses[module].append(tgt.address) + else: + modules_with_multiple_implementations[module].update( + {*modules_to_addresses[module], tgt.address} + ) # Remove modules with ambiguous owners. for module in modules_with_multiple_implementations: diff --git a/src/python/pants/backend/python/dependency_inference/module_mapper_test.py b/src/python/pants/backend/python/dependency_inference/module_mapper_test.py index 357c2cd267b..44823b12138 100644 --- a/src/python/pants/backend/python/dependency_inference/module_mapper_test.py +++ b/src/python/pants/backend/python/dependency_inference/module_mapper_test.py @@ -24,7 +24,7 @@ ThirdPartyPythonModuleMapping, ) from pants.backend.python.dependency_inference.module_mapper import rules as module_mapper_rules -from pants.backend.python.target_types import PythonLibrary, PythonRequirementTarget +from pants.backend.python.target_types import PythonRequirementTarget, PythonSourcesGeneratorTarget from pants.core.util_rules import stripped_source_files from pants.engine.addresses import Address from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -174,7 +174,11 @@ def rule_runner() -> RuleRunner: QueryRule(ThirdPartyPythonModuleMapping, []), QueryRule(PythonModuleOwners, [PythonModule]), ], - target_types=[PythonLibrary, PythonRequirementTarget, ProtobufSourcesGeneratorTarget], + target_types=[ + PythonSourcesGeneratorTarget, + PythonRequirementTarget, + ProtobufSourcesGeneratorTarget, + ], ) diff --git a/src/python/pants/backend/python/dependency_inference/rules.py b/src/python/pants/backend/python/dependency_inference/rules.py index d4abb8fed3e..b15794eb1e7 100644 --- a/src/python/pants/backend/python/dependency_inference/rules.py +++ b/src/python/pants/backend/python/dependency_inference/rules.py @@ -11,7 +11,7 @@ ParsePythonImportsRequest, ) from pants.backend.python.dependency_inference.module_mapper import PythonModule, PythonModuleOwners -from pants.backend.python.target_types import PythonSources, PythonTestsSources +from pants.backend.python.target_types import PythonSourceField, PythonTestSourceField from pants.backend.python.util_rules import ancestor_files, pex from pants.backend.python.util_rules.ancestor_files import AncestorFiles, AncestorFilesRequest from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints @@ -133,7 +133,7 @@ def entry_points(self) -> bool: class InferPythonImportDependencies(InferDependenciesRequest): - infer_from = PythonSources + infer_from = PythonSourceField @rule(desc="Inferring Python dependencies by analyzing imports") @@ -151,7 +151,7 @@ async def infer_python_dependencies_via_imports( Get( ParsedPythonImports, ParsePythonImportsRequest( - request.sources_field, + cast(PythonSourceField, request.sources_field), InterpreterConstraints.create_from_targets([wrapped_tgt.target], python_setup), string_imports=python_infer_subsystem.string_imports, string_imports_min_dots=python_infer_subsystem.string_imports_min_dots, @@ -181,7 +181,7 @@ async def infer_python_dependencies_via_imports( class InferInitDependencies(InferDependenciesRequest): - infer_from = PythonSources + infer_from = PythonSourceField @rule(desc="Inferring dependencies on `__init__.py` files") @@ -209,7 +209,7 @@ async def infer_python_init_dependencies( class InferConftestDependencies(InferDependenciesRequest): - infer_from = PythonTestsSources + infer_from = PythonTestSourceField @rule(desc="Inferring dependencies on `conftest.py` files") diff --git a/src/python/pants/backend/python/dependency_inference/rules_test.py b/src/python/pants/backend/python/dependency_inference/rules_test.py index 42dae657163..464fef6966f 100644 --- a/src/python/pants/backend/python/dependency_inference/rules_test.py +++ b/src/python/pants/backend/python/dependency_inference/rules_test.py @@ -14,10 +14,10 @@ infer_python_init_dependencies, ) from pants.backend.python.target_types import ( - PythonLibrary, PythonRequirementTarget, - PythonSources, - PythonTests, + PythonSourceField, + PythonSourcesGeneratorTarget, + PythonTestsGeneratorTarget, ) from pants.backend.python.util_rules import ancestor_files from pants.engine.addresses import Address @@ -33,7 +33,7 @@ def test_infer_python_imports(caplog) -> None: *target_types_rules.rules(), QueryRule(InferredDependencies, [InferPythonImportDependencies]), ], - target_types=[PythonLibrary, PythonRequirementTarget], + target_types=[PythonSourcesGeneratorTarget, PythonRequirementTarget], ) rule_runner.add_to_build_file( "3rdparty/python", @@ -91,23 +91,23 @@ def run_dep_inference( rule_runner.set_options(args, env_inherit={"PATH", "PYENV_ROOT", "HOME"}) target = rule_runner.get_target(address) return rule_runner.request( - InferredDependencies, [InferPythonImportDependencies(target[PythonSources])] + InferredDependencies, [InferPythonImportDependencies(target[PythonSourceField])] ) - build_address = Address("src/python") - assert run_dep_inference(build_address) == InferredDependencies( + assert run_dep_inference( + Address("src/python", relative_file_path="app.py") + ) == InferredDependencies( [ Address("3rdparty/python", target_name="Django"), - Address("src/python", relative_file_path="app.py"), Address("src/python/util", relative_file_path="dep.py"), ], ) - file_address = Address("src/python", relative_file_path="f2.py") - assert run_dep_inference(file_address) == InferredDependencies( + addr = Address("src/python", relative_file_path="f2.py") + assert run_dep_inference(addr) == InferredDependencies( [Address("src/python", relative_file_path="app.py")] ) - assert run_dep_inference(file_address, enable_string_imports=True) == InferredDependencies( + assert run_dep_inference(addr, enable_string_imports=True) == InferredDependencies( [ Address("src/python", relative_file_path="app.py"), Address("src/python/str_import/subdir", relative_file_path="f.py"), @@ -138,7 +138,7 @@ def run_dep_inference( ), ) assert run_dep_inference( - Address("src/python/ambiguous", target_name="main") + Address("src/python/ambiguous", target_name="main", relative_file_path="main.py") ) == InferredDependencies( [ Address( @@ -149,7 +149,7 @@ def run_dep_inference( ], ) assert len(caplog.records) == 1 - assert "The target src/python/ambiguous:main imports `ambiguous.dep`" in caplog.text + assert "The target src/python/ambiguous/main.py:main imports `ambiguous.dep`" in caplog.text assert "['src/python/ambiguous/dep.py:dep1', 'src/python/ambiguous/dep.py:dep2']" in caplog.text assert "disambiguated_via_ignores.py" not in caplog.text @@ -163,7 +163,7 @@ def test_infer_python_inits() -> None: SubsystemRule(PythonInferSubsystem), QueryRule(InferredDependencies, (InferInitDependencies,)), ], - target_types=[PythonLibrary], + target_types=[PythonSourcesGeneratorTarget], ) rule_runner.set_options( [ @@ -181,19 +181,23 @@ def test_infer_python_inits() -> None: rule_runner.add_to_build_file("src/python/root/mid", "python_sources()") rule_runner.create_file("src/python/root/mid/leaf/__init__.py") + rule_runner.create_file("src/python/root/mid/leaf/f.py") rule_runner.add_to_build_file("src/python/root/mid/leaf", "python_sources()") def run_dep_inference(address: Address) -> InferredDependencies: target = rule_runner.get_target(address) return rule_runner.request( InferredDependencies, - [InferInitDependencies(target[PythonSources])], + [InferInitDependencies(target[PythonSourceField])], ) - assert run_dep_inference(Address("src/python/root/mid/leaf")) == InferredDependencies( + assert run_dep_inference( + Address("src/python/root/mid/leaf", relative_file_path="f.py") + ) == InferredDependencies( [ - Address("src/python/root", relative_file_path="__init__.py", target_name="root"), - Address("src/python/root/mid", relative_file_path="__init__.py", target_name="mid"), + Address("src/python/root", relative_file_path="__init__.py"), + Address("src/python/root/mid", relative_file_path="__init__.py"), + Address("src/python/root/mid/leaf", relative_file_path="__init__.py"), ], ) @@ -207,7 +211,7 @@ def test_infer_python_conftests() -> None: SubsystemRule(PythonInferSubsystem), QueryRule(InferredDependencies, (InferConftestDependencies,)), ], - target_types=[PythonTests], + target_types=[PythonTestsGeneratorTarget], ) rule_runner.set_options( ["--backend-packages=pants.backend.python", "--source-root-patterns=src/python"], @@ -228,12 +232,15 @@ def run_dep_inference(address: Address) -> InferredDependencies: target = rule_runner.get_target(address) return rule_runner.request( InferredDependencies, - [InferConftestDependencies(target[PythonSources])], + [InferConftestDependencies(target[PythonSourceField])], ) - assert run_dep_inference(Address("src/python/root/mid/leaf")) == InferredDependencies( + assert run_dep_inference( + Address("src/python/root/mid/leaf", relative_file_path="this_is_a_test.py") + ) == InferredDependencies( [ - Address("src/python/root", relative_file_path="conftest.py", target_name="root"), - Address("src/python/root/mid", relative_file_path="conftest.py", target_name="mid"), + Address("src/python/root", relative_file_path="conftest.py"), + Address("src/python/root/mid", relative_file_path="conftest.py"), + Address("src/python/root/mid/leaf", relative_file_path="conftest.py"), ], ) diff --git a/src/python/pants/backend/python/goals/BUILD b/src/python/pants/backend/python/goals/BUILD index fb747f9862e..f1884fa409e 100644 --- a/src/python/pants/backend/python/goals/BUILD +++ b/src/python/pants/backend/python/goals/BUILD @@ -34,13 +34,13 @@ python_tests( sources=["repl_integration_test.py"], # We want to make sure the default lockfile works for both macOS and Linux. tags=["platform_specific_behavior"], - timeout=240, + timeout=300, ) python_tests( name="run_pex_binary_integration", sources=["run_pex_binary_integration_test.py"], - timeout=120, + timeout=180, ) python_tests( diff --git a/src/python/pants/backend/python/goals/publish_test.py b/src/python/pants/backend/python/goals/publish_test.py index ba51b713d39..e10cc7cd70e 100644 --- a/src/python/pants/backend/python/goals/publish_test.py +++ b/src/python/pants/backend/python/goals/publish_test.py @@ -9,7 +9,7 @@ from pants.backend.python.goals.publish import PublishToPyPiFieldSet, PublishToPyPiRequest, rules from pants.backend.python.macros.python_artifact import PythonArtifact -from pants.backend.python.target_types import PythonDistribution, PythonLibrary +from pants.backend.python.target_types import PythonDistribution, PythonSourcesGeneratorTarget from pants.backend.python.util_rules import pex_from_targets from pants.core.goals.package import BuiltPackage, BuiltPackageArtifact from pants.core.goals.publish import PublishPackages, PublishProcesses @@ -29,7 +29,7 @@ def rule_runner() -> RuleRunner: *rules(), QueryRule(PublishProcesses, [PublishToPyPiRequest]), ], - target_types=[PythonLibrary, PythonDistribution], + target_types=[PythonSourcesGeneratorTarget, PythonDistribution], objects={"python_artifact": PythonArtifact}, ) rule_runner.set_options( diff --git a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py index fbf918522a1..942194afd1e 100644 --- a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py +++ b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py @@ -21,9 +21,9 @@ from pants.backend.python.target_types import ( PexBinary, PythonDistribution, - PythonLibrary, PythonRequirementTarget, - PythonTests, + PythonSourcesGeneratorTarget, + PythonTestsGeneratorTarget, ) from pants.backend.python.util_rules import local_dists, pex_from_targets from pants.core.goals.test import ( @@ -69,8 +69,8 @@ def rule_runner() -> RuleRunner: ], target_types=[ PexBinary, - PythonLibrary, - PythonTests, + PythonSourcesGeneratorTarget, + PythonTestsGeneratorTarget, PythonRequirementTarget, PythonDistribution, ], @@ -542,7 +542,9 @@ def test_bar(): ), } ) - tgt = rule_runner.get_target(Address(os.path.join(PACKAGE, "foo"), target_name="tests")) + tgt = rule_runner.get_target( + Address(os.path.join(PACKAGE, "foo"), target_name="tests", relative_file_path="bar_test.py") + ) result = run_pytest(rule_runner, tgt) assert result.exit_code == 0 diff --git a/src/python/pants/backend/python/goals/repl_integration_test.py b/src/python/pants/backend/python/goals/repl_integration_test.py index 33fb9a0fb71..e1a009c01ec 100644 --- a/src/python/pants/backend/python/goals/repl_integration_test.py +++ b/src/python/pants/backend/python/goals/repl_integration_test.py @@ -8,7 +8,7 @@ from pants.backend.codegen.protobuf.target_types import ProtobufSourceTarget from pants.backend.python.goals import repl as python_repl from pants.backend.python.subsystems.ipython import rules as ipython_subsystem_rules -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.backend.python.util_rules import local_dists, pex_from_targets from pants.backend.python.util_rules.pex import PexProcess from pants.core.goals.repl import Repl @@ -30,7 +30,7 @@ def rule_runner() -> RuleRunner: *local_dists.rules(), QueryRule(Process, (PexProcess,)), ], - target_types=[PythonLibrary, ProtobufSourceTarget], + target_types=[PythonSourcesGeneratorTarget, ProtobufSourceTarget], ) rule_runner.write_files( { diff --git a/src/python/pants/backend/python/goals/setup_py.py b/src/python/pants/backend/python/goals/setup_py.py index a3378a092bd..6e245fd032d 100644 --- a/src/python/pants/backend/python/goals/setup_py.py +++ b/src/python/pants/backend/python/goals/setup_py.py @@ -19,9 +19,10 @@ from pants.backend.python.target_types import ( GenerateSetupField, PythonDistributionEntryPointsField, + PythonGeneratingSourcesBase, PythonProvidesField, PythonRequirementsField, - PythonSources, + PythonSourceField, ResolvedPythonDistributionEntryPoints, ResolvePythonDistributionEntryPointsRequest, SDistConfigSettingsField, @@ -907,10 +908,14 @@ def is_ownable_target(tgt: Target, union_membership: UnionMembership) -> bool: # This isn't particularly useful (3rdparty requirements should be on the python_sources # that consumes them)... but users may expect it to work anyway. tgt.has_field(PythonProvidesField) - or tgt.has_field(PythonSources) + or tgt.has_field(PythonSourceField) or tgt.has_field(ResourceSourceField) - or tgt.get(SourcesField).can_generate(PythonSources, union_membership) + or tgt.get(SourcesField).can_generate(PythonSourceField, union_membership) or tgt.get(SourcesField).can_generate(ResourceSourceField, union_membership) + # We also check for generating sources so that dependencies on `python_sources(sources=[])` + # is included. Those won't generate any `python_source` targets, but still can be + # dependended upon. + or tgt.has_field(PythonGeneratingSourcesBase) ) diff --git a/src/python/pants/backend/python/goals/setup_py_test.py b/src/python/pants/backend/python/goals/setup_py_test.py index 8a6d9bfbd53..7e9752d704e 100644 --- a/src/python/pants/backend/python/goals/setup_py_test.py +++ b/src/python/pants/backend/python/goals/setup_py_test.py @@ -44,8 +44,8 @@ from pants.backend.python.target_types import ( PexBinary, PythonDistribution, - PythonLibrary, PythonRequirementTarget, + PythonSourcesGeneratorTarget, ) from pants.backend.python.util_rules import python_sources from pants.core.target_types import FileTarget, ResourceTarget @@ -65,7 +65,7 @@ def create_setup_py_rule_runner(*, rules: Iterable) -> RuleRunner: target_types=[ PexBinary, PythonDistribution, - PythonLibrary, + PythonSourcesGeneratorTarget, PythonRequirementTarget, ResourceTarget, FileTarget, @@ -556,6 +556,7 @@ def test_get_sources() -> None: get_sources, get_owned_dependencies, get_exporting_owner, + *target_types_rules.rules(), *python_sources.rules(), QueryRule(OwnedDependencies, (DependencyOwner,)), QueryRule(SetupPySources, (SetupPyChrootRequest,)), diff --git a/src/python/pants/backend/python/goals/tailor.py b/src/python/pants/backend/python/goals/tailor.py index fe50a61de1b..5cf0f878cbd 100644 --- a/src/python/pants/backend/python/goals/tailor.py +++ b/src/python/pants/backend/python/goals/tailor.py @@ -13,9 +13,9 @@ from pants.backend.python.target_types import ( PexBinary, PexEntryPointField, - PythonLibrary, - PythonTests, - PythonTestsSources, + PythonSourcesGeneratorTarget, + PythonTestsGeneratingSourcesField, + PythonTestsGeneratorTarget, ResolvedPexEntryPoint, ResolvePexEntryPointRequest, ) @@ -45,13 +45,13 @@ class PutativePythonTargetsRequest(PutativeTargetsRequest): def classify_source_files(paths: Iterable[str]) -> dict[type[Target], set[str]]: """Returns a dict of target type -> files that belong to targets of that type.""" - tests_filespec = Filespec(includes=list(PythonTestsSources.default)) + tests_filespec = Filespec(includes=list(PythonTestsGeneratingSourcesField.default)) test_filenames = set( matches_filespec(tests_filespec, paths=[os.path.basename(path) for path in paths]) ) test_files = {path for path in paths if os.path.basename(path) in test_filenames} library_files = set(paths) - test_files - return {PythonTests: test_files, PythonLibrary: library_files} + return {PythonTestsGeneratorTarget: test_files, PythonSourcesGeneratorTarget: library_files} # The order "__main__" == __name__ would also technically work, but is very @@ -83,11 +83,11 @@ async def find_putative_targets( pts = [] for tgt_type, paths in classified_unowned_py_files.items(): for dirname, filenames in group_by_dir(paths).items(): - name = "tests" if tgt_type == PythonTests else os.path.basename(dirname) - kwargs = {"name": name} if tgt_type == PythonTests else {} + name = "tests" if tgt_type == PythonTestsGeneratorTarget else os.path.basename(dirname) + kwargs = {"name": name} if tgt_type == PythonTestsGeneratorTarget else {} if ( python_setup.tailor_ignore_solitary_init_files - and tgt_type == PythonLibrary + and tgt_type == PythonSourcesGeneratorTarget and filenames == {"__init__.py"} ): continue diff --git a/src/python/pants/backend/python/goals/tailor_test.py b/src/python/pants/backend/python/goals/tailor_test.py index 9d8fa122977..7d1ce41a305 100644 --- a/src/python/pants/backend/python/goals/tailor_test.py +++ b/src/python/pants/backend/python/goals/tailor_test.py @@ -11,7 +11,11 @@ classify_source_files, is_entry_point, ) -from pants.backend.python.target_types import PexBinary, PythonLibrary, PythonTests +from pants.backend.python.target_types import ( + PexBinary, + PythonSourcesGeneratorTarget, + PythonTestsGeneratorTarget, +) from pants.core.goals.tailor import ( AllOwnedSources, PutativeTarget, @@ -34,9 +38,10 @@ def test_classify_source_files() -> None: } lib_files = {"foo/bar/baz.py", "foo/bar_baz.py", "foo.pyi"} - assert {PythonTests: test_files, PythonLibrary: lib_files} == classify_source_files( - test_files | lib_files - ) + assert { + PythonTestsGeneratorTarget: test_files, + PythonSourcesGeneratorTarget: lib_files, + } == classify_source_files(test_files | lib_files) @pytest.fixture @@ -97,13 +102,16 @@ def test_find_putative_targets(rule_runner: RuleRunner) -> None: kwargs={"requirements_relpath": "requirements-test.txt"}, ), PutativeTarget.for_target_type( - PythonLibrary, "src/python/foo", "foo", ["__init__.py"] + PythonSourcesGeneratorTarget, "src/python/foo", "foo", ["__init__.py"] ), PutativeTarget.for_target_type( - PythonLibrary, "src/python/foo/bar", "bar", ["baz2.py", "baz3.py"] + PythonSourcesGeneratorTarget, + "src/python/foo/bar", + "bar", + ["baz2.py", "baz3.py"], ), PutativeTarget.for_target_type( - PythonTests, + PythonTestsGeneratorTarget, "src/python/foo/bar", "tests", ["baz1_test.py", "baz2_test.py"], @@ -143,14 +151,14 @@ def test_find_putative_targets_subset(rule_runner: RuleRunner) -> None: PutativeTargets( [ PutativeTarget.for_target_type( - PythonTests, + PythonTestsGeneratorTarget, "src/python/foo/bar", "tests", ["bar_test.py"], kwargs={"name": "tests"}, ), PutativeTarget.for_target_type( - PythonLibrary, "src/python/foo/qux", "qux", ["qux.py"] + PythonSourcesGeneratorTarget, "src/python/foo/qux", "qux", ["qux.py"] ), ] ) @@ -223,10 +231,13 @@ def test_ignore_solitary_init(rule_runner: RuleRunner) -> None: PutativeTargets( [ PutativeTarget.for_target_type( - PythonLibrary, "src/python/foo/bar", "bar", ["__init__.py", "bar.py"] + PythonSourcesGeneratorTarget, + "src/python/foo/bar", + "bar", + ["__init__.py", "bar.py"], ), PutativeTarget.for_target_type( - PythonLibrary, "src/python/foo/qux", "qux", ["qux.py"] + PythonSourcesGeneratorTarget, "src/python/foo/qux", "qux", ["qux.py"] ), ] ) diff --git a/src/python/pants/backend/python/lint/autoflake/rules.py b/src/python/pants/backend/python/lint/autoflake/rules.py index 0ae97846d1e..44519f2d08f 100644 --- a/src/python/pants/backend/python/lint/autoflake/rules.py +++ b/src/python/pants/backend/python/lint/autoflake/rules.py @@ -7,7 +7,7 @@ from pants.backend.python.lint.autoflake.skip_field import SkipAutoflakeField from pants.backend.python.lint.autoflake.subsystem import Autoflake from pants.backend.python.lint.python_fmt import PythonFmtRequest -from pants.backend.python.target_types import InterpreterConstraintsField, PythonSources +from pants.backend.python.target_types import InterpreterConstraintsField, PythonSourceField from pants.backend.python.util_rules import pex from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess from pants.core.goals.fmt import FmtResult @@ -24,9 +24,9 @@ @dataclass(frozen=True) class AutoflakeFieldSet(FieldSet): - required_fields = (PythonSources,) + required_fields = (PythonSourceField,) - sources: PythonSources + sources: PythonSourceField interpreter_constraints: InterpreterConstraintsField @classmethod diff --git a/src/python/pants/backend/python/lint/autoflake/rules_integration_test.py b/src/python/pants/backend/python/lint/autoflake/rules_integration_test.py index 6eb8315fbb7..8e102f7a3b6 100644 --- a/src/python/pants/backend/python/lint/autoflake/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/autoflake/rules_integration_test.py @@ -10,7 +10,7 @@ from pants.backend.python.lint.autoflake.rules import rules as autoflake_rules from pants.backend.python.lint.autoflake.subsystem import Autoflake from pants.backend.python.lint.autoflake.subsystem import rules as autoflake_subsystem_rules -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.core.goals.fmt import FmtResult from pants.core.goals.lint import LintResult, LintResults from pants.core.util_rules import config_files, source_files @@ -35,7 +35,7 @@ def rule_runner() -> RuleRunner: QueryRule(FmtResult, (AutoflakeRequest,)), QueryRule(SourceFiles, (SourceFilesRequest,)), ], - target_types=[PythonLibrary], + target_types=[PythonSourcesGeneratorTarget], ) diff --git a/src/python/pants/backend/python/lint/autoflake/skip_field.py b/src/python/pants/backend/python/lint/autoflake/skip_field.py index 9e1877f3a8e..eda01fffe35 100644 --- a/src/python/pants/backend/python/lint/autoflake/skip_field.py +++ b/src/python/pants/backend/python/lint/autoflake/skip_field.py @@ -1,7 +1,12 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.backend.python.target_types import PythonLibrary, PythonTests +from pants.backend.python.target_types import ( + PythonSourcesGeneratorTarget, + PythonSourceTarget, + PythonTestsGeneratorTarget, + PythonTestTarget, +) from pants.engine.target import BoolField @@ -13,6 +18,8 @@ class SkipAutoflakeField(BoolField): def rules(): return [ - PythonLibrary.register_plugin_field(SkipAutoflakeField), - PythonTests.register_plugin_field(SkipAutoflakeField), + PythonSourceTarget.register_plugin_field(SkipAutoflakeField), + PythonSourcesGeneratorTarget.register_plugin_field(SkipAutoflakeField), + PythonTestTarget.register_plugin_field(SkipAutoflakeField), + PythonTestsGeneratorTarget.register_plugin_field(SkipAutoflakeField), ] diff --git a/src/python/pants/backend/python/lint/bandit/rules_integration_test.py b/src/python/pants/backend/python/lint/bandit/rules_integration_test.py index be119ea12cf..87dbdeee92f 100644 --- a/src/python/pants/backend/python/lint/bandit/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/bandit/rules_integration_test.py @@ -13,7 +13,7 @@ from pants.backend.python.lint.bandit.rules import rules as bandit_rules from pants.backend.python.lint.bandit.subsystem import BanditFieldSet from pants.backend.python.lint.bandit.subsystem import rules as bandit_subsystem_rules -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.core.goals.lint import LintResult, LintResults from pants.core.util_rules import config_files, source_files from pants.engine.addresses import Address @@ -38,7 +38,7 @@ def rule_runner() -> RuleRunner: *target_types_rules.rules(), QueryRule(LintResults, (BanditRequest,)), ], - target_types=[PythonLibrary], + target_types=[PythonSourcesGeneratorTarget], ) diff --git a/src/python/pants/backend/python/lint/bandit/skip_field.py b/src/python/pants/backend/python/lint/bandit/skip_field.py index 8647d7371f2..360e3ed5eab 100644 --- a/src/python/pants/backend/python/lint/bandit/skip_field.py +++ b/src/python/pants/backend/python/lint/bandit/skip_field.py @@ -1,7 +1,12 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.backend.python.target_types import PythonLibrary, PythonTests +from pants.backend.python.target_types import ( + PythonSourcesGeneratorTarget, + PythonSourceTarget, + PythonTestsGeneratorTarget, + PythonTestTarget, +) from pants.engine.target import BoolField @@ -13,6 +18,8 @@ class SkipBanditField(BoolField): def rules(): return [ - PythonLibrary.register_plugin_field(SkipBanditField), - PythonTests.register_plugin_field(SkipBanditField), + PythonSourceTarget.register_plugin_field(SkipBanditField), + PythonSourcesGeneratorTarget.register_plugin_field(SkipBanditField), + PythonTestTarget.register_plugin_field(SkipBanditField), + PythonTestsGeneratorTarget.register_plugin_field(SkipBanditField), ] diff --git a/src/python/pants/backend/python/lint/bandit/subsystem.py b/src/python/pants/backend/python/lint/bandit/subsystem.py index 4ce36b5afbf..e49c4d867fc 100644 --- a/src/python/pants/backend/python/lint/bandit/subsystem.py +++ b/src/python/pants/backend/python/lint/bandit/subsystem.py @@ -13,13 +13,13 @@ from pants.backend.python.target_types import ( ConsoleScript, InterpreterConstraintsField, - PythonSources, + PythonSourceField, ) from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.base.specs import AddressSpecs, DescendantAddresses from pants.core.util_rules.config_files import ConfigFilesRequest from pants.engine.rules import Get, collect_rules, rule -from pants.engine.target import FieldSet, Target, UnexpandedTargets +from pants.engine.target import FieldSet, Target, Targets from pants.engine.unions import UnionRule from pants.option.custom_types import file_option, shell_str from pants.python.python_setup import PythonSetup @@ -29,9 +29,9 @@ @dataclass(frozen=True) class BanditFieldSet(FieldSet): - required_fields = (PythonSources,) + required_fields = (PythonSourceField,) - sources: PythonSources + sources: PythonSourceField interpreter_constraints: InterpreterConstraintsField @classmethod @@ -132,10 +132,10 @@ async def setup_bandit_lockfile( # # This ORs all unique interpreter constraints. The net effect is that every possible Python # interpreter used will be covered. - all_build_targets = await Get(UnexpandedTargets, AddressSpecs([DescendantAddresses("")])) + all_tgts = await Get(Targets, AddressSpecs([DescendantAddresses("")])) unique_constraints = { InterpreterConstraints.create_from_targets([tgt], python_setup) - for tgt in all_build_targets + for tgt in all_tgts if BanditFieldSet.is_applicable(tgt) } constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints)) diff --git a/src/python/pants/backend/python/lint/bandit/subsystem_test.py b/src/python/pants/backend/python/lint/bandit/subsystem_test.py index b067ba9a871..939a8bff48b 100644 --- a/src/python/pants/backend/python/lint/bandit/subsystem_test.py +++ b/src/python/pants/backend/python/lint/bandit/subsystem_test.py @@ -5,11 +5,12 @@ from textwrap import dedent +from pants.backend.python import target_types_rules from pants.backend.python.goals.lockfile import PythonLockfileRequest from pants.backend.python.lint.bandit import skip_field from pants.backend.python.lint.bandit.subsystem import BanditLockfileSentinel from pants.backend.python.lint.bandit.subsystem import rules as subsystem_rules -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.core.target_types import GenericTarget from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -20,9 +21,10 @@ def test_setup_lockfile_interpreter_constraints() -> None: rules=[ *subsystem_rules(), *skip_field.rules(), + *target_types_rules.rules(), QueryRule(PythonLockfileRequest, [BanditLockfileSentinel]), ], - target_types=[PythonLibrary, GenericTarget], + target_types=[PythonSourcesGeneratorTarget, GenericTarget], ) global_constraint = "==3.9.*" @@ -32,7 +34,7 @@ def test_setup_lockfile_interpreter_constraints() -> None: ) def assert_ics(build_file: str, expected: list[str]) -> None: - rule_runner.write_files({"project/BUILD": build_file}) + rule_runner.write_files({"project/BUILD": build_file, "project/f.py": ""}) lockfile_request = rule_runner.request(PythonLockfileRequest, [BanditLockfileSentinel()]) assert lockfile_request.interpreter_constraints == InterpreterConstraints(expected) diff --git a/src/python/pants/backend/python/lint/black/rules.py b/src/python/pants/backend/python/lint/black/rules.py index ffac61c0b8f..5f6e1a68439 100644 --- a/src/python/pants/backend/python/lint/black/rules.py +++ b/src/python/pants/backend/python/lint/black/rules.py @@ -7,7 +7,7 @@ from pants.backend.python.lint.black.skip_field import SkipBlackField from pants.backend.python.lint.black.subsystem import Black from pants.backend.python.lint.python_fmt import PythonFmtRequest -from pants.backend.python.target_types import InterpreterConstraintsField, PythonSources +from pants.backend.python.target_types import InterpreterConstraintsField, PythonSourceField from pants.backend.python.util_rules import pex from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess @@ -27,9 +27,9 @@ @dataclass(frozen=True) class BlackFieldSet(FieldSet): - required_fields = (PythonSources,) + required_fields = (PythonSourceField,) - sources: PythonSources + sources: PythonSourceField interpreter_constraints: InterpreterConstraintsField @classmethod diff --git a/src/python/pants/backend/python/lint/black/rules_integration_test.py b/src/python/pants/backend/python/lint/black/rules_integration_test.py index 7441b6b6a3a..ef66eda222f 100644 --- a/src/python/pants/backend/python/lint/black/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/black/rules_integration_test.py @@ -12,7 +12,7 @@ from pants.backend.python.lint.black.rules import rules as black_rules from pants.backend.python.lint.black.subsystem import Black from pants.backend.python.lint.black.subsystem import rules as black_subsystem_rules -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.core.goals.fmt import FmtResult from pants.core.goals.lint import LintResult, LintResults from pants.core.util_rules import config_files, source_files @@ -41,7 +41,7 @@ def rule_runner() -> RuleRunner: QueryRule(FmtResult, (BlackRequest,)), QueryRule(SourceFiles, (SourceFilesRequest,)), ], - target_types=[PythonLibrary], + target_types=[PythonSourcesGeneratorTarget], ) diff --git a/src/python/pants/backend/python/lint/black/skip_field.py b/src/python/pants/backend/python/lint/black/skip_field.py index 7309ae513a9..ff9f7293c28 100644 --- a/src/python/pants/backend/python/lint/black/skip_field.py +++ b/src/python/pants/backend/python/lint/black/skip_field.py @@ -1,7 +1,12 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.backend.python.target_types import PythonLibrary, PythonTests +from pants.backend.python.target_types import ( + PythonSourcesGeneratorTarget, + PythonSourceTarget, + PythonTestsGeneratorTarget, + PythonTestTarget, +) from pants.engine.target import BoolField @@ -13,6 +18,8 @@ class SkipBlackField(BoolField): def rules(): return [ - PythonLibrary.register_plugin_field(SkipBlackField), - PythonTests.register_plugin_field(SkipBlackField), + PythonSourceTarget.register_plugin_field(SkipBlackField), + PythonSourcesGeneratorTarget.register_plugin_field(SkipBlackField), + PythonTestTarget.register_plugin_field(SkipBlackField), + PythonTestsGeneratorTarget.register_plugin_field(SkipBlackField), ] diff --git a/src/python/pants/backend/python/lint/black/subsystem_test.py b/src/python/pants/backend/python/lint/black/subsystem_test.py index ad12fd09969..2001c1dce77 100644 --- a/src/python/pants/backend/python/lint/black/subsystem_test.py +++ b/src/python/pants/backend/python/lint/black/subsystem_test.py @@ -9,7 +9,7 @@ from pants.backend.python.lint.black import skip_field from pants.backend.python.lint.black.subsystem import Black, BlackLockfileSentinel from pants.backend.python.lint.black.subsystem import rules as subsystem_rules -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.core.target_types import GenericTarget from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -22,7 +22,7 @@ def test_setup_lockfile_interpreter_constraints() -> None: *skip_field.rules(), QueryRule(PythonLockfileRequest, [BlackLockfileSentinel]), ], - target_types=[PythonLibrary, GenericTarget], + target_types=[PythonSourcesGeneratorTarget, GenericTarget], ) global_constraint = "==3.9.*" diff --git a/src/python/pants/backend/python/lint/docformatter/rules.py b/src/python/pants/backend/python/lint/docformatter/rules.py index b27a9a2b7b0..be66c6130d5 100644 --- a/src/python/pants/backend/python/lint/docformatter/rules.py +++ b/src/python/pants/backend/python/lint/docformatter/rules.py @@ -7,7 +7,7 @@ from pants.backend.python.lint.docformatter.skip_field import SkipDocformatterField from pants.backend.python.lint.docformatter.subsystem import Docformatter from pants.backend.python.lint.python_fmt import PythonFmtRequest -from pants.backend.python.target_types import PythonSources +from pants.backend.python.target_types import PythonSourceField from pants.backend.python.util_rules import pex from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess from pants.core.goals.fmt import FmtResult @@ -24,9 +24,9 @@ @dataclass(frozen=True) class DocformatterFieldSet(FieldSet): - required_fields = (PythonSources,) + required_fields = (PythonSourceField,) - sources: PythonSources + sources: PythonSourceField @classmethod def opt_out(cls, tgt: Target) -> bool: diff --git a/src/python/pants/backend/python/lint/docformatter/rules_integration_test.py b/src/python/pants/backend/python/lint/docformatter/rules_integration_test.py index 86b5c5f0dba..dcdd276771c 100644 --- a/src/python/pants/backend/python/lint/docformatter/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/docformatter/rules_integration_test.py @@ -10,7 +10,7 @@ from pants.backend.python.lint.docformatter.rules import rules as docformatter_rules from pants.backend.python.lint.docformatter.subsystem import Docformatter from pants.backend.python.lint.docformatter.subsystem import rules as docformatter_subsystem_rules -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.core.goals.fmt import FmtResult from pants.core.goals.lint import LintResult, LintResults from pants.core.util_rules import source_files @@ -34,7 +34,7 @@ def rule_runner() -> RuleRunner: QueryRule(FmtResult, (DocformatterRequest,)), QueryRule(SourceFiles, (SourceFilesRequest,)), ], - target_types=[PythonLibrary], + target_types=[PythonSourcesGeneratorTarget], ) diff --git a/src/python/pants/backend/python/lint/docformatter/skip_field.py b/src/python/pants/backend/python/lint/docformatter/skip_field.py index b0273c89181..09f6e783310 100644 --- a/src/python/pants/backend/python/lint/docformatter/skip_field.py +++ b/src/python/pants/backend/python/lint/docformatter/skip_field.py @@ -1,7 +1,12 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.backend.python.target_types import PythonLibrary, PythonTests +from pants.backend.python.target_types import ( + PythonSourcesGeneratorTarget, + PythonSourceTarget, + PythonTestsGeneratorTarget, + PythonTestTarget, +) from pants.engine.target import BoolField @@ -13,6 +18,8 @@ class SkipDocformatterField(BoolField): def rules(): return [ - PythonLibrary.register_plugin_field(SkipDocformatterField), - PythonTests.register_plugin_field(SkipDocformatterField), + PythonSourceTarget.register_plugin_field(SkipDocformatterField), + PythonSourcesGeneratorTarget.register_plugin_field(SkipDocformatterField), + PythonTestTarget.register_plugin_field(SkipDocformatterField), + PythonTestsGeneratorTarget.register_plugin_field(SkipDocformatterField), ] diff --git a/src/python/pants/backend/python/lint/flake8/rules_integration_test.py b/src/python/pants/backend/python/lint/flake8/rules_integration_test.py index 8f399869e13..7b3e0be5950 100644 --- a/src/python/pants/backend/python/lint/flake8/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/flake8/rules_integration_test.py @@ -12,7 +12,7 @@ from pants.backend.python.lint.flake8.rules import rules as flake8_rules from pants.backend.python.lint.flake8.subsystem import Flake8FieldSet from pants.backend.python.lint.flake8.subsystem import rules as flake8_subsystem_rules -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.core.goals.lint import LintResult, LintResults from pants.core.util_rules import config_files, source_files from pants.engine.addresses import Address @@ -37,7 +37,7 @@ def rule_runner() -> RuleRunner: *target_types_rules.rules(), QueryRule(LintResults, [Flake8Request]), ], - target_types=[PythonLibrary], + target_types=[PythonSourcesGeneratorTarget], ) diff --git a/src/python/pants/backend/python/lint/flake8/skip_field.py b/src/python/pants/backend/python/lint/flake8/skip_field.py index 0e7d1b23164..ecdc3bd281e 100644 --- a/src/python/pants/backend/python/lint/flake8/skip_field.py +++ b/src/python/pants/backend/python/lint/flake8/skip_field.py @@ -1,7 +1,12 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.backend.python.target_types import PythonLibrary, PythonTests +from pants.backend.python.target_types import ( + PythonSourcesGeneratorTarget, + PythonSourceTarget, + PythonTestsGeneratorTarget, + PythonTestTarget, +) from pants.engine.target import BoolField @@ -13,6 +18,8 @@ class SkipFlake8Field(BoolField): def rules(): return [ - PythonLibrary.register_plugin_field(SkipFlake8Field), - PythonTests.register_plugin_field(SkipFlake8Field), + PythonSourceTarget.register_plugin_field(SkipFlake8Field), + PythonSourcesGeneratorTarget.register_plugin_field(SkipFlake8Field), + PythonTestTarget.register_plugin_field(SkipFlake8Field), + PythonTestsGeneratorTarget.register_plugin_field(SkipFlake8Field), ] diff --git a/src/python/pants/backend/python/lint/flake8/subsystem.py b/src/python/pants/backend/python/lint/flake8/subsystem.py index 73bdad034e1..ae747baef97 100644 --- a/src/python/pants/backend/python/lint/flake8/subsystem.py +++ b/src/python/pants/backend/python/lint/flake8/subsystem.py @@ -13,13 +13,13 @@ from pants.backend.python.target_types import ( ConsoleScript, InterpreterConstraintsField, - PythonSources, + PythonSourceField, ) from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.base.specs import AddressSpecs, DescendantAddresses from pants.core.util_rules.config_files import ConfigFilesRequest from pants.engine.rules import Get, collect_rules, rule -from pants.engine.target import FieldSet, Target, UnexpandedTargets +from pants.engine.target import FieldSet, Target, Targets from pants.engine.unions import UnionRule from pants.option.custom_types import file_option, shell_str from pants.python.python_setup import PythonSetup @@ -29,9 +29,9 @@ @dataclass(frozen=True) class Flake8FieldSet(FieldSet): - required_fields = (PythonSources,) + required_fields = (PythonSourceField,) - sources: PythonSources + sources: PythonSourceField interpreter_constraints: InterpreterConstraintsField @classmethod @@ -141,10 +141,10 @@ async def setup_flake8_lockfile( # # This ORs all unique interpreter constraints. The net effect is that every possible Python # interpreter used will be covered. - all_build_targets = await Get(UnexpandedTargets, AddressSpecs([DescendantAddresses("")])) + all_tgts = await Get(Targets, AddressSpecs([DescendantAddresses("")])) unique_constraints = { InterpreterConstraints.create_from_targets([tgt], python_setup) - for tgt in all_build_targets + for tgt in all_tgts if Flake8FieldSet.is_applicable(tgt) } constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints)) diff --git a/src/python/pants/backend/python/lint/flake8/subsystem_test.py b/src/python/pants/backend/python/lint/flake8/subsystem_test.py index f3afb014f66..60558f1932b 100644 --- a/src/python/pants/backend/python/lint/flake8/subsystem_test.py +++ b/src/python/pants/backend/python/lint/flake8/subsystem_test.py @@ -5,11 +5,12 @@ from textwrap import dedent +from pants.backend.python import target_types_rules from pants.backend.python.goals.lockfile import PythonLockfileRequest from pants.backend.python.lint.flake8 import skip_field from pants.backend.python.lint.flake8.subsystem import Flake8LockfileSentinel from pants.backend.python.lint.flake8.subsystem import rules as subsystem_rules -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.core.target_types import GenericTarget from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -20,9 +21,10 @@ def test_setup_lockfile_interpreter_constraints() -> None: rules=[ *subsystem_rules(), *skip_field.rules(), + *target_types_rules.rules(), QueryRule(PythonLockfileRequest, [Flake8LockfileSentinel]), ], - target_types=[PythonLibrary, GenericTarget], + target_types=[PythonSourcesGeneratorTarget, GenericTarget], ) global_constraint = "==3.9.*" @@ -32,7 +34,7 @@ def test_setup_lockfile_interpreter_constraints() -> None: ) def assert_ics(build_file: str, expected: list[str]) -> None: - rule_runner.write_files({"project/BUILD": build_file}) + rule_runner.write_files({"project/BUILD": build_file, "project/f.py": ""}) lockfile_request = rule_runner.request(PythonLockfileRequest, [Flake8LockfileSentinel()]) assert lockfile_request.interpreter_constraints == InterpreterConstraints(expected) diff --git a/src/python/pants/backend/python/lint/isort/rules.py b/src/python/pants/backend/python/lint/isort/rules.py index 88fee76ea1d..76d951ad2a6 100644 --- a/src/python/pants/backend/python/lint/isort/rules.py +++ b/src/python/pants/backend/python/lint/isort/rules.py @@ -7,7 +7,7 @@ from pants.backend.python.lint.isort.skip_field import SkipIsortField from pants.backend.python.lint.isort.subsystem import Isort from pants.backend.python.lint.python_fmt import PythonFmtRequest -from pants.backend.python.target_types import PythonSources +from pants.backend.python.target_types import PythonSourceField from pants.backend.python.util_rules import pex from pants.backend.python.util_rules.pex import PexRequest, PexResolveInfo, VenvPex, VenvPexProcess from pants.core.goals.fmt import FmtResult @@ -25,9 +25,9 @@ @dataclass(frozen=True) class IsortFieldSet(FieldSet): - required_fields = (PythonSources,) + required_fields = (PythonSourceField,) - sources: PythonSources + sources: PythonSourceField @classmethod def opt_out(cls, tgt: Target) -> bool: diff --git a/src/python/pants/backend/python/lint/isort/rules_integration_test.py b/src/python/pants/backend/python/lint/isort/rules_integration_test.py index f36fd62f07c..65e410f632d 100644 --- a/src/python/pants/backend/python/lint/isort/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/isort/rules_integration_test.py @@ -10,7 +10,7 @@ from pants.backend.python.lint.isort.rules import rules as isort_rules from pants.backend.python.lint.isort.subsystem import Isort from pants.backend.python.lint.isort.subsystem import rules as isort_subsystem_rules -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.core.goals.fmt import FmtResult from pants.core.goals.lint import LintResult, LintResults from pants.core.util_rules import config_files, source_files @@ -35,7 +35,7 @@ def rule_runner() -> RuleRunner: QueryRule(FmtResult, (IsortRequest,)), QueryRule(SourceFiles, (SourceFilesRequest,)), ], - target_types=[PythonLibrary], + target_types=[PythonSourcesGeneratorTarget], ) diff --git a/src/python/pants/backend/python/lint/isort/skip_field.py b/src/python/pants/backend/python/lint/isort/skip_field.py index f1e7c323003..89426431850 100644 --- a/src/python/pants/backend/python/lint/isort/skip_field.py +++ b/src/python/pants/backend/python/lint/isort/skip_field.py @@ -1,7 +1,12 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.backend.python.target_types import PythonLibrary, PythonTests +from pants.backend.python.target_types import ( + PythonSourcesGeneratorTarget, + PythonSourceTarget, + PythonTestsGeneratorTarget, + PythonTestTarget, +) from pants.engine.target import BoolField @@ -13,6 +18,8 @@ class SkipIsortField(BoolField): def rules(): return [ - PythonLibrary.register_plugin_field(SkipIsortField), - PythonTests.register_plugin_field(SkipIsortField), + PythonSourceTarget.register_plugin_field(SkipIsortField), + PythonSourcesGeneratorTarget.register_plugin_field(SkipIsortField), + PythonTestTarget.register_plugin_field(SkipIsortField), + PythonTestsGeneratorTarget.register_plugin_field(SkipIsortField), ] diff --git a/src/python/pants/backend/python/lint/pylint/rules_integration_test.py b/src/python/pants/backend/python/lint/pylint/rules_integration_test.py index ed2f7ecc8c8..35f82e6e527 100644 --- a/src/python/pants/backend/python/lint/pylint/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/pylint/rules_integration_test.py @@ -12,7 +12,7 @@ from pants.backend.python.lint.pylint.rules import PylintRequest from pants.backend.python.lint.pylint.rules import rules as pylint_rules from pants.backend.python.lint.pylint.subsystem import PylintFieldSet -from pants.backend.python.target_types import PythonLibrary, PythonRequirementTarget +from pants.backend.python.target_types import PythonRequirementTarget, PythonSourcesGeneratorTarget from pants.core.goals.lint import LintResult, LintResults from pants.core.util_rules import config_files from pants.engine.addresses import Address @@ -35,7 +35,7 @@ def rule_runner() -> RuleRunner: *target_types_rules.rules(), QueryRule(LintResults, [PylintRequest]), ], - target_types=[PythonLibrary, PythonRequirementTarget], + target_types=[PythonSourcesGeneratorTarget, PythonRequirementTarget], ) diff --git a/src/python/pants/backend/python/lint/pylint/skip_field.py b/src/python/pants/backend/python/lint/pylint/skip_field.py index cbdbf4b50bd..086fc11292d 100644 --- a/src/python/pants/backend/python/lint/pylint/skip_field.py +++ b/src/python/pants/backend/python/lint/pylint/skip_field.py @@ -1,7 +1,12 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.backend.python.target_types import PythonLibrary, PythonTests +from pants.backend.python.target_types import ( + PythonSourcesGeneratorTarget, + PythonSourceTarget, + PythonTestsGeneratorTarget, + PythonTestTarget, +) from pants.engine.target import BoolField @@ -13,6 +18,8 @@ class SkipPylintField(BoolField): def rules(): return [ - PythonLibrary.register_plugin_field(SkipPylintField), - PythonTests.register_plugin_field(SkipPylintField), + PythonSourceTarget.register_plugin_field(SkipPylintField), + PythonSourcesGeneratorTarget.register_plugin_field(SkipPylintField), + PythonTestTarget.register_plugin_field(SkipPylintField), + PythonTestsGeneratorTarget.register_plugin_field(SkipPylintField), ] diff --git a/src/python/pants/backend/python/lint/pylint/subsystem.py b/src/python/pants/backend/python/lint/pylint/subsystem.py index 03f404428d5..bb7064571f7 100644 --- a/src/python/pants/backend/python/lint/pylint/subsystem.py +++ b/src/python/pants/backend/python/lint/pylint/subsystem.py @@ -15,7 +15,7 @@ ConsoleScript, InterpreterConstraintsField, PythonRequirementsField, - PythonSources, + PythonSourceField, ) from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.backend.python.util_rules.pex import PexRequirements @@ -33,9 +33,9 @@ DependenciesRequest, FieldSet, Target, + Targets, TransitiveTargets, TransitiveTargetsRequest, - UnexpandedTargets, ) from pants.engine.unions import UnionRule from pants.option.custom_types import file_option, shell_str, target_option @@ -47,9 +47,9 @@ @dataclass(frozen=True) class PylintFieldSet(FieldSet): - required_fields = (PythonSources,) + required_fields = (PythonSourceField,) - sources: PythonSources + sources: PythonSourceField dependencies: Dependencies @classmethod @@ -257,11 +257,10 @@ async def setup_pylint_lockfile( # dependencies (which will AND across each target in the closure). Then, it ORs all unique # resulting interpreter constraints. The net effect is that every possible Python interpreter # used will be covered. - all_build_targets = await Get(UnexpandedTargets, AddressSpecs([DescendantAddresses("")])) - relevant_targets = tuple(tgt for tgt in all_build_targets if PylintFieldSet.is_applicable(tgt)) + all_tgts = await Get(Targets, AddressSpecs([DescendantAddresses("")])) + relevant_targets = tuple(tgt for tgt in all_tgts if PylintFieldSet.is_applicable(tgt)) direct_deps_per_target = await MultiGet( - Get(UnexpandedTargets, DependenciesRequest(tgt.get(Dependencies))) - for tgt in relevant_targets + Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in relevant_targets ) unique_constraints = set() diff --git a/src/python/pants/backend/python/lint/pylint/subsystem_test.py b/src/python/pants/backend/python/lint/pylint/subsystem_test.py index 38a38aae271..756934887bc 100644 --- a/src/python/pants/backend/python/lint/pylint/subsystem_test.py +++ b/src/python/pants/backend/python/lint/pylint/subsystem_test.py @@ -7,6 +7,7 @@ import pytest +from pants.backend.python import target_types_rules from pants.backend.python.goals.lockfile import PythonLockfileRequest from pants.backend.python.lint.pylint import skip_field from pants.backend.python.lint.pylint.subsystem import ( @@ -17,8 +18,8 @@ from pants.backend.python.lint.pylint.subsystem import rules as subsystem_rules from pants.backend.python.target_types import ( InterpreterConstraintsField, - PythonLibrary, PythonRequirementTarget, + PythonSourcesGeneratorTarget, ) from pants.backend.python.util_rules import python_sources from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints @@ -35,10 +36,11 @@ def rule_runner() -> RuleRunner: *subsystem_rules(), *skip_field.rules(), *python_sources.rules(), + *target_types_rules.rules(), QueryRule(PylintFirstPartyPlugins, []), QueryRule(PythonLockfileRequest, [PylintLockfileSentinel]), ], - target_types=[PythonLibrary, GenericTarget, PythonRequirementTarget], + target_types=[PythonSourcesGeneratorTarget, GenericTarget, PythonRequirementTarget], ) @@ -55,13 +57,13 @@ def test_first_party_plugins(rule_runner: RuleRunner) -> None: "pylint-plugins/subdir1/BUILD": dedent( """\ python_sources( - interpreter_constraints=['==3.5.*'], + interpreter_constraints=['==3.9.*'], dependencies=['pylint-plugins/subdir2'] ) """ ), "pylint-plugins/subdir2/another_util.py": "", - "pylint-plugins/subdir2/BUILD": ("python_sources(interpreter_constraints=['==3.4.*'])"), + "pylint-plugins/subdir2/BUILD": "python_sources(interpreter_constraints=['==3.8.*'])", "pylint-plugins/plugin.py": "", "pylint-plugins/BUILD": dedent( """\ @@ -76,7 +78,8 @@ def test_first_party_plugins(rule_runner: RuleRunner) -> None: [ "--source-root-patterns=pylint-plugins", "--pylint-source-plugins=pylint-plugins/plugin.py", - ] + ], + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, ) first_party_plugins = rule_runner.request(PylintFirstPartyPlugins, []) assert first_party_plugins.requirement_strings == FrozenOrderedSet( @@ -85,7 +88,7 @@ def test_first_party_plugins(rule_runner: RuleRunner) -> None: assert first_party_plugins.interpreter_constraints_fields == FrozenOrderedSet( [ InterpreterConstraintsField(ic, Address("", target_name="tgt")) - for ic in (None, ["==3.5.*"], ["==3.4.*"]) + for ic in (None, ["==3.9.*"], ["==3.8.*"]) ] ) assert ( @@ -110,10 +113,11 @@ def assert_lockfile_request( extra_expected_requirements: list[str] | None = None, extra_args: list[str] | None = None, ) -> None: - rule_runner.write_files({"project/BUILD": build_file}) + rule_runner.write_files({"project/BUILD": build_file, "project/f.py": ""}) rule_runner.set_options( ["--pylint-lockfile=lockfile.txt", *(extra_args or [])], env={"PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS": f"['{global_constraint}']"}, + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, ) lockfile_request = rule_runner.request(PythonLockfileRequest, [PylintLockfileSentinel()]) assert lockfile_request.interpreter_constraints == InterpreterConstraints(expected_ics) @@ -128,7 +132,7 @@ def assert_lockfile_request( assert_lockfile_request("python_sources()", [global_constraint]) assert_lockfile_request("python_sources(interpreter_constraints=['==2.7.*'])", ["==2.7.*"]) assert_lockfile_request( - "python_sources(interpreter_constraints=['==2.7.*', '==3.5.*'])", ["==2.7.*", "==3.5.*"] + "python_sources(interpreter_constraints=['==2.7.*', '==3.8.*'])", ["==2.7.*", "==3.8.*"] ) # If no Python targets in repo, fall back to global python-setup constraints. @@ -139,7 +143,7 @@ def assert_lockfile_request( dedent( """\ python_sources(name='a', interpreter_constraints=['==2.7.*']) - python_sources(name='b', interpreter_constraints=['==3.5.*'], skip_pylint=True) + python_sources(name='b', interpreter_constraints=['==3.8.*'], skip_pylint=True) """ ), ["==2.7.*"], @@ -151,29 +155,29 @@ def assert_lockfile_request( dedent( """\ python_sources(name='a', interpreter_constraints=['==2.7.*']) - python_sources(name='b', interpreter_constraints=['==3.5.*']) + python_sources(name='b', interpreter_constraints=['==3.8.*']) """ ), - ["==2.7.*", "==3.5.*"], + ["==2.7.*", "==3.8.*"], ) assert_lockfile_request( dedent( """\ - python_sources(name='a', interpreter_constraints=['==2.7.*', '==3.5.*']) - python_sources(name='b', interpreter_constraints=['>=3.5']) + python_sources(name='a', interpreter_constraints=['==2.7.*', '==3.8.*']) + python_sources(name='b', interpreter_constraints=['>=3.8']) """ ), - ["==2.7.*", "==3.5.*", ">=3.5"], + ["==2.7.*", "==3.8.*", ">=3.8"], ) assert_lockfile_request( dedent( """\ python_sources(name='a') python_sources(name='b', interpreter_constraints=['==2.7.*']) - python_sources(name='c', interpreter_constraints=['>=3.6']) + python_sources(name='c', interpreter_constraints=['>=3.8']) """ ), - ["==2.7.*", global_constraint, ">=3.6"], + ["==2.7.*", global_constraint, ">=3.8"], ) # Also consider direct deps (but not transitive). They should be ANDed within each target's @@ -187,29 +191,29 @@ def assert_lockfile_request( python_sources( name='dep', dependencies=[":transitive_dep"], - interpreter_constraints=['==2.7.*', '==3.6.*'], + interpreter_constraints=['==2.7.*', '==3.8.*'], skip_pylint=True, ) python_sources(name='app', dependencies=[":dep"], interpreter_constraints=['==2.7.*']) """ ), - ["==2.7.*", "==2.7.*,==3.6.*"], + ["==2.7.*", "==2.7.*,==3.8.*"], ) assert_lockfile_request( dedent( """\ python_sources( - name='lib1', interpreter_constraints=['==2.7.*', '==3.6.*'], skip_pylint=True + name='lib1', interpreter_constraints=['==2.7.*', '==3.8.*'], skip_pylint=True ) python_sources(name='app1', dependencies=[":lib1"], interpreter_constraints=['==2.7.*']) python_sources( name='lib2', interpreter_constraints=['>=3.7'], skip_pylint=True ) - python_sources(name='app2', dependencies=[":lib2"], interpreter_constraints=['==3.8.*']) + python_sources(name='app2', dependencies=[":lib2"], interpreter_constraints=['==3.9.*']) """ ), - ["==2.7.*", "==2.7.*,==3.6.*", ">=3.7,==3.8.*"], + ["==2.7.*", "==2.7.*,==3.8.*", ">=3.7,==3.9.*"], ) # Check that source_plugins are included, even if they aren't linted directly. Plugins @@ -219,47 +223,42 @@ def assert_lockfile_request( """\ python_sources( name="lib", - sources=[], - interpreter_constraints=['==3.6.*'], + interpreter_constraints=['==3.8.*'], ) python_sources( name="plugin", - sources=[], interpreter_constraints=['==2.7.*'], skip_pylint=True, ) """ ), - ["==2.7.*,==3.6.*"], + ["==2.7.*,==3.8.*"], extra_args=["--pylint-source-plugins=project:plugin"], ) assert_lockfile_request( dedent( """\ python_sources( - sources=[], dependencies=[":direct_dep"], - interpreter_constraints=['==3.6.*'], + interpreter_constraints=['==3.8.*'], skip_pylint=True, ) python_sources( name="direct_dep", - sources=[], dependencies=[":transitive_dep"], - interpreter_constraints=['==3.6.*'], + interpreter_constraints=['==3.8.*'], skip_pylint=True, ) python_sources( name="transitive_dep", - sources=[], dependencies=[":thirdparty"], - interpreter_constraints=['==2.7.*', '==3.6.*'], + interpreter_constraints=['==2.7.*', '==3.8.*'], skip_pylint=True, ) python_requirement(name="thirdparty", requirements=["ansicolors"]) """ ), - ["==2.7.*,==3.6.*", "==3.6.*"], + ["==2.7.*,==3.8.*", "==3.8.*"], extra_args=["--pylint-source-plugins=project"], extra_expected_requirements=["ansicolors"], ) diff --git a/src/python/pants/backend/python/lint/python_fmt.py b/src/python/pants/backend/python/lint/python_fmt.py index 72f42a9bb25..eb334ba00c9 100644 --- a/src/python/pants/backend/python/lint/python_fmt.py +++ b/src/python/pants/backend/python/lint/python_fmt.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from typing import Iterable -from pants.backend.python.target_types import PythonSources +from pants.backend.python.target_types import PythonSourceField from pants.core.goals.fmt import FmtResult, LanguageFmtResults, LanguageFmtTargets from pants.core.goals.style_request import StyleRequest from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest @@ -15,7 +15,7 @@ @dataclass(frozen=True) class PythonFmtTargets(LanguageFmtTargets): - required_fields = (PythonSources,) + required_fields = (PythonSourceField,) @union @@ -29,7 +29,7 @@ async def format_python_target( ) -> LanguageFmtResults: original_sources = await Get( SourceFiles, - SourceFilesRequest(target[PythonSources] for target in python_fmt_targets.targets), + SourceFilesRequest(target[PythonSourceField] for target in python_fmt_targets.targets), ) prior_formatter_result = original_sources.snapshot diff --git a/src/python/pants/backend/python/lint/python_fmt_integration_test.py b/src/python/pants/backend/python/lint/python_fmt_integration_test.py index bd172297f0a..4559242fc33 100644 --- a/src/python/pants/backend/python/lint/python_fmt_integration_test.py +++ b/src/python/pants/backend/python/lint/python_fmt_integration_test.py @@ -5,12 +5,13 @@ import pytest +from pants.backend.python import target_types_rules from pants.backend.python.lint.black.rules import rules as black_rules from pants.backend.python.lint.black.subsystem import rules as black_subsystem_rules from pants.backend.python.lint.isort.rules import rules as isort_rules from pants.backend.python.lint.isort.subsystem import rules as isort_subsystem_rules from pants.backend.python.lint.python_fmt import PythonFmtTargets, format_python_target -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.core.goals.fmt import LanguageFmtResults from pants.core.util_rules import config_files, source_files from pants.engine.addresses import Address @@ -33,9 +34,10 @@ def rule_runner() -> RuleRunner: *isort_subsystem_rules(), *source_files.rules(), *config_files.rules(), + *target_types_rules.rules(), QueryRule(LanguageFmtResults, (PythonFmtTargets,)), ], - target_types=[PythonLibrary], + target_types=[PythonSourcesGeneratorTarget], ) diff --git a/src/python/pants/backend/python/lint/pyupgrade/rules.py b/src/python/pants/backend/python/lint/pyupgrade/rules.py index 4383e73300d..06de02e770e 100644 --- a/src/python/pants/backend/python/lint/pyupgrade/rules.py +++ b/src/python/pants/backend/python/lint/pyupgrade/rules.py @@ -7,7 +7,7 @@ from pants.backend.python.lint.python_fmt import PythonFmtRequest from pants.backend.python.lint.pyupgrade.skip_field import SkipPyUpgradeField from pants.backend.python.lint.pyupgrade.subsystem import PyUpgrade -from pants.backend.python.target_types import PythonSources +from pants.backend.python.target_types import PythonSourceField from pants.backend.python.util_rules import pex from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess from pants.core.goals.fmt import FmtResult @@ -24,9 +24,9 @@ @dataclass(frozen=True) class PyUpgradeFieldSet(FieldSet): - required_fields = (PythonSources,) + required_fields = (PythonSourceField,) - sources: PythonSources + sources: PythonSourceField @classmethod def opt_out(cls, tgt: Target) -> bool: diff --git a/src/python/pants/backend/python/lint/pyupgrade/rules_integration_test.py b/src/python/pants/backend/python/lint/pyupgrade/rules_integration_test.py index 137dbc9a937..5568f7b46b1 100644 --- a/src/python/pants/backend/python/lint/pyupgrade/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/pyupgrade/rules_integration_test.py @@ -10,7 +10,7 @@ from pants.backend.python.lint.pyupgrade.rules import rules as pyupgrade_rules from pants.backend.python.lint.pyupgrade.subsystem import PyUpgrade from pants.backend.python.lint.pyupgrade.subsystem import rules as pyupgrade_subsystem_rules -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.core.goals.fmt import FmtResult from pants.core.goals.lint import LintResult, LintResults from pants.core.util_rules import config_files, source_files @@ -35,7 +35,7 @@ def rule_runner() -> RuleRunner: QueryRule(FmtResult, (PyUpgradeRequest,)), QueryRule(SourceFiles, (SourceFilesRequest,)), ], - target_types=[PythonLibrary], + target_types=[PythonSourcesGeneratorTarget], ) diff --git a/src/python/pants/backend/python/lint/pyupgrade/skip_field.py b/src/python/pants/backend/python/lint/pyupgrade/skip_field.py index b3ea8d7d7a8..eeb80ba3518 100644 --- a/src/python/pants/backend/python/lint/pyupgrade/skip_field.py +++ b/src/python/pants/backend/python/lint/pyupgrade/skip_field.py @@ -1,7 +1,12 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.backend.python.target_types import PythonLibrary, PythonTests +from pants.backend.python.target_types import ( + PythonSourcesGeneratorTarget, + PythonSourceTarget, + PythonTestsGeneratorTarget, + PythonTestTarget, +) from pants.engine.target import BoolField @@ -13,6 +18,8 @@ class SkipPyUpgradeField(BoolField): def rules(): return [ - PythonLibrary.register_plugin_field(SkipPyUpgradeField), - PythonTests.register_plugin_field(SkipPyUpgradeField), + PythonSourceTarget.register_plugin_field(SkipPyUpgradeField), + PythonSourcesGeneratorTarget.register_plugin_field(SkipPyUpgradeField), + PythonTestTarget.register_plugin_field(SkipPyUpgradeField), + PythonTestsGeneratorTarget.register_plugin_field(SkipPyUpgradeField), ] diff --git a/src/python/pants/backend/python/lint/yapf/rules.py b/src/python/pants/backend/python/lint/yapf/rules.py index dbf56b57d8b..1fbddaab6fc 100644 --- a/src/python/pants/backend/python/lint/yapf/rules.py +++ b/src/python/pants/backend/python/lint/yapf/rules.py @@ -7,7 +7,7 @@ from pants.backend.python.lint.python_fmt import PythonFmtRequest from pants.backend.python.lint.yapf.skip_field import SkipYapfField from pants.backend.python.lint.yapf.subsystem import Yapf -from pants.backend.python.target_types import PythonSources +from pants.backend.python.target_types import PythonSourceField from pants.backend.python.util_rules import pex from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess from pants.core.goals.fmt import FmtResult @@ -25,9 +25,9 @@ @dataclass(frozen=True) class YapfFieldSet(FieldSet): - required_fields = (PythonSources,) + required_fields = (PythonSourceField,) - sources: PythonSources + sources: PythonSourceField @classmethod def opt_out(cls, tgt: Target) -> bool: diff --git a/src/python/pants/backend/python/lint/yapf/rules_integration_test.py b/src/python/pants/backend/python/lint/yapf/rules_integration_test.py index f05997a2cbf..cbc7f615bdc 100644 --- a/src/python/pants/backend/python/lint/yapf/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/yapf/rules_integration_test.py @@ -10,7 +10,7 @@ from pants.backend.python.lint.yapf.rules import rules as yapf_rules from pants.backend.python.lint.yapf.subsystem import Yapf from pants.backend.python.lint.yapf.subsystem import rules as yapf_subsystem_rules -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.core.goals.fmt import FmtResult from pants.core.goals.lint import LintResult, LintResults from pants.core.util_rules import config_files, source_files @@ -35,7 +35,7 @@ def rule_runner() -> RuleRunner: QueryRule(FmtResult, (YapfRequest,)), QueryRule(SourceFiles, (SourceFilesRequest,)), ], - target_types=[PythonLibrary], + target_types=[PythonSourcesGeneratorTarget], ) diff --git a/src/python/pants/backend/python/lint/yapf/skip_field.py b/src/python/pants/backend/python/lint/yapf/skip_field.py index b4c70b09278..9de02254583 100644 --- a/src/python/pants/backend/python/lint/yapf/skip_field.py +++ b/src/python/pants/backend/python/lint/yapf/skip_field.py @@ -1,7 +1,12 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.backend.python.target_types import PythonLibrary, PythonTests +from pants.backend.python.target_types import ( + PythonSourcesGeneratorTarget, + PythonSourceTarget, + PythonTestsGeneratorTarget, + PythonTestTarget, +) from pants.engine.target import BoolField @@ -13,6 +18,8 @@ class SkipYapfField(BoolField): def rules(): return [ - PythonLibrary.register_plugin_field(SkipYapfField), - PythonTests.register_plugin_field(SkipYapfField), + PythonSourceTarget.register_plugin_field(SkipYapfField), + PythonSourcesGeneratorTarget.register_plugin_field(SkipYapfField), + PythonTestTarget.register_plugin_field(SkipYapfField), + PythonTestsGeneratorTarget.register_plugin_field(SkipYapfField), ] diff --git a/src/python/pants/backend/python/mixed_interpreter_constraints/py_constraints_test.py b/src/python/pants/backend/python/mixed_interpreter_constraints/py_constraints_test.py index 4e6dbfdde23..daa5be6fb73 100644 --- a/src/python/pants/backend/python/mixed_interpreter_constraints/py_constraints_test.py +++ b/src/python/pants/backend/python/mixed_interpreter_constraints/py_constraints_test.py @@ -12,7 +12,10 @@ from pants.backend.python.mixed_interpreter_constraints.py_constraints import ( rules as py_constraints_rules, ) -from pants.backend.python.target_types import PythonLibrary, PythonTests +from pants.backend.python.target_types import ( + PythonSourcesGeneratorTarget, + PythonTestsGeneratorTarget, +) from pants.core.target_types import FileTarget from pants.testutil.rule_runner import GoalRuleResult, RuleRunner @@ -21,7 +24,7 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=(*py_constraints_rules(), *target_types_rules.rules()), - target_types=[FileTarget, PythonLibrary, PythonTests], + target_types=[FileTarget, PythonSourcesGeneratorTarget, PythonTestsGeneratorTarget], ) diff --git a/src/python/pants/backend/python/register.py b/src/python/pants/backend/python/register.py index 57f4813dd9a..bf06eeb0d3d 100644 --- a/src/python/pants/backend/python/register.py +++ b/src/python/pants/backend/python/register.py @@ -27,10 +27,10 @@ from pants.backend.python.target_types import ( PexBinary, PythonDistribution, - PythonLibrary, PythonRequirementsFile, PythonRequirementTarget, - PythonTests, + PythonSourcesGeneratorTarget, + PythonTestsGeneratorTarget, ) from pants.backend.python.util_rules import ( ancestor_files, @@ -86,8 +86,8 @@ def target_types(): return [ PexBinary, PythonDistribution, - PythonLibrary, + PythonSourcesGeneratorTarget, PythonRequirementTarget, PythonRequirementsFile, - PythonTests, + PythonTestsGeneratorTarget, ] diff --git a/src/python/pants/backend/python/subsystems/ipython_test.py b/src/python/pants/backend/python/subsystems/ipython_test.py index d6bf25dd0e2..43255250f30 100644 --- a/src/python/pants/backend/python/subsystems/ipython_test.py +++ b/src/python/pants/backend/python/subsystems/ipython_test.py @@ -8,7 +8,7 @@ from pants.backend.python.goals.lockfile import PythonLockfileRequest from pants.backend.python.subsystems.ipython import IPythonLockfileSentinel from pants.backend.python.subsystems.ipython import rules as subsystem_rules -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.core.target_types import GenericTarget from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -17,7 +17,7 @@ def test_setup_lockfile_interpreter_constraints() -> None: rule_runner = RuleRunner( rules=[*subsystem_rules(), QueryRule(PythonLockfileRequest, [IPythonLockfileSentinel])], - target_types=[PythonLibrary, GenericTarget], + target_types=[PythonSourcesGeneratorTarget, GenericTarget], ) global_constraint = "==3.9.*" diff --git a/src/python/pants/backend/python/subsystems/pytest.py b/src/python/pants/backend/python/subsystems/pytest.py index 97fd9779936..146de9c092e 100644 --- a/src/python/pants/backend/python/subsystems/pytest.py +++ b/src/python/pants/backend/python/subsystems/pytest.py @@ -18,7 +18,7 @@ ConsoleScript, PythonResolveField, PythonTestsExtraEnvVars, - PythonTestsSources, + PythonTestSourceField, PythonTestsTimeout, SkipPythonTestsField, format_invalid_requirement_string_error, @@ -28,12 +28,7 @@ from pants.core.goals.test import RuntimePackageDependenciesField, TestFieldSet from pants.core.util_rules.config_files import ConfigFilesRequest from pants.engine.rules import Get, MultiGet, collect_rules, rule -from pants.engine.target import ( - Target, - TransitiveTargets, - TransitiveTargetsRequest, - UnexpandedTargets, -) +from pants.engine.target import Target, Targets, TransitiveTargets, TransitiveTargetsRequest from pants.engine.unions import UnionRule from pants.option.custom_types import shell_str from pants.python.python_setup import PythonSetup @@ -44,9 +39,9 @@ @dataclass(frozen=True) class PythonTestFieldSet(TestFieldSet): - required_fields = (PythonTestsSources,) + required_fields = (PythonTestSourceField,) - sources: PythonTestsSources + sources: PythonTestSourceField timeout: PythonTestsTimeout runtime_package_dependencies: RuntimePackageDependenciesField extra_env_vars: PythonTestsExtraEnvVars @@ -56,8 +51,8 @@ class PythonTestFieldSet(TestFieldSet): def opt_out(cls, tgt: Target) -> bool: if tgt.get(SkipPythonTestsField).value: return True - if not tgt.address.is_file_target: - return False + # TODO: Replace this by having `python_tests` generate `python_source` targets for these + # files. file_name = PurePath(tgt.address.filename) return file_name.name == "conftest.py" or file_name.suffix == ".pyi" @@ -241,14 +236,14 @@ async def setup_pytest_lockfile( # Even though we run each python_tests target in isolation, we need a single lockfile that # works with them all (and their transitive deps). # - # This first computes the constraints for each individual `python_tests` target + # This first computes the constraints for each individual `python_test` target # (which will AND across each target in the closure). Then, it ORs all unique resulting # interpreter constraints. The net effect is that every possible Python interpreter used will # be covered. - all_build_targets = await Get(UnexpandedTargets, AddressSpecs([DescendantAddresses("")])) + all_tgts = await Get(Targets, AddressSpecs([DescendantAddresses("")])) transitive_targets_per_test = await MultiGet( Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address])) - for tgt in all_build_targets + for tgt in all_tgts if PythonTestFieldSet.is_applicable(tgt) ) unique_constraints = { diff --git a/src/python/pants/backend/python/subsystems/pytest_test.py b/src/python/pants/backend/python/subsystems/pytest_test.py index 4bad58bef35..5e6269a1e19 100644 --- a/src/python/pants/backend/python/subsystems/pytest_test.py +++ b/src/python/pants/backend/python/subsystems/pytest_test.py @@ -7,10 +7,14 @@ import pytest +from pants.backend.python import target_types_rules from pants.backend.python.goals.lockfile import PythonLockfileRequest from pants.backend.python.subsystems.pytest import PyTest, PytestLockfileSentinel from pants.backend.python.subsystems.pytest import rules as subsystem_rules -from pants.backend.python.target_types import PythonLibrary, PythonTests +from pants.backend.python.target_types import ( + PythonSourcesGeneratorTarget, + PythonTestsGeneratorTarget, +) from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.core.target_types import GenericTarget from pants.option.ranked_value import Rank, RankedValue @@ -20,25 +24,30 @@ def test_setup_lockfile_interpreter_constraints() -> None: rule_runner = RuleRunner( - rules=[*subsystem_rules(), QueryRule(PythonLockfileRequest, [PytestLockfileSentinel])], - target_types=[PythonLibrary, PythonTests, GenericTarget], + rules=[ + *subsystem_rules(), + *target_types_rules.rules(), + QueryRule(PythonLockfileRequest, [PytestLockfileSentinel]), + ], + target_types=[PythonSourcesGeneratorTarget, PythonTestsGeneratorTarget, GenericTarget], ) global_constraint = "==3.9.*" rule_runner.set_options( ["--pytest-lockfile=lockfile.txt"], env={"PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS": f"['{global_constraint}']"}, + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, ) def assert_ics(build_file: str, expected: list[str]) -> None: - rule_runner.write_files({"project/BUILD": build_file}) + rule_runner.write_files({"project/BUILD": build_file, "project/f_test.py": ""}) lockfile_request = rule_runner.request(PythonLockfileRequest, [PytestLockfileSentinel()]) assert lockfile_request.interpreter_constraints == InterpreterConstraints(expected) assert_ics("python_tests()", [global_constraint]) assert_ics("python_tests(interpreter_constraints=['==2.7.*'])", ["==2.7.*"]) assert_ics( - "python_tests(interpreter_constraints=['==2.7.*', '==3.5.*'])", ["==2.7.*", "==3.5.*"] + "python_tests(interpreter_constraints=['==2.7.*', '==3.8.*'])", ["==2.7.*", "==3.8.*"] ) # If no Python targets in repo, fall back to global python-setup constraints. @@ -52,7 +61,7 @@ def assert_ics(build_file: str, expected: list[str]) -> None: dedent( """\ python_tests(name='a', interpreter_constraints=['==2.7.*']) - python_tests(name='b', interpreter_constraints=['==3.5.*'], skip_tests=True) + python_tests(name='b', interpreter_constraints=['==3.8.*'], skip_tests=True) """ ), ["==2.7.*"], @@ -64,19 +73,19 @@ def assert_ics(build_file: str, expected: list[str]) -> None: dedent( """\ python_tests(name='a', interpreter_constraints=['==2.7.*']) - python_tests(name='b', interpreter_constraints=['==3.5.*']) + python_tests(name='b', interpreter_constraints=['==3.8.*']) """ ), - ["==2.7.*", "==3.5.*"], + ["==2.7.*", "==3.8.*"], ) assert_ics( dedent( """\ - python_tests(name='a', interpreter_constraints=['==2.7.*', '==3.5.*']) - python_tests(name='b', interpreter_constraints=['>=3.5']) + python_tests(name='a', interpreter_constraints=['==2.7.*', '==3.8.*']) + python_tests(name='b', interpreter_constraints=['>=3.8']) """ ), - ["==2.7.*", "==3.5.*", ">=3.5"], + ["==2.7.*", "==3.8.*", ">=3.8"], ) assert_ics( dedent( diff --git a/src/python/pants/backend/python/subsystems/setuptools_test.py b/src/python/pants/backend/python/subsystems/setuptools_test.py index 8d9a1bd9987..686682a8eb3 100644 --- a/src/python/pants/backend/python/subsystems/setuptools_test.py +++ b/src/python/pants/backend/python/subsystems/setuptools_test.py @@ -9,7 +9,7 @@ from pants.backend.python.macros.python_artifact import PythonArtifact from pants.backend.python.subsystems import setuptools from pants.backend.python.subsystems.setuptools import SetuptoolsLockfileSentinel -from pants.backend.python.target_types import PythonDistribution, PythonLibrary +from pants.backend.python.target_types import PythonDistribution, PythonSourcesGeneratorTarget from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -20,7 +20,7 @@ def test_setup_lockfile_interpreter_constraints() -> None: *setuptools.rules(), QueryRule(PythonLockfileRequest, [SetuptoolsLockfileSentinel]), ], - target_types=[PythonLibrary, PythonDistribution], + target_types=[PythonSourcesGeneratorTarget, PythonDistribution], objects={"setup_py": PythonArtifact}, ) diff --git a/src/python/pants/backend/python/target_types.py b/src/python/pants/backend/python/target_types.py index 89f6e6525f5..24e400a67c7 100644 --- a/src/python/pants/backend/python/target_types.py +++ b/src/python/pants/backend/python/target_types.py @@ -49,6 +49,7 @@ ProvidesField, ScalarField, SecondaryOwnerMixin, + SingleSourceField, StringField, StringSequenceField, Target, @@ -71,11 +72,15 @@ # ----------------------------------------------------------------------------------------------- -class PythonSources(MultipleSourcesField): +class PythonSourceField(SingleSourceField): # Note that Python scripts often have no file ending. expected_file_extensions = ("", ".py", ".pyi") +class PythonGeneratingSourcesBase(MultipleSourcesField): + expected_file_extensions = ("", ".py", ".pyi") + + class InterpreterConstraintsField(StringSequenceField): alias = "interpreter_constraints" help = ( @@ -542,20 +547,12 @@ def validate(self) -> None: # ----------------------------------------------------------------------------------------------- -# `python_tests` target +# `python_test` and `python_tests` targets # ----------------------------------------------------------------------------------------------- -class PythonTestsSources(PythonSources): - default = ( - "test_*.py", - "*_test.py", - "tests.py", - "conftest.py", - "test_*.pyi", - "*_test.pyi", - "tests.pyi", - ) +class PythonTestSourceField(PythonSourceField): + pass class PythonTestsDependencies(Dependencies): @@ -610,19 +607,44 @@ class SkipPythonTestsField(BoolField): help = "If true, don't run this target's tests." -class PythonTests(Target): - alias = "python_tests" - core_fields = ( - *COMMON_TARGET_FIELDS, - InterpreterConstraintsField, - PythonResolveField, - PythonTestsSources, - PythonTestsDependencies, - PythonTestsTimeout, - RuntimePackageDependenciesField, - PythonTestsExtraEnvVars, - SkipPythonTestsField, +_PYTHON_TEST_COMMON_FIELDS = ( + *COMMON_TARGET_FIELDS, + InterpreterConstraintsField, + PythonResolveField, + PythonTestsDependencies, + PythonTestsTimeout, + RuntimePackageDependenciesField, + PythonTestsExtraEnvVars, + SkipPythonTestsField, +) + + +class PythonTestTarget(Target): + alias = "python_tests" # TODO: rename to `python_test` + core_fields = (*_PYTHON_TEST_COMMON_FIELDS, PythonTestSourceField) + help = ( + "A single Python test file, written in either Pytest style or unittest style.\n\n" + "All test util code, including `conftest.py`, should go into a dedicated `python_source` " + "target and then be included in the `dependencies` field.\n\n" + f"See {doc_url('python-test-goal')}" ) + + +class PythonTestsGeneratingSourcesField(PythonGeneratingSourcesBase): + default = ( + "test_*.py", + "*_test.py", + "tests.py", + "conftest.py", + "test_*.pyi", + "*_test.pyi", + "tests.pyi", + ) + + +class PythonTestsGeneratorTarget(Target): + alias = "python_tests" + core_fields = (*_PYTHON_TEST_COMMON_FIELDS, PythonTestsGeneratingSourcesField) help = ( "Python tests, written in either Pytest style or unittest style.\n\nAll test util code, " "other than `conftest.py`, should go into a dedicated `python_sources()` target and then " @@ -631,21 +653,34 @@ class PythonTests(Target): # ----------------------------------------------------------------------------------------------- -# `python_sources` target +# `python_source` and `python_sources` targets # ----------------------------------------------------------------------------------------------- -class PythonLibrarySources(PythonSources): - default = ("*.py", "*.pyi") + tuple(f"!{pat}" for pat in PythonTestsSources.default) +class PythonSourceTarget(Target): + alias = "python_sources" # TODO: rename to `python_source` + core_fields = ( + *COMMON_TARGET_FIELDS, + InterpreterConstraintsField, + Dependencies, + PythonSourceField, + ) + help = "A single Python source file." + + +class PythonSourcesGeneratingSourcesField(PythonGeneratingSourcesBase): + default = ("*.py", "*.pyi") + tuple( + f"!{pat}" for pat in PythonTestsGeneratingSourcesField.default + ) -class PythonLibrary(Target): +class PythonSourcesGeneratorTarget(Target): alias = "python_sources" core_fields = ( *COMMON_TARGET_FIELDS, InterpreterConstraintsField, Dependencies, - PythonLibrarySources, + PythonSourcesGeneratingSourcesField, ) help = ( "Python source code.\n\nA `python_sources` does not necessarily correspond to a " diff --git a/src/python/pants/backend/python/target_types_rules.py b/src/python/pants/backend/python/target_types_rules.py index 6e11ab278e0..4f97831e2af 100644 --- a/src/python/pants/backend/python/target_types_rules.py +++ b/src/python/pants/backend/python/target_types_rules.py @@ -25,11 +25,13 @@ PythonDistributionDependencies, PythonDistributionEntryPoint, PythonDistributionEntryPointsField, - PythonLibrary, - PythonLibrarySources, PythonProvidesField, - PythonTests, - PythonTestsSources, + PythonSourcesGeneratingSourcesField, + PythonSourcesGeneratorTarget, + PythonSourceTarget, + PythonTestsGeneratingSourcesField, + PythonTestsGeneratorTarget, + PythonTestTarget, ResolvedPexEntryPoint, ResolvedPythonDistributionEntryPoints, ResolvePexEntryPointRequest, @@ -69,7 +71,7 @@ class GenerateTargetsFromPythonTests(GenerateTargetsRequest): - generate_from = PythonTests + generate_from = PythonTestsGeneratorTarget @rule @@ -78,35 +80,37 @@ async def generate_targets_from_python_tests( python_infer: PythonInferSubsystem, union_membership: UnionMembership, ) -> GeneratedTargets: - paths = await Get(SourcesPaths, SourcesPathsRequest(request.generator[PythonTestsSources])) + paths = await Get( + SourcesPaths, SourcesPathsRequest(request.generator[PythonTestsGeneratingSourcesField]) + ) return generate_file_level_targets( - PythonTests, + PythonTestTarget, request.generator, paths.files, union_membership, add_dependencies_on_all_siblings=not python_infer.imports, - use_source_field=False, ) -class GenerateTargetsFromPythonLibrary(GenerateTargetsRequest): - generate_from = PythonLibrary +class GenerateTargetsFromPythonSources(GenerateTargetsRequest): + generate_from = PythonSourcesGeneratorTarget @rule async def generate_targets_from_python_sources( - request: GenerateTargetsFromPythonLibrary, + request: GenerateTargetsFromPythonSources, python_infer: PythonInferSubsystem, union_membership: UnionMembership, ) -> GeneratedTargets: - paths = await Get(SourcesPaths, SourcesPathsRequest(request.generator[PythonLibrarySources])) + paths = await Get( + SourcesPaths, SourcesPathsRequest(request.generator[PythonSourcesGeneratingSourcesField]) + ) return generate_file_level_targets( - PythonLibrary, + PythonSourceTarget, request.generator, paths.files, union_membership, add_dependencies_on_all_siblings=not python_infer.imports, - use_source_field=False, ) @@ -416,7 +420,7 @@ def rules(): *collect_rules(), *import_rules(), UnionRule(GenerateTargetsRequest, GenerateTargetsFromPythonTests), - UnionRule(GenerateTargetsRequest, GenerateTargetsFromPythonLibrary), + UnionRule(GenerateTargetsRequest, GenerateTargetsFromPythonSources), UnionRule(InjectDependenciesRequest, InjectPexBinaryEntryPointDependency), UnionRule(InjectDependenciesRequest, InjectPythonDistributionDependencies), ) diff --git a/src/python/pants/backend/python/target_types_test.py b/src/python/pants/backend/python/target_types_test.py index b6469ad7ed6..8764acf1e5b 100644 --- a/src/python/pants/backend/python/target_types_test.py +++ b/src/python/pants/backend/python/target_types_test.py @@ -25,9 +25,9 @@ PexScriptField, PythonDistribution, PythonDistributionDependencies, - PythonLibrary, PythonRequirementsField, PythonRequirementTarget, + PythonSourcesGeneratorTarget, PythonTestsTimeout, ResolvedPexEntryPoint, ResolvePexEntryPointRequest, @@ -195,7 +195,7 @@ def test_inject_pex_binary_entry_point_dependency(caplog) -> None: *import_rules(), QueryRule(InjectedDependencies, [InjectPexBinaryEntryPointDependency]), ], - target_types=[PexBinary, PythonRequirementTarget, PythonLibrary], + target_types=[PexBinary, PythonRequirementTarget, PythonSourcesGeneratorTarget], ) rule_runner.write_files( { @@ -391,7 +391,12 @@ def test_inject_python_distribution_dependencies() -> None: *python_sources.rules(), QueryRule(InjectedDependencies, [InjectPythonDistributionDependencies]), ], - target_types=[PythonDistribution, PythonRequirementTarget, PythonLibrary, PexBinary], + target_types=[ + PythonDistribution, + PythonRequirementTarget, + PythonSourcesGeneratorTarget, + PexBinary, + ], objects={"setup_py": PythonArtifact}, ) rule_runner.add_to_build_file( diff --git a/src/python/pants/backend/python/typecheck/mypy/rules.py b/src/python/pants/backend/python/typecheck/mypy/rules.py index b20a47d249f..70d7ac9cb88 100644 --- a/src/python/pants/backend/python/typecheck/mypy/rules.py +++ b/src/python/pants/backend/python/typecheck/mypy/rules.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import Iterable, Optional, Tuple -from pants.backend.python.target_types import PythonSources +from pants.backend.python.target_types import PythonSourceField from pants.backend.python.typecheck.mypy.skip_field import SkipMyPyField from pants.backend.python.typecheck.mypy.subsystem import ( MyPy, @@ -42,9 +42,9 @@ @dataclass(frozen=True) class MyPyFieldSet(FieldSet): - required_fields = (PythonSources,) + required_fields = (PythonSourceField,) - sources: PythonSources + sources: PythonSourceField @classmethod def opt_out(cls, tgt: Target) -> bool: @@ -123,7 +123,8 @@ async def mypy_typecheck_partition( closure_sources_get = Get(PythonSourceFiles, PythonSourceFilesRequest(partition.closure)) roots_sources_get = Get( - SourceFiles, SourceFilesRequest(tgt.get(PythonSources) for tgt in partition.root_targets) + SourceFiles, + SourceFilesRequest(tgt.get(PythonSourceField) for tgt in partition.root_targets), ) # See `requirements_venv_pex` for how this will get wrapped in a `VenvPex`. diff --git a/src/python/pants/backend/python/typecheck/mypy/rules_integration_test.py b/src/python/pants/backend/python/typecheck/mypy/rules_integration_test.py index e2b56678c9f..5e67245d9a1 100644 --- a/src/python/pants/backend/python/typecheck/mypy/rules_integration_test.py +++ b/src/python/pants/backend/python/typecheck/mypy/rules_integration_test.py @@ -14,7 +14,7 @@ from pants.backend.codegen.protobuf.target_types import ProtobufSourceTarget from pants.backend.python import target_types_rules from pants.backend.python.dependency_inference import rules as dependency_inference_rules -from pants.backend.python.target_types import PythonLibrary, PythonRequirementTarget +from pants.backend.python.target_types import PythonRequirementTarget, PythonSourcesGeneratorTarget from pants.backend.python.typecheck.mypy.rules import ( MyPyFieldSet, MyPyRequest, @@ -51,7 +51,7 @@ def rule_runner() -> RuleRunner: *target_types_rules.rules(), QueryRule(CheckResults, (MyPyRequest,)), ], - target_types=[PythonLibrary, PythonRequirementTarget], + target_types=[PythonSourcesGeneratorTarget, PythonRequirementTarget], ) @@ -257,10 +257,12 @@ def test_thirdparty_plugin(rule_runner: RuleRunner) -> None: ), } ) - tgt = rule_runner.get_target(Address(PACKAGE)) result = run_mypy( rule_runner, - [tgt], + [ + rule_runner.get_target(Address(PACKAGE, relative_file_path="app.py")), + rule_runner.get_target(Address(PACKAGE, relative_file_path="settings.py")), + ], extra_args=[ "--mypy-extra-requirements=django-stubs==1.8.0", "--mypy-extra-type-stubs=django-stubs==1.8.0", diff --git a/src/python/pants/backend/python/typecheck/mypy/skip_field.py b/src/python/pants/backend/python/typecheck/mypy/skip_field.py index 02a19c3ee7f..27119cd4cd8 100644 --- a/src/python/pants/backend/python/typecheck/mypy/skip_field.py +++ b/src/python/pants/backend/python/typecheck/mypy/skip_field.py @@ -1,7 +1,12 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.backend.python.target_types import PythonLibrary, PythonTests +from pants.backend.python.target_types import ( + PythonSourcesGeneratorTarget, + PythonSourceTarget, + PythonTestsGeneratorTarget, + PythonTestTarget, +) from pants.engine.target import BoolField @@ -13,6 +18,8 @@ class SkipMyPyField(BoolField): def rules(): return [ - PythonLibrary.register_plugin_field(SkipMyPyField), - PythonTests.register_plugin_field(SkipMyPyField), + PythonSourceTarget.register_plugin_field(SkipMyPyField), + PythonSourcesGeneratorTarget.register_plugin_field(SkipMyPyField), + PythonTestTarget.register_plugin_field(SkipMyPyField), + PythonTestsGeneratorTarget.register_plugin_field(SkipMyPyField), ] diff --git a/src/python/pants/backend/python/typecheck/mypy/subsystem.py b/src/python/pants/backend/python/typecheck/mypy/subsystem.py index dc49c4b9bfa..9c4c56c3f85 100644 --- a/src/python/pants/backend/python/typecheck/mypy/subsystem.py +++ b/src/python/pants/backend/python/typecheck/mypy/subsystem.py @@ -10,7 +10,11 @@ from pants.backend.python.goals.lockfile import PythonLockfileRequest, PythonToolLockfileSentinel from pants.backend.python.subsystems.python_tool_base import PythonToolBase -from pants.backend.python.target_types import ConsoleScript, PythonRequirementsField, PythonSources +from pants.backend.python.target_types import ( + ConsoleScript, + PythonRequirementsField, + PythonSourceField, +) from pants.backend.python.typecheck.mypy.skip_field import SkipMyPyField from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.backend.python.util_rules.pex import PexRequirements @@ -26,9 +30,9 @@ from pants.engine.target import ( FieldSet, Target, + Targets, TransitiveTargets, TransitiveTargetsRequest, - UnexpandedTargets, ) from pants.engine.unions import UnionRule from pants.option.custom_types import file_option, shell_str, target_option @@ -42,13 +46,13 @@ @dataclass(frozen=True) class MyPyFieldSet(FieldSet): - required_fields = (PythonSources,) + required_fields = (PythonSourceField,) - sources: PythonSources + sources: PythonSourceField @classmethod def opt_out(cls, tgt: Target) -> bool: - return tgt.get(SkipMyPyField).value and not tgt.address.is_file_target + return tgt.get(SkipMyPyField).value # -------------------------------------------------------------------------------------- @@ -291,10 +295,10 @@ async def setup_mypy_lockfile( constraints = mypy.interpreter_constraints if mypy.options.is_default("interpreter_constraints"): - all_build_targets = await Get(UnexpandedTargets, AddressSpecs([DescendantAddresses("")])) + all_tgts = await Get(Targets, AddressSpecs([DescendantAddresses("")])) all_transitive_targets = await MultiGet( Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address])) - for tgt in all_build_targets + for tgt in all_tgts if MyPyFieldSet.is_applicable(tgt) ) unique_constraints = { diff --git a/src/python/pants/backend/python/typecheck/mypy/subsystem_test.py b/src/python/pants/backend/python/typecheck/mypy/subsystem_test.py index 523b8c3ad24..cbee9ec3f33 100644 --- a/src/python/pants/backend/python/typecheck/mypy/subsystem_test.py +++ b/src/python/pants/backend/python/typecheck/mypy/subsystem_test.py @@ -7,8 +7,9 @@ import pytest +from pants.backend.python import target_types_rules from pants.backend.python.goals.lockfile import PythonLockfileRequest -from pants.backend.python.target_types import PythonLibrary, PythonRequirementTarget +from pants.backend.python.target_types import PythonRequirementTarget, PythonSourcesGeneratorTarget from pants.backend.python.typecheck.mypy import skip_field, subsystem from pants.backend.python.typecheck.mypy.subsystem import ( MyPy, @@ -33,11 +34,12 @@ def rule_runner() -> RuleRunner: *skip_field.rules(), *config_files.rules(), *python_sources.rules(), + *target_types_rules.rules(), QueryRule(MyPyConfigFile, []), QueryRule(MyPyFirstPartyPlugins, []), QueryRule(PythonLockfileRequest, [MyPyLockfileSentinel]), ], - target_types=[PythonLibrary, PythonRequirementTarget, GenericTarget], + target_types=[PythonSourcesGeneratorTarget, PythonRequirementTarget, GenericTarget], ) @@ -48,7 +50,8 @@ def test_warn_if_python_version_configured(rule_runner: RuleRunner, caplog) -> N def maybe_assert_configured(*, has_config: bool, args: list[str], warning: str = "") -> None: rule_runner.set_options( - [f"--mypy-args={repr(args)}", f"--mypy-config-discovery={has_config}"] + [f"--mypy-args={repr(args)}", f"--mypy-config-discovery={has_config}"], + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, ) result = rule_runner.request(MyPyConfigFile, []) @@ -121,7 +124,8 @@ def test_first_party_plugins(rule_runner: RuleRunner) -> None: [ "--source-root-patterns=mypy-plugins", "--mypy-source-plugins=mypy-plugins/plugin.py", - ] + ], + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, ) first_party_plugins = rule_runner.request(MyPyFirstPartyPlugins, []) assert first_party_plugins.requirement_strings == FrozenOrderedSet(["ansicolors", "mypy==0.81"]) @@ -148,10 +152,11 @@ def assert_lockfile_request( extra_expected_requirements: list[str] | None = None, extra_args: list[str] | None = None, ) -> None: - rule_runner.write_files({"project/BUILD": build_file}) + rule_runner.write_files({"project/BUILD": build_file, "project/f.py": ""}) rule_runner.set_options( ["--mypy-lockfile=lockfile.txt", *(extra_args or [])], env={"PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS": f"['{global_constraint}']"}, + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, ) lockfile_request = rule_runner.request(PythonLockfileRequest, [MyPyLockfileSentinel()]) assert lockfile_request.interpreter_constraints == InterpreterConstraints(expected_ics) @@ -165,28 +170,19 @@ def assert_lockfile_request( # If all code is Py38+, use those constraints. Otherwise, use subsystem constraints. assert_lockfile_request("python_sources()", [global_constraint]) - assert_lockfile_request("python_sources(interpreter_constraints=['==3.10.*'])", ["==3.10.*"]) + assert_lockfile_request("python_sources(interpreter_constraints=['==3.9.*'])", ["==3.9.*"]) assert_lockfile_request( - "python_sources(interpreter_constraints=['==3.8.*', '==3.10.*'])", ["==3.8.*", "==3.10.*"] + "python_sources(interpreter_constraints=['==3.8.*', '==3.9.*'])", ["==3.8.*", "==3.9.*"] ) assert_lockfile_request( - "python_sources(interpreter_constraints=['==3.6.*'])", - MyPy.default_interpreter_constraints, - ) - assert_lockfile_request( - dedent( - """\ - python_sources(name='t1', interpreter_constraints=['==3.6.*']) - python_sources(name='t2', interpreter_constraints=['==3.8.*']) - """ - ), + "python_sources(interpreter_constraints=['>=3.6'])", MyPy.default_interpreter_constraints, ) assert_lockfile_request( dedent( """\ - python_sources(name='t1', interpreter_constraints=['==3.6.*', '>=3.8']) + python_sources(name='t1', interpreter_constraints=['>=3.6']) python_sources(name='t2', interpreter_constraints=['==3.8.*']) """ ), @@ -201,7 +197,7 @@ def assert_lockfile_request( dedent( """\ python_sources(name='a', interpreter_constraints=['==3.8.*']) - python_sources(name='b', interpreter_constraints=['==3.5.*'], skip_mypy=True) + python_sources(name='b', interpreter_constraints=['>=3.6.*'], skip_mypy=True) """ ), ["==3.8.*"], @@ -221,7 +217,7 @@ def assert_lockfile_request( assert_lockfile_request( dedent( """\ - python_sources(name='lib1', interpreter_constraints=['==2.7.*', '==3.6.*'], skip_mypy=True) + python_sources(name='lib1', interpreter_constraints=['==2.7.*', '>=3.6'], skip_mypy=True) python_sources(name='lib2', dependencies=[":lib1"], interpreter_constraints=['==2.7.*']) python_sources(name='lib3', interpreter_constraints=['>=3.8'], skip_mypy=True) diff --git a/src/python/pants/backend/python/util_rules/local_dists_test.py b/src/python/pants/backend/python/util_rules/local_dists_test.py index 713e56ff5f1..24fd5c34f25 100644 --- a/src/python/pants/backend/python/util_rules/local_dists_test.py +++ b/src/python/pants/backend/python/util_rules/local_dists_test.py @@ -14,7 +14,7 @@ from pants.backend.python.goals.setup_py import rules as setup_py_rules from pants.backend.python.macros.python_artifact import PythonArtifact from pants.backend.python.subsystems.setuptools import rules as setuptools_rules -from pants.backend.python.target_types import PythonDistribution, PythonLibrary +from pants.backend.python.target_types import PythonDistribution, PythonSourcesGeneratorTarget from pants.backend.python.util_rules import local_dists from pants.backend.python.util_rules.local_dists import LocalDistsPex, LocalDistsPexRequest from pants.backend.python.util_rules.python_sources import PythonSourceFiles @@ -34,7 +34,7 @@ def rule_runner() -> RuleRunner: *target_types_rules.rules(), QueryRule(LocalDistsPex, (LocalDistsPexRequest,)), ], - target_types=[PythonLibrary, PythonDistribution], + target_types=[PythonSourcesGeneratorTarget, PythonDistribution], objects={"python_artifact": PythonArtifact}, ) diff --git a/src/python/pants/backend/python/util_rules/pex_from_targets_test.py b/src/python/pants/backend/python/util_rules/pex_from_targets_test.py index 26637a3f4b8..3f9fe332785 100644 --- a/src/python/pants/backend/python/util_rules/pex_from_targets_test.py +++ b/src/python/pants/backend/python/util_rules/pex_from_targets_test.py @@ -14,7 +14,7 @@ import pytest from _pytest.tmpdir import TempPathFactory -from pants.backend.python.target_types import PythonLibrary, PythonRequirementTarget +from pants.backend.python.target_types import PythonRequirementTarget, PythonSourcesGeneratorTarget from pants.backend.python.util_rules import pex_from_targets from pants.backend.python.util_rules.pex import Pex, PexPlatforms, PexRequest, PexRequirements from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest @@ -32,7 +32,7 @@ def rule_runner() -> RuleRunner: *pex_from_targets.rules(), QueryRule(PexRequest, (PexFromTargetsRequest,)), ], - target_types=[PythonLibrary, PythonRequirementTarget], + target_types=[PythonSourcesGeneratorTarget, PythonRequirementTarget], ) diff --git a/src/python/pants/backend/python/util_rules/python_sources.py b/src/python/pants/backend/python/util_rules/python_sources.py index f055c49cfec..dc45a31b88a 100644 --- a/src/python/pants/backend/python/util_rules/python_sources.py +++ b/src/python/pants/backend/python/util_rules/python_sources.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from typing import Iterable, List, Tuple, Type -from pants.backend.python.target_types import PythonSources +from pants.backend.python.target_types import PythonSourceField from pants.backend.python.util_rules import ancestor_files from pants.backend.python.util_rules.ancestor_files import AncestorFiles, AncestorFilesRequest from pants.core.target_types import FileSourceField, ResourceSourceField @@ -67,7 +67,7 @@ def __init__( @property def valid_sources_types(self) -> Tuple[Type[SourcesField], ...]: - types: List[Type[SourcesField]] = [PythonSources] + types: List[Type[SourcesField]] = [PythonSourceField] if self.include_resources: types.append(ResourceSourceField) if self.include_files: @@ -105,9 +105,9 @@ async def prepare_python_sources( python_and_resources_targets = [] codegen_targets = [] for tgt in request.targets: - if tgt.has_field(PythonSources) or tgt.has_field(ResourceSourceField): + if tgt.has_field(PythonSourceField) or tgt.has_field(ResourceSourceField): python_and_resources_targets.append(tgt) - elif tgt.get(SourcesField).can_generate(PythonSources, union_membership) or tgt.get( + elif tgt.get(SourcesField).can_generate(PythonSourceField, union_membership) or tgt.get( SourcesField ).can_generate(ResourceSourceField, union_membership): codegen_targets.append(tgt) diff --git a/src/python/pants/backend/python/util_rules/python_sources_test.py b/src/python/pants/backend/python/util_rules/python_sources_test.py index 3df99b0cb25..9b7d71ccd28 100644 --- a/src/python/pants/backend/python/util_rules/python_sources_test.py +++ b/src/python/pants/backend/python/util_rules/python_sources_test.py @@ -15,7 +15,7 @@ ) from pants.backend.codegen.protobuf.python.rules import rules as protobuf_rules from pants.backend.codegen.protobuf.target_types import ProtobufSourceTarget -from pants.backend.python.target_types import PythonSources +from pants.backend.python.target_types import PythonSourceField from pants.backend.python.util_rules.python_sources import ( PythonSourceFiles, PythonSourceFilesRequest, @@ -24,18 +24,18 @@ from pants.backend.python.util_rules.python_sources import rules as python_sources_rules from pants.core.target_types import FileTarget, ResourceTarget from pants.engine.addresses import Address -from pants.engine.target import MultipleSourcesField, SingleSourceField, Target +from pants.engine.target import SingleSourceField, Target from pants.testutil.rule_runner import QueryRule, RuleRunner class PythonTarget(Target): alias = "python_target" - core_fields = (PythonSources,) + core_fields = (PythonSourceField,) class NonPythonTarget(Target): alias = "non_python_target" - core_fields = (MultipleSourcesField,) + core_fields = (SingleSourceField,) @pytest.fixture @@ -56,24 +56,12 @@ def rule_runner() -> RuleRunner: def create_target( rule_runner: RuleRunner, parent_directory: str, - files: str | list[str], + file: str, target_cls: type[Target] = PythonTarget, ) -> Target: - rule_runner.write_files( - { - os.path.join(parent_directory, f): "" - for f in (files if isinstance(files, list) else (files,)) - } - ) + rule_runner.write_files({os.path.join(parent_directory, file): ""}) address = Address(parent_directory, target_name="target") - return target_cls( - ( - {MultipleSourcesField.alias: files} - if isinstance(files, list) - else {SingleSourceField.alias: files} - ), - address, - ) + return target_cls({SingleSourceField.alias: file}, address) def get_stripped_sources( @@ -125,10 +113,10 @@ def get_unstripped_sources( def test_filters_out_irrelevant_targets(rule_runner: RuleRunner) -> None: targets = [ - create_target(rule_runner, "src/python", ["p.py"], PythonTarget), + create_target(rule_runner, "src/python", "p.py", PythonTarget), create_target(rule_runner, "src/python", "f.txt", FileTarget), create_target(rule_runner, "src/python", "r.txt", ResourceTarget), - create_target(rule_runner, "src/python", ["j.java"], NonPythonTarget), + create_target(rule_runner, "src/python", "j.java", NonPythonTarget), ] def assert_stripped( @@ -187,19 +175,19 @@ def assert_unstripped( def test_top_level_source_root(rule_runner: RuleRunner) -> None: - targets = [create_target(rule_runner, "", ["f1.py", "f2.py"])] + targets = [create_target(rule_runner, "", "f.py")] stripped_result = get_stripped_sources(rule_runner, targets, source_roots=["/"]) - assert stripped_result.stripped_source_files.snapshot.files == ("f1.py", "f2.py") + assert stripped_result.stripped_source_files.snapshot.files == ("f.py",) unstripped_result = get_unstripped_sources(rule_runner, targets, source_roots=["/"]) - assert unstripped_result.source_files.snapshot.files == ("f1.py", "f2.py") + assert unstripped_result.source_files.snapshot.files == ("f.py",) assert unstripped_result.source_roots == (".",) def test_files_not_used_for_source_roots(rule_runner: RuleRunner) -> None: targets = [ - create_target(rule_runner, "src/py", ["f.py"], PythonTarget), + create_target(rule_runner, "src/py", "f.py", PythonTarget), create_target(rule_runner, "src/files", "f.txt", FileTarget), ] assert get_unstripped_sources( diff --git a/src/python/pants/backend/shell/shunit2_test_runner_integration_test.py b/src/python/pants/backend/shell/shunit2_test_runner_integration_test.py index f0da7070750..92ab8d7d7d2 100644 --- a/src/python/pants/backend/shell/shunit2_test_runner_integration_test.py +++ b/src/python/pants/backend/shell/shunit2_test_runner_integration_test.py @@ -9,7 +9,7 @@ import pytest from pants.backend.python.goals import package_pex_binary -from pants.backend.python.target_types import PexBinary, PythonLibrary +from pants.backend.python.target_types import PexBinary, PythonSourcesGeneratorTarget from pants.backend.python.target_types_rules import rules as python_target_type_rules from pants.backend.python.util_rules import pex_from_targets from pants.backend.shell import shunit2_test_runner @@ -58,7 +58,7 @@ def rule_runner() -> RuleRunner: target_types=[ ShellSourcesGeneratorTarget, Shunit2TestsGeneratorTarget, - PythonLibrary, + PythonSourcesGeneratorTarget, PexBinary, ], ) diff --git a/src/python/pants/core/goals/publish_test.py b/src/python/pants/core/goals/publish_test.py index c86ceb6b6a7..3c5d324290f 100644 --- a/src/python/pants/core/goals/publish_test.py +++ b/src/python/pants/core/goals/publish_test.py @@ -10,7 +10,7 @@ from pants.backend.python.macros.python_artifact import PythonArtifact from pants.backend.python.register import rules as python_rules -from pants.backend.python.target_types import PythonDistribution, PythonLibrary +from pants.backend.python.target_types import PythonDistribution, PythonSourcesGeneratorTarget from pants.core.goals.publish import ( Publish, PublishFieldSet, @@ -74,7 +74,7 @@ def rule_runner() -> RuleRunner: PythonDistribution.register_plugin_field(MockRepositoriesField), *PublishTestFieldSet.rules(), ], - target_types=[PythonLibrary, PythonDistribution], + target_types=[PythonSourcesGeneratorTarget, PythonDistribution], objects={"python_artifact": PythonArtifact}, ) diff --git a/src/python/pants/core/goals/test_test.py b/src/python/pants/core/goals/test_test.py index db9424d9ac4..4c4137462b7 100644 --- a/src/python/pants/core/goals/test_test.py +++ b/src/python/pants/core/goals/test_test.py @@ -12,7 +12,7 @@ import pytest from pants.backend.python.goals import package_pex_binary -from pants.backend.python.target_types import PexBinary, PythonLibrary +from pants.backend.python.target_types import PexBinary, PythonSourcesGeneratorTarget from pants.backend.python.target_types_rules import rules as python_target_type_rules from pants.backend.python.util_rules import pex_from_targets from pants.core.goals.test import ( @@ -421,7 +421,7 @@ def test_runtime_package_dependencies() -> None: *python_target_type_rules(), QueryRule(BuiltPackageDependencies, [BuildPackageDependenciesRequest]), ], - target_types=[PythonLibrary, PexBinary], + target_types=[PythonSourcesGeneratorTarget, PexBinary], ) rule_runner.set_options(args=[], env_inherit={"PATH", "PYENV_ROOT", "HOME"}) diff --git a/src/python/pants/engine/internals/engine_test.py b/src/python/pants/engine/internals/engine_test.py index f50c7d7e01c..0e77d977371 100644 --- a/src/python/pants/engine/internals/engine_test.py +++ b/src/python/pants/engine/internals/engine_test.py @@ -10,7 +10,7 @@ import pytest -from pants.backend.python.target_types import PythonLibrary +from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.base.build_environment import get_buildroot from pants.base.specs import Specs from pants.base.specs_parser import SpecsParser @@ -871,7 +871,7 @@ def __call__(self, **kwargs) -> None: def test_streaming_workunits_expanded_specs(run_tracker: RunTracker) -> None: rule_runner = RuleRunner( - target_types=[PythonLibrary], + target_types=[PythonSourcesGeneratorTarget], rules=[ QueryRule(ProcessResult, (Process,)), ], diff --git a/src/python/pants/engine/target.py b/src/python/pants/engine/target.py index f69a6b310a7..088e96623c9 100644 --- a/src/python/pants/engine/target.py +++ b/src/python/pants/engine/target.py @@ -583,7 +583,7 @@ class Targets(Collection[Target]): """A heterogeneous collection of instances of Target subclasses. While every element will be a subclass of `Target`, there may be many different `Target` types - in this collection, e.g. some `Files` targets and some `PythonLibrary` targets. + in this collection, e.g. some `FileTarget` and some `PythonTestTarget`. Often, you will want to filter out the relevant targets by looking at what fields they have registered, e.g.: @@ -591,7 +591,8 @@ class Targets(Collection[Target]): valid_tgts = [tgt for tgt in tgts if tgt.has_fields([Compatibility, PythonSources])] You should not check the Target's actual type because this breaks custom target types; - for example, prefer `tgt.has_field(PythonTestsSources)` to `isinstance(tgt, PythonTests)`. + for example, prefer `tgt.has_field(PythonTestsSourcesField)` to + `isinstance(tgt, PythonTestsTarget)`. """ def expect_single(self) -> Target: diff --git a/src/python/pants/vcs/changed_integration_test.py b/src/python/pants/vcs/changed_integration_test.py index 01f009c7dbb..4076e88a0a2 100644 --- a/src/python/pants/vcs/changed_integration_test.py +++ b/src/python/pants/vcs/changed_integration_test.py @@ -204,7 +204,6 @@ def run_pants_with_workdir(*args, **kwargs) -> PantsResult: direct=[ "src/python/python_targets/test_binary.py:binary_file", "src/python/python_targets/test_library.py:test_library", - "src/python/python_targets:binary_file", "src/python/python_targets:test_library", # NB: 'src/python/python_targets:test_binary' does not show up here because it is # not a direct dependee of `test_library.py:test_library`; instead, it depends on