From 8ba0dfa41aff5cebb84fd48beda78b31cbe8bc6f Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Wed, 29 Sep 2021 21:40:07 -0700 Subject: [PATCH] Have `peek` emit expanded sources and dependencies (#12882) This will allow it to be used to produce a detailed JSON dependency graph that can be consumed outside Pants. [ci skip-rust] [ci skip-build-wheels] --- src/python/pants/backend/project_info/peek.py | 115 +++++++++++---- .../pants/backend/project_info/peek_test.py | 137 ++++++++++++++---- src/python/pants/engine/internals/graph.py | 2 +- src/python/pants/engine/target.py | 2 +- src/python/pants/python/python_setup.py | 14 +- 5 files changed, 200 insertions(+), 70 deletions(-) diff --git a/src/python/pants/backend/project_info/peek.py b/src/python/pants/backend/project_info/peek.py index 1c87edbb9cc..f752bcb9a59 100644 --- a/src/python/pants/backend/project_info/peek.py +++ b/src/python/pants/backend/project_info/peek.py @@ -1,35 +1,43 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -import collections.abc +from __future__ import annotations + +import collections import json import os -from dataclasses import asdict, is_dataclass +from dataclasses import asdict, dataclass, is_dataclass from enum import Enum from typing import Any, Iterable, Mapping, cast from pkg_resources import Requirement from pants.engine.addresses import Address, BuildFileAddress +from pants.engine.collection import Collection from pants.engine.console import Console from pants.engine.fs import DigestContents, FileContent, PathGlobs from pants.engine.goal import Goal, GoalSubsystem, Outputting -from pants.engine.rules import Get, MultiGet, collect_rules, goal_rule -from pants.engine.target import Target, UnexpandedTargets - - +from pants.engine.rules import Get, MultiGet, collect_rules, goal_rule, rule +from pants.engine.target import ( + Dependencies, + DependenciesRequest, + HydratedSources, + HydrateSourcesRequest, + Sources, + Target, + Targets, + UnexpandedTargets, +) + + +# TODO: Delete this in 2.9.0.dev0. class OutputOptions(Enum): RAW = "raw" JSON = "json" class PeekSubsystem(Outputting, GoalSubsystem): - """Display BUILD file info to the console. - - In its most basic form, `peek` just prints the contents of a BUILD file. It can also display - multiple BUILD files, or render normalized target metadata as JSON for consumption by other - programs. - """ + """Display detailed target information in JSON form.""" name = "peek" help = "Display BUILD target info" @@ -41,6 +49,9 @@ def register_options(cls, register): "--output", type=OutputOptions, default=OutputOptions.JSON, + removal_version="2.9.0.dev0", + removal_hint="Output will always be JSON. If you need the raw BUILD file contents, " + "look at it directly!", help=( "Which output style peek should use: `json` will show each target as a seperate " "entry, whereas `raw` will simply show the original non-normalized BUILD files." @@ -50,12 +61,10 @@ def register_options(cls, register): "--exclude-defaults", type=bool, default=False, - help=( - "Whether to leave off values that match the target-defined default values " - "when using `json` output." - ), + help="Whether to leave off values that match the target-defined default values.", ) + # TODO: Delete this in 2.9.0.dev0. @property def output_type(self) -> OutputOptions: """Get the output type from options. @@ -73,12 +82,14 @@ class Peek(Goal): subsystem_cls = PeekSubsystem +# TODO: Delete this in 2.9.0.dev0. def _render_raw(fcs: Iterable[FileContent]) -> str: sorted_fcs = sorted(fcs, key=lambda fc: fc.path) rendereds = map(_render_raw_build_file, sorted_fcs) return os.linesep.join(rendereds) +# TODO: Delete this in 2.9.0.dev0. def _render_raw_build_file(fc: FileContent, encoding: str = "utf-8") -> str: dashes = "-" * len(fc.path) content = fc.content.decode(encoding) @@ -88,27 +99,39 @@ def _render_raw_build_file(fc: FileContent, encoding: str = "utf-8") -> str: return os.linesep.join(parts) -_nothing = object() +@dataclass(frozen=True) +class TargetData: + target: Target + expanded_sources: tuple[str, ...] | None # Target may not be of a type that has sources. + expanded_dependencies: tuple[str, ...] + + +class TargetDatas(Collection[TargetData]): + pass -def _render_json(ts: Iterable[Target], exclude_defaults: bool = False) -> str: +def _render_json(tds: Iterable[TargetData], exclude_defaults: bool = False) -> str: + nothing = object() + targets: Iterable[Mapping[str, Any]] = [ { - "address": t.address.spec, - "target_type": t.alias, + "address": td.target.address.spec, + "target_type": td.target.alias, **{ - k.alias: v.value - for k, v in t.field_values.items() - if not (exclude_defaults and getattr(k, "default", _nothing) == v.value) + (f"{k.alias}_raw" if k.alias in {"sources", "dependencies"} else k.alias): v.value + for k, v in td.target.field_values.items() + if not (exclude_defaults and getattr(k, "default", nothing) == v.value) }, + **({} if td.expanded_sources is None else {"sources": td.expanded_sources}), + "dependencies": td.expanded_dependencies, } - for t in ts + for td in tds ] return f"{json.dumps(targets, indent=2, cls=_PeekJsonEncoder)}\n" class _PeekJsonEncoder(json.JSONEncoder): - """Allow us to serialize some commmonly-found types in BUILD files.""" + """Allow us to serialize some commmonly found types in BUILD files.""" safe_to_str_types = (Requirement,) @@ -126,27 +149,57 @@ def default(self, o): return str(o) +@rule +async def get_target_data(targets: UnexpandedTargets) -> TargetDatas: + sorted_targets = sorted(targets, key=lambda tgt: tgt.address) + + dependencies_per_target = await MultiGet( + Get( + Targets, + DependenciesRequest(tgt.get(Dependencies), include_special_cased_deps=True), + ) + for tgt in sorted_targets + ) + + # Not all targets have a sources field, so we have to do a dance here. + targets_with_sources = [tgt for tgt in sorted_targets if tgt.has_field(Sources)] + all_hydrated_sources = await MultiGet( + Get(HydratedSources, HydrateSourcesRequest(tgt[Sources])) for tgt in targets_with_sources + ) + hydrated_sources_map = { + tgt.address: hs for tgt, hs in zip(targets_with_sources, all_hydrated_sources) + } + sources_per_target = [hydrated_sources_map.get(tgt.address) for tgt in sorted_targets] + + return TargetDatas( + TargetData( + tgt, srcs.snapshot.files if srcs else None, tuple(dep.address.spec for dep in deps) + ) + for tgt, srcs, deps in zip(sorted_targets, sources_per_target, dependencies_per_target) + ) + + @goal_rule async def peek( console: Console, subsys: PeekSubsystem, targets: UnexpandedTargets, ) -> Peek: + # TODO: Delete this entire conditional in 2.9.0.dev0. if subsys.output_type == OutputOptions.RAW: build_file_addresses = await MultiGet( Get(BuildFileAddress, Address, t.address) for t in targets ) build_file_paths = {a.rel_path for a in build_file_addresses} digest_contents = await Get(DigestContents, PathGlobs(build_file_paths)) - output = _render_raw(digest_contents) - elif subsys.output_type == OutputOptions.JSON: - output = _render_json(targets, subsys.exclude_defaults) - else: - raise AssertionError(f"output_type not one of {tuple(OutputOptions)}") + with subsys.output(console) as write_stdout: + write_stdout(_render_raw(digest_contents)) + return Peek(exit_code=0) + tds = await Get(TargetDatas, UnexpandedTargets, targets) + output = _render_json(tds, subsys.exclude_defaults) with subsys.output(console) as write_stdout: write_stdout(output) - return Peek(exit_code=0) diff --git a/src/python/pants/backend/project_info/peek_test.py b/src/python/pants/backend/project_info/peek_test.py index 27576c5f03e..961dd739fb7 100644 --- a/src/python/pants/backend/project_info/peek_test.py +++ b/src/python/pants/backend/project_info/peek_test.py @@ -6,14 +6,16 @@ import pytest from pants.backend.project_info import peek -from pants.backend.project_info.peek import Peek -from pants.core.target_types import ArchiveTarget, Files +from pants.backend.project_info.peek import Peek, TargetData, TargetDatas +from pants.base.specs import AddressSpecs, DescendantAddresses +from pants.core.target_types import ArchiveTarget, Files, GenericTarget from pants.engine.addresses import Address +from pants.engine.rules import QueryRule from pants.testutil.rule_runner import RuleRunner @pytest.mark.parametrize( - "targets, exclude_defaults, expected_output", + "expanded_target_infos, exclude_defaults, expected_output", [ pytest.param( [], @@ -22,7 +24,13 @@ id="null-case", ), pytest.param( - [Files({"sources": []}, Address("example", target_name="files_target"))], + [ + TargetData( + Files({"sources": ["*.txt"]}, Address("example", target_name="files_target")), + ("foo.txt", "bar.txt"), + tuple(), + ) + ], True, dedent( """\ @@ -30,7 +38,14 @@ { "address": "example:files_target", "target_type": "files", - "sources": [] + "sources_raw": [ + "*.txt" + ], + "sources": [ + "foo.txt", + "bar.txt" + ], + "dependencies": [] } ] """ @@ -38,7 +53,13 @@ id="single-files-target/exclude-defaults", ), pytest.param( - [Files({"sources": []}, Address("example", target_name="files_target"))], + [ + TargetData( + Files({"sources": []}, Address("example", target_name="files_target")), + tuple(), + tuple(), + ) + ], False, dedent( """\ @@ -46,10 +67,12 @@ { "address": "example:files_target", "target_type": "files", - "dependencies": null, + "dependencies_raw": null, "description": null, + "sources_raw": [], + "tags": null, "sources": [], - "tags": null + "dependencies": [] } ] """ @@ -58,17 +81,25 @@ ), pytest.param( [ - Files( - {"sources": ["*.txt"], "tags": ["zippable"]}, - Address("example", target_name="files_target"), + TargetData( + Files( + {"sources": ["*.txt"], "tags": ["zippable"]}, + Address("example", target_name="files_target"), + ), + tuple(), + tuple(), ), - ArchiveTarget( - { - "output_path": "my-archive.zip", - "format": "zip", - "files": ["example:files_target"], - }, - Address("example", target_name="archive_target"), + TargetData( + ArchiveTarget( + { + "output_path": "my-archive.zip", + "format": "zip", + "files": ["example:files_target"], + }, + Address("example", target_name="archive_target"), + ), + None, + ("foo/bar:baz", "qux:quux"), ), ], True, @@ -78,12 +109,14 @@ { "address": "example:files_target", "target_type": "files", - "sources": [ + "sources_raw": [ "*.txt" ], "tags": [ "zippable" - ] + ], + "sources": [], + "dependencies": [] }, { "address": "example:archive_target", @@ -92,7 +125,11 @@ "example:files_target" ], "format": "zip", - "output_path": "my-archive.zip" + "output_path": "my-archive.zip", + "dependencies": [ + "foo/bar:baz", + "qux:quux" + ] } ] """ @@ -101,14 +138,60 @@ ), ], ) -def test_render_targets_as_json(targets, exclude_defaults, expected_output): - actual_output = peek._render_json(targets, exclude_defaults) +def test_render_targets_as_json(expanded_target_infos, exclude_defaults, expected_output): + actual_output = peek._render_json(expanded_target_infos, exclude_defaults) assert actual_output == expected_output @pytest.fixture def rule_runner() -> RuleRunner: - return RuleRunner(rules=peek.rules(), target_types=[Files]) + return RuleRunner( + rules=[ + *peek.rules(), + QueryRule(TargetDatas, [AddressSpecs]), + ], + target_types=[Files, GenericTarget], + ) + + +def test_non_matching_build_target(rule_runner: RuleRunner) -> None: + rule_runner.add_to_build_file("some_name", "files(sources=[])") + result = rule_runner.run_goal_rule(Peek, args=["other_name"]) + assert result.stdout == "[]\n" + + +def test_get_target_data(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "foo/BUILD": dedent( + """\ + target(name="bar", dependencies=[":baz"]) + + files(name="baz", sources=["*.txt"]) + """ + ), + "foo/a.txt": "", + "foo/b.txt": "", + } + ) + tds = rule_runner.request(TargetDatas, [AddressSpecs([DescendantAddresses("foo")])]) + assert tds == TargetDatas( + [ + TargetData( + GenericTarget({"dependencies": [":baz"]}, Address("foo", target_name="bar")), + None, + ("foo:baz",), + ), + TargetData( + Files({"sources": ["*.txt"]}, Address("foo", target_name="baz")), + ("foo/a.txt", "foo/b.txt"), + tuple(), + ), + ] + ) + + +# TODO: Delete everything below this in 2.9.0.dev0. def test_raw_output_single_build_file(rule_runner: RuleRunner) -> None: @@ -152,9 +235,3 @@ def test_raw_output_non_matching_build_target(rule_runner: RuleRunner) -> None: rule_runner.add_to_build_file("some_name", "files(sources=[])") result = rule_runner.run_goal_rule(Peek, args=["--output=raw", "other_name"]) assert result.stdout == "" - - -def test_standard_json_output_non_matching_build_target(rule_runner: RuleRunner) -> None: - rule_runner.add_to_build_file("some_name", "files(sources=[])") - result = rule_runner.run_goal_rule(Peek, args=["other_name"]) - assert result.stdout == "[]\n" diff --git a/src/python/pants/engine/internals/graph.py b/src/python/pants/engine/internals/graph.py index 113b1b50f03..04a4275897b 100644 --- a/src/python/pants/engine/internals/graph.py +++ b/src/python/pants/engine/internals/graph.py @@ -422,7 +422,7 @@ async def find_owners(owners_request: OwnersRequest) -> Owners: deleted_dirs = FrozenOrderedSet(os.path.dirname(s) for s in deleted_files) # Walk up the buildroot looking for targets that would conceivably claim changed sources. - # For live files, we use ExpandedTargets, which causes more precise, often file-level, targets + # For live files, we use Targets, which causes more precise, often file-level, targets # to be created. For deleted files we use UnexpandedTargets, which have the original declared # glob. live_candidate_specs = tuple(AscendantAddresses(directory=d) for d in live_dirs) diff --git a/src/python/pants/engine/target.py b/src/python/pants/engine/target.py index 37c2f4925a1..61479b7cbcf 100644 --- a/src/python/pants/engine/target.py +++ b/src/python/pants/engine/target.py @@ -605,7 +605,7 @@ def expect_single(self) -> Target: @dataclass(frozen=True) class CoarsenedTarget(EngineAwareParameter): - """A set of Targets which cyclicly reach one another, and are thus indivisable.""" + """A set of Targets which cyclicly reach one another, and are thus indivisible.""" # The members of the cycle. members: Tuple[Target, ...] diff --git a/src/python/pants/python/python_setup.py b/src/python/pants/python/python_setup.py index 4669897e138..cf6109dce57 100644 --- a/src/python/pants/python/python_setup.py +++ b/src/python/pants/python/python_setup.py @@ -164,15 +164,15 @@ def register_options(cls, register): "and/or to directories containing interpreter binaries. The order of entries does " "not matter.\n\n" "The following special strings are supported:\n\n" - '* "", the contents of the PATH env var\n' - '* "", all Python versions currently configured by ASDF ' - "(asdf shell, ${HOME}/.tool-versions), with a fallback to all installed versions\n" - '* "", the ASDF interpreter with the version in ' + "* ``, the contents of the PATH env var\n" + "* ``, all Python versions currently configured by ASDF " + "`(asdf shell, ${HOME}/.tool-versions)`, with a fallback to all installed versions\n" + "* ``, the ASDF interpreter with the version in " "BUILD_ROOT/.tool-versions\n" - '* "", all Python versions under $(pyenv root)/versions\n' - '* "", the Pyenv interpreter with the version in ' + "* ``, all Python versions under $(pyenv root)/versions\n" + "* ``, the Pyenv interpreter with the version in " "BUILD_ROOT/.python-version\n" - '* "", paths in the PEX_PYTHON_PATH variable in /etc/pexrc or ~/.pexrc' + "* ``, paths in the PEX_PYTHON_PATH variable in /etc/pexrc or ~/.pexrc" ), ) register(