Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add options to fast-filedeps to bring it close to parity with V1 filedeps #8184

Merged
merged 5 commits into from
Nov 8, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/python/pants/init/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ python_library(
'3rdparty/python/twitter/commons:twitter.common.collections',
'src/python/pants/backend/python/subsystems',
'src/python/pants/base:build_environment',
'src/python/pants/base:build_root',
'src/python/pants/base:exception_sink',
'src/python/pants/base:exceptions',
'src/python/pants/base:target_roots',
Expand Down
6 changes: 6 additions & 0 deletions src/python/pants/init/engine_initializer.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from pants.backend.python.targets.python_library import PythonLibrary
from pants.backend.python.targets.python_tests import PythonTests
from pants.base.build_environment import get_buildroot
from pants.base.build_root import BuildRoot
from pants.base.exiter import PANTS_SUCCEEDED_EXIT_CODE
from pants.base.file_system_project_tree import FileSystemProjectTree
from pants.build_graph.build_configuration import BuildConfiguration
Expand Down Expand Up @@ -366,6 +367,10 @@ def symbol_table_singleton() -> SymbolTable:
def union_membership_singleton() -> UnionMembership:
return UnionMembership(build_configuration.union_rules())

@rule
def build_root_singleton() -> BuildRoot:
return BuildRoot.instance

# Create a Scheduler containing graph and filesystem rules, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
rules = (
Expand All @@ -375,6 +380,7 @@ def union_membership_singleton() -> UnionMembership:
build_configuration_singleton,
symbol_table_singleton,
union_membership_singleton,
build_root_singleton,
] +
create_legacy_graph_tasks() +
create_fs_rules() +
Expand Down
1 change: 1 addition & 0 deletions src/python/pants/rules/core/BUILD
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
python_library(
dependencies = [
'3rdparty/python:dataclasses',
'src/python/pants/base:build_root',
'src/python/pants/base:exiter',
'src/python/pants/build_graph',
'src/python/pants/engine:goal',
Expand Down
40 changes: 32 additions & 8 deletions src/python/pants/rules/core/filedeps.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from pex.orderedset import OrderedSet
from pathlib import Path
from typing import Set

from pants.base.build_root import BuildRoot
from pants.engine.console import Console
from pants.engine.goal import Goal, LineOriented
from pants.engine.legacy.graph import TransitiveHydratedTargets
Expand All @@ -16,28 +18,50 @@ class Filedeps(LineOriented, Goal):
closure of targets are also included.
"""

# TODO: Until this implements more of the options of `filedeps`, it can't claim the name!
name = 'fast-filedeps'

@classmethod
def register_options(cls, register):
super().register_options(register)
register(
'--absolute', type=bool, default=True,
help='If True, output with absolute path; else, output with path relative to the build root'
)
register(
'--globs', type=bool,
help='Instead of outputting filenames, output globs (ignoring excludes)'
)


@console_rule
def file_deps(
console: Console,
filedeps_options: Filedeps.Options,
build_root: BuildRoot,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So... one development since this was opened is the Workspace type.

I think that the semantics of the BuildRoot type (exposes some information) are sufficiently different from the Workspace type (implies mutable access) that having them separate could be reasonable. I could also see merging them, and exposing Workspace.root or something. "Relatively small single purpose things to DI" is a thing... but the BuildRoot is almost too specific a thing, and would always have a subset of the fields of Workspace.

Fine either way though.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

transitive_hydrated_targets: TransitiveHydratedTargets
) -> Filedeps:

uniq_set = OrderedSet()
absolute = filedeps_options.values.absolute
globs = filedeps_options.values.globs

unique_rel_paths: Set[str] = set()

for hydrated_target in transitive_hydrated_targets.closure:
adaptor = hydrated_target.adaptor
if hydrated_target.address.rel_path:
uniq_set.add(hydrated_target.address.rel_path)
if hasattr(hydrated_target.adaptor, "sources"):
uniq_set.update(hydrated_target.adaptor.sources.snapshot.files)
unique_rel_paths.add(hydrated_target.address.rel_path)
if hasattr(adaptor, "sources"):
sources_paths = (
adaptor.sources.snapshot.files
if not globs
else adaptor.sources.filespec["globs"]
)
unique_rel_paths.update(sources_paths)

with Filedeps.line_oriented(filedeps_options, console) as print_stdout:
for f_path in uniq_set:
print_stdout(f_path)
for rel_path in sorted(unique_rel_paths):
final_path = str(Path(build_root.path, rel_path)) if absolute else rel_path
print_stdout(final_path)

return Filedeps(exit_code=0)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,27 +40,25 @@ def alias_groups(cls):
def rules(cls):
return super().rules() + list_targets.rules()

def setUp(self):
def setUp(self) -> None:
super().setUp()

# Setup a BUILD tree for various list tests
class Lib:

def __init__(self, name, provides=False):
def __init__(self, name: str, provides: bool = False) -> None:
self.name = name
self.provides = dedent("""
self.provides = dedent(f"""
artifact(
org='com.example',
name='{0}',
name='{name}',
repo=public
)
""".format(name)).strip() if provides else 'None'
""").strip() if provides else 'None'

def create_library(path, *libs):
def create_library(path: str, *libs: Lib) -> None:
libs = libs or [Lib(os.path.basename(os.path.dirname(self.build_path(path))))]
for lib in libs:
target = "java_library(name='{name}', provides={provides}, sources=[])\n".format(
name=lib.name, provides=lib.provides)
target = f"java_library(name='{lib.name}', provides={lib.provides}, sources=[])\n"
self.add_to_build_file(path, target)

create_library('a')
Expand Down
7 changes: 4 additions & 3 deletions tests/python/pants_test/rules/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,12 @@ python_tests(
name='filedeps',
source='test_filedeps.py',
dependencies=[
'src/python/pants/backend/codegen/thrift/java',
'src/python/pants/backend/jvm/targets:all',
'src/python/pants/backend/python/targets',
'src/python/pants/build_graph',
'src/python/pants/engine/legacy:graph',
'src/python/pants/rules/core',
'tests/python/pants_test:test_base',
'tests/python/pants_test/engine:util',
'tests/python/pants_test:console_rule_test_base',
]
)

Expand Down
Loading