diff --git a/src/python/pants/engine/build_files.py b/src/python/pants/engine/build_files.py index 870757a005f7..d3ad0cdc05c1 100644 --- a/src/python/pants/engine/build_files.py +++ b/src/python/pants/engine/build_files.py @@ -21,7 +21,7 @@ from pants.engine.mapper import AddressFamily, AddressMap, AddressMapper, ResolveError from pants.engine.objects import Locatable, SerializableFactory, Validatable from pants.engine.rules import RootRule, SingletonRule, TaskRule, rule -from pants.engine.selectors import Select, SelectDependencies, SelectProjection +from pants.engine.selectors import Get, Select, SelectDependencies from pants.engine.struct import Struct from pants.util.dirutil import fast_relpath_optional from pants.util.objects import datatype @@ -36,51 +36,30 @@ def _key_func(entry): return key -class BuildDirs(datatype('BuildDirs', ['dependencies'])): - """A list of Stat objects for directories containing build files.""" - - -class BuildFiles(datatype('BuildFiles', ['files_content'])): - """The FileContents of BUILD files in some directory""" - - -class BuildFileGlobs(datatype('BuildFilesGlobs', ['path_globs'])): - """A wrapper around PathGlobs that are known to match a build file pattern.""" - - class Specs(Collection.of(Spec)): """A collection of Spec subclasses.""" -@rule(BuildFiles, - [SelectProjection(FilesContent, PathGlobs, 'path_globs', BuildFileGlobs)]) -def build_files(files_content): - return BuildFiles(files_content) - - -@rule(BuildFileGlobs, [Select(AddressMapper), Select(Dir)]) -def buildfile_path_globs_for_dir(address_mapper, directory): - patterns = tuple(join(directory.path, p) for p in address_mapper.build_patterns) - return BuildFileGlobs(PathGlobs.create('', - include=patterns, - exclude=address_mapper.build_ignore_patterns)) - - -@rule(AddressFamily, [Select(AddressMapper), Select(Dir), Select(BuildFiles)]) -def parse_address_family(address_mapper, path, build_files): +@rule(AddressFamily, [Select(AddressMapper), Select(Dir)]) +def parse_address_family(address_mapper, directory): """Given the contents of the build files in one directory, return an AddressFamily. The AddressFamily may be empty, but it will not be None. """ - files_content = build_files.files_content.dependencies + patterns = tuple(join(directory.path, p) for p in address_mapper.build_patterns) + path_globs = PathGlobs.create('', + include=patterns, + exclude=address_mapper.build_ignore_patterns) + files_content = yield Get(FilesContent, PathGlobs, path_globs) + if not files_content: - raise ResolveError('Directory "{}" does not contain build files.'.format(path)) + raise ResolveError('Directory "{}" does not contain build files.'.format(directory.path)) address_maps = [] - for filecontent_product in files_content: + for filecontent_product in files_content.dependencies: address_maps.append(AddressMap.parse(filecontent_product.path, filecontent_product.content, address_mapper.parser)) - return AddressFamily.create(path.path, address_maps) + yield AddressFamily.create(directory.path, address_maps) class UnhydratedStruct(datatype('UnhydratedStruct', ['address', 'struct', 'dependencies'])): @@ -112,17 +91,16 @@ def _raise_did_you_mean(address_family, name): .format(name, address_family.namespace, possibilities)) -@rule(UnhydratedStruct, - [Select(AddressMapper), - SelectProjection(AddressFamily, Dir, 'spec_path', Address), - Select(Address)]) -def resolve_unhydrated_struct(address_mapper, address_family, address): +@rule(UnhydratedStruct, [Select(AddressMapper), Select(Address)]) +def resolve_unhydrated_struct(address_mapper, address): """Given an Address and its AddressFamily, resolve an UnhydratedStruct. Recursively collects any embedded addressables within the Struct, but will not walk into a dependencies field, since those are requested explicitly by tasks using SelectDependencies. """ + address_family = yield Get(AddressFamily, Dir(address.spec_path)) + struct = address_family.addressables.get(address) addresses = address_family.addressables if not struct or address not in addresses: @@ -153,7 +131,7 @@ def collect_dependencies(item): collect_dependencies(struct) - return UnhydratedStruct( + yield UnhydratedStruct( filter(lambda build_address: build_address == address, addresses)[0], struct, dependencies) @@ -227,17 +205,18 @@ def _hydrate(item_type, spec_path, **kwargs): return item -@rule(BuildFileAddresses, - [Select(AddressMapper), - SelectDependencies(AddressFamily, BuildDirs, field_types=(Dir,)), - Select(Specs)]) -def addresses_from_address_families(address_mapper, address_families, specs): +@rule(BuildFileAddresses, [Select(AddressMapper), Select(Specs)]) +def addresses_from_address_families(address_mapper, specs): """Given a list of AddressFamilies matching a list of Specs, return matching Addresses. Raises a AddressLookupError if: - there were no matching AddressFamilies, or - the Spec matches no addresses for SingleAddresses. """ + # Capture a Snapshot covering all paths for these Specs, then group by directory. + snapshot = yield Get(Snapshot, PathGlobs, _spec_to_globs(address_mapper, specs)) + dirnames = set(dirname(f.stat.path) for f in snapshot.files) + address_families = yield [Get(AddressFamily, Dir(d)) for d in dirnames] # NB: `@memoized` does not work on local functions. def by_directory(): @@ -299,20 +278,11 @@ def include(address_families, predicate=None): else: raise ValueError('Unrecognized Spec type: {}'.format(spec)) - return BuildFileAddresses(addresses) + yield BuildFileAddresses(addresses) -@rule(BuildDirs, [Select(AddressMapper), Select(Snapshot)]) -def filter_build_dirs(address_mapper, snapshot): - """Given a Snapshot matching a build pattern, return parent directories as BuildDirs.""" - dirnames = set(dirname(f.stat.path) for f in snapshot.files) - return BuildDirs(tuple(Dir(d) for d in dirnames)) - - -@rule(PathGlobs, [Select(AddressMapper), Select(Specs)]) -def spec_to_globs(address_mapper, specs): - """Given a Spec object, return a PathGlobs object for the build files that it matches. - """ +def _spec_to_globs(address_mapper, specs): + """Given a Specs object, return a PathGlobs object for the build files that it matches.""" patterns = set() for spec in specs.dependencies: if type(spec) is DescendantAddresses: @@ -351,9 +321,6 @@ def _recursive_dirname(f): HydratedStructs = Collection.of(Struct) -BuildFilesCollection = Collection.of(BuildFiles) - - def create_graph_rules(address_mapper, symbol_table): """Creates tasks used to parse Structs from BUILD files. @@ -362,9 +329,6 @@ def create_graph_rules(address_mapper, symbol_table): """ symbol_table_constraint = symbol_table.constraint() return [ - TaskRule(BuildFilesCollection, - [SelectDependencies(BuildFiles, BuildDirs, field_types=(Dir,))], - BuildFilesCollection), # A singleton to provide the AddressMapper. SingletonRule(AddressMapper, address_mapper), # Support for resolving Structs from Addresses. @@ -386,16 +350,13 @@ def create_graph_rules(address_mapper, symbol_table): ), # BUILD file parsing. parse_address_family, - build_files, - buildfile_path_globs_for_dir, # Spec handling: locate directories that contain build files, and request # AddressFamilies for each of them. addresses_from_address_families, - filter_build_dirs, - spec_to_globs, # Root rules representing parameters that might be provided via root subjects. RootRule(Address), RootRule(BuildFileAddress), RootRule(BuildFileAddresses), + RootRule(Dir), RootRule(Specs), ] diff --git a/src/python/pants/engine/legacy/address_mapper.py b/src/python/pants/engine/legacy/address_mapper.py index c51da69929cb..7822b936885b 100644 --- a/src/python/pants/engine/legacy/address_mapper.py +++ b/src/python/pants/engine/legacy/address_mapper.py @@ -13,7 +13,7 @@ from pants.build_graph.address_lookup_error import AddressLookupError from pants.build_graph.address_mapper import AddressMapper from pants.engine.addressable import BuildFileAddresses -from pants.engine.build_files import BuildFilesCollection, Specs +from pants.engine.build_files import Specs from pants.engine.mapper import ResolveError from pants.engine.nodes import Throw from pants.util.dirutil import fast_relpath @@ -34,6 +34,7 @@ def __init__(self, scheduler, build_root): def scan_build_files(self, base_path): specs = (DescendantAddresses(base_path),) + # TODO build_files_collection, = self._scheduler.product_request(BuildFilesCollection, [Specs(specs)]) build_files_set = set() diff --git a/src/python/pants/engine/legacy/graph.py b/src/python/pants/engine/legacy/graph.py index 99afb4cbcb81..5d3e9da62c6e 100644 --- a/src/python/pants/engine/legacy/graph.py +++ b/src/python/pants/engine/legacy/graph.py @@ -25,7 +25,7 @@ from pants.engine.fs import PathGlobs, Snapshot from pants.engine.legacy.structs import BundleAdaptor, BundlesField, SourcesField, TargetAdaptor from pants.engine.rules import TaskRule, rule -from pants.engine.selectors import Select, SelectDependencies, SelectProjection +from pants.engine.selectors import Get, Select, SelectDependencies from pants.source.wrapped_globs import EagerFilesetWithSpec, FilesetRelPathWrapper from pants.util.dirutil import fast_relpath from pants.util.objects import datatype @@ -380,15 +380,15 @@ def _eager_fileset_with_spec(spec_path, filespec, snapshot, include_dirs=False): files_hash=snapshot.fingerprint) -@rule(HydratedField, - [Select(SourcesField), - SelectProjection(Snapshot, PathGlobs, 'path_globs', SourcesField)]) -def hydrate_sources(sources_field, snapshot): +@rule(HydratedField, [Select(SourcesField)]) +def hydrate_sources(sources_field): """Given a SourcesField and a Snapshot for its path_globs, create an EagerFilesetWithSpec.""" + + snapshot = yield Get(Snapshot, PathGlobs, sources_field.path_globs) fileset_with_spec = _eager_fileset_with_spec(sources_field.address.spec_path, sources_field.filespecs, snapshot) - return HydratedField(sources_field.arg, fileset_with_spec) + yield HydratedField(sources_field.arg, fileset_with_spec) @rule(HydratedField,