Replace all usage of SelectProjection and some uses of SelectDependen…

…cies with the generator syntax.

src/python/pants/engine/build_files.py

@@ -21,7 +21,7 @@
 from pants.engine.mapper import AddressFamily, AddressMap, AddressMapper, ResolveError
 from pants.engine.objects import Locatable, SerializableFactory, Validatable
 from pants.engine.rules import RootRule, SingletonRule, TaskRule, rule
-from pants.engine.selectors import Select, SelectDependencies, SelectProjection
+from pants.engine.selectors import Get, Select, SelectDependencies
 from pants.engine.struct import Struct
 from pants.util.dirutil import fast_relpath_optional
 from pants.util.objects import datatype
@@ -36,51 +36,30 @@ def _key_func(entry):
   return key
 
 
-class BuildDirs(datatype('BuildDirs', ['dependencies'])):
-  """A list of Stat objects for directories containing build files."""
-
-
-class BuildFiles(datatype('BuildFiles', ['files_content'])):
-  """The FileContents of BUILD files in some directory"""
-
-
-class BuildFileGlobs(datatype('BuildFilesGlobs', ['path_globs'])):
-  """A wrapper around PathGlobs that are known to match a build file pattern."""
-
-
 class Specs(Collection.of(Spec)):
   """A collection of Spec subclasses."""
 
 
-@rule(BuildFiles,
-      [SelectProjection(FilesContent, PathGlobs, 'path_globs', BuildFileGlobs)])
-def build_files(files_content):
-  return BuildFiles(files_content)
-
-
-@rule(BuildFileGlobs, [Select(AddressMapper), Select(Dir)])
-def buildfile_path_globs_for_dir(address_mapper, directory):
-  patterns = tuple(join(directory.path, p) for p in address_mapper.build_patterns)
-  return BuildFileGlobs(PathGlobs.create('',
-                                         include=patterns,
-                                         exclude=address_mapper.build_ignore_patterns))
-
-
-@rule(AddressFamily, [Select(AddressMapper), Select(Dir), Select(BuildFiles)])
-def parse_address_family(address_mapper, path, build_files):
+@rule(AddressFamily, [Select(AddressMapper), Select(Dir)])
+def parse_address_family(address_mapper, directory):
   """Given the contents of the build files in one directory, return an AddressFamily.
 
   The AddressFamily may be empty, but it will not be None.
   """
-  files_content = build_files.files_content.dependencies
+  patterns = tuple(join(directory.path, p) for p in address_mapper.build_patterns)
+  path_globs = PathGlobs.create('',
+                                include=patterns,
+                                exclude=address_mapper.build_ignore_patterns)
+  files_content = yield Get(FilesContent, PathGlobs, path_globs)
+
   if not files_content:
-    raise ResolveError('Directory "{}" does not contain build files.'.format(path))
+    raise ResolveError('Directory "{}" does not contain build files.'.format(directory.path))
   address_maps = []
-  for filecontent_product in files_content:
+  for filecontent_product in files_content.dependencies:
     address_maps.append(AddressMap.parse(filecontent_product.path,
                                          filecontent_product.content,
                                          address_mapper.parser))
-  return AddressFamily.create(path.path, address_maps)
+  yield AddressFamily.create(directory.path, address_maps)
 
 
 class UnhydratedStruct(datatype('UnhydratedStruct', ['address', 'struct', 'dependencies'])):
@@ -112,17 +91,16 @@ def _raise_did_you_mean(address_family, name):
                      .format(name, address_family.namespace, possibilities))
 
 
-@rule(UnhydratedStruct,
-      [Select(AddressMapper),
-       SelectProjection(AddressFamily, Dir, 'spec_path', Address),
-       Select(Address)])
-def resolve_unhydrated_struct(address_mapper, address_family, address):
+@rule(UnhydratedStruct, [Select(AddressMapper), Select(Address)])
+def resolve_unhydrated_struct(address_mapper, address):
   """Given an Address and its AddressFamily, resolve an UnhydratedStruct.
 
   Recursively collects any embedded addressables within the Struct, but will not walk into a
   dependencies field, since those are requested explicitly by tasks using SelectDependencies.
   """
 
+  address_family = yield Get(AddressFamily, Dir(address.spec_path))
+
   struct = address_family.addressables.get(address)
   addresses = address_family.addressables
   if not struct or address not in addresses:
@@ -153,7 +131,7 @@ def collect_dependencies(item):
 
   collect_dependencies(struct)
 
-  return UnhydratedStruct(
+  yield UnhydratedStruct(
     filter(lambda build_address: build_address == address, addresses)[0], struct, dependencies)
 
 
@@ -227,17 +205,18 @@ def _hydrate(item_type, spec_path, **kwargs):
   return item
 
 
-@rule(BuildFileAddresses,
-      [Select(AddressMapper),
-       SelectDependencies(AddressFamily, BuildDirs, field_types=(Dir,)),
-       Select(Specs)])
-def addresses_from_address_families(address_mapper, address_families, specs):
+@rule(BuildFileAddresses, [Select(AddressMapper), Select(Specs)])
+def addresses_from_address_families(address_mapper, specs):
   """Given a list of AddressFamilies matching a list of Specs, return matching Addresses.
 
   Raises a AddressLookupError if:
      - there were no matching AddressFamilies, or
      - the Spec matches no addresses for SingleAddresses.
   """
+  # Capture a Snapshot covering all paths for these Specs, then group by directory.
+  snapshot = yield Get(Snapshot, PathGlobs, _spec_to_globs(address_mapper, specs))
+  dirnames = set(dirname(f.stat.path) for f in snapshot.files)
+  address_families = yield [Get(AddressFamily, Dir(d)) for d in dirnames]
 
   # NB: `@memoized` does not work on local functions.
   def by_directory():
@@ -299,20 +278,11 @@ def include(address_families, predicate=None):
     else:
       raise ValueError('Unrecognized Spec type: {}'.format(spec))
 
-  return BuildFileAddresses(addresses)
+  yield BuildFileAddresses(addresses)
 
 
-@rule(BuildDirs, [Select(AddressMapper), Select(Snapshot)])
-def filter_build_dirs(address_mapper, snapshot):
-  """Given a Snapshot matching a build pattern, return parent directories as BuildDirs."""
-  dirnames = set(dirname(f.stat.path) for f in snapshot.files)
-  return BuildDirs(tuple(Dir(d) for d in dirnames))
-
-
-@rule(PathGlobs, [Select(AddressMapper), Select(Specs)])
-def spec_to_globs(address_mapper, specs):
-  """Given a Spec object, return a PathGlobs object for the build files that it matches.
-  """
+def _spec_to_globs(address_mapper, specs):
+  """Given a Specs object, return a PathGlobs object for the build files that it matches."""
   patterns = set()
   for spec in specs.dependencies:
     if type(spec) is DescendantAddresses:
@@ -351,9 +321,6 @@ def _recursive_dirname(f):
 HydratedStructs = Collection.of(Struct)
 
 
-BuildFilesCollection = Collection.of(BuildFiles)
-
-
 def create_graph_rules(address_mapper, symbol_table):
   """Creates tasks used to parse Structs from BUILD files.
 
@@ -362,9 +329,6 @@ def create_graph_rules(address_mapper, symbol_table):
   """
   symbol_table_constraint = symbol_table.constraint()
   return [
-    TaskRule(BuildFilesCollection,
-             [SelectDependencies(BuildFiles, BuildDirs, field_types=(Dir,))],
-             BuildFilesCollection),
     # A singleton to provide the AddressMapper.
     SingletonRule(AddressMapper, address_mapper),
     # Support for resolving Structs from Addresses.
@@ -386,16 +350,13 @@ def create_graph_rules(address_mapper, symbol_table):
     ),
     # BUILD file parsing.
     parse_address_family,
-    build_files,
-    buildfile_path_globs_for_dir,
     # Spec handling: locate directories that contain build files, and request
     # AddressFamilies for each of them.
     addresses_from_address_families,
-    filter_build_dirs,
-    spec_to_globs,
     # Root rules representing parameters that might be provided via root subjects.
     RootRule(Address),
     RootRule(BuildFileAddress),
     RootRule(BuildFileAddresses),
+    RootRule(Dir),
     RootRule(Specs),
   ]

src/python/pants/engine/legacy/address_mapper.py

@@ -13,7 +13,7 @@
 from pants.build_graph.address_lookup_error import AddressLookupError
 from pants.build_graph.address_mapper import AddressMapper
 from pants.engine.addressable import BuildFileAddresses
-from pants.engine.build_files import BuildFilesCollection, Specs
+from pants.engine.build_files import Specs
 from pants.engine.mapper import ResolveError
 from pants.engine.nodes import Throw
 from pants.util.dirutil import fast_relpath
@@ -34,6 +34,7 @@ def __init__(self, scheduler, build_root):
 
   def scan_build_files(self, base_path):
     specs = (DescendantAddresses(base_path),)
+    # TODO
     build_files_collection, = self._scheduler.product_request(BuildFilesCollection, [Specs(specs)])
 
     build_files_set = set()

src/python/pants/engine/legacy/graph.py

@@ -25,7 +25,7 @@
 from pants.engine.fs import PathGlobs, Snapshot
 from pants.engine.legacy.structs import BundleAdaptor, BundlesField, SourcesField, TargetAdaptor
 from pants.engine.rules import TaskRule, rule
-from pants.engine.selectors import Select, SelectDependencies, SelectProjection
+from pants.engine.selectors import Get, Select, SelectDependencies
 from pants.source.wrapped_globs import EagerFilesetWithSpec, FilesetRelPathWrapper
 from pants.util.dirutil import fast_relpath
 from pants.util.objects import datatype
@@ -380,15 +380,15 @@ def _eager_fileset_with_spec(spec_path, filespec, snapshot, include_dirs=False):
                               files_hash=snapshot.fingerprint)
 
 
-@rule(HydratedField,
-      [Select(SourcesField),
-       SelectProjection(Snapshot, PathGlobs, 'path_globs', SourcesField)])
-def hydrate_sources(sources_field, snapshot):
+@rule(HydratedField, [Select(SourcesField)])
+def hydrate_sources(sources_field):
   """Given a SourcesField and a Snapshot for its path_globs, create an EagerFilesetWithSpec."""
+
+  snapshot = yield Get(Snapshot, PathGlobs, sources_field.path_globs)
   fileset_with_spec = _eager_fileset_with_spec(sources_field.address.spec_path,
                                                sources_field.filespecs,
                                                snapshot)
-  return HydratedField(sources_field.arg, fileset_with_spec)
+  yield HydratedField(sources_field.arg, fileset_with_spec)
 
 
 @rule(HydratedField,