Add the ability to consume scoped Options from @rules

src/python/pants/bin/local_pants_runner.py

@@ -30,7 +30,7 @@ class LocalPantsRunner(object):
 
   @staticmethod
   def parse_options(args, env, setup_logging=False, options_bootstrapper=None):
-    options_bootstrapper = options_bootstrapper or OptionsBootstrapper(args=args, env=env)
+    options_bootstrapper = options_bootstrapper or OptionsBootstrapper.create(args=args, env=env)
     bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
     if setup_logging:
       # Bootstrap logging and then fully initialize options.

src/python/pants/bin/pants_runner.py

@@ -36,7 +36,7 @@ def run(self):
     # this point onwards will use that exiter in the case of a fatal error.
     ExceptionSink.reset_exiter(self._exiter)
 
-    options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args)
+    options_bootstrapper = OptionsBootstrapper.create(env=self._env, args=self._args)
     bootstrap_options = options_bootstrapper.get_bootstrap_options()
     global_bootstrap_options = bootstrap_options.for_global_scope()
 

src/python/pants/binaries/binary_util.py

@@ -491,7 +491,7 @@ def main():
   # Parse positional arguments to the script.
   args = _create_bootstrap_binary_arg_parser().parse_args()
   # Resolve bootstrap options with a fake empty command line.
-  options_bootstrapper = OptionsBootstrapper(args=[sys.argv[0]])
+  options_bootstrapper = OptionsBootstrapper.create(args=[sys.argv[0]])
   subsystems = (GlobalOptionsRegistrar, BinaryUtil.Factory)
   known_scope_infos = reduce(set.union, (ss.known_scope_infos() for ss in subsystems), set())
   options = options_bootstrapper.get_full_options(known_scope_infos)

src/python/pants/init/engine_initializer.py

@@ -311,7 +311,8 @@ def setup_legacy_graph_extended(
     """
 
     build_root = build_root or get_buildroot()
-    build_configuration = build_configuration or BuildConfigInitializer.get(OptionsBootstrapper())
+    options_bootstrapper = OptionsBootstrapper.create()
+    build_configuration = build_configuration or BuildConfigInitializer.get(options_bootstrapper)
     build_file_aliases = build_configuration.registered_aliases()
     rules = build_configuration.rules()
     console = Console()

src/python/pants/option/options_bootstrapper.py

@@ -8,7 +8,9 @@
 import logging
 import os
 import sys
-from builtins import filter, next, object, open
+from builtins import filter
+
+from future.utils import iteritems
 
 from pants.base.build_environment import get_default_pants_config_file
 from pants.engine.fs import FileContent
@@ -17,14 +19,22 @@
 from pants.option.custom_types import ListValueComponent
 from pants.option.global_options import GlobalOptionsRegistrar
 from pants.option.options import Options
+from pants.util.dirutil import read_file
+from pants.util.memo import memoized_method, memoized_property
+from pants.util.objects import SubclassesOf, datatype
 from pants.util.strutil import ensure_text
 
 
 logger = logging.getLogger(__name__)
 
 
-class OptionsBootstrapper(object):
-  """An object that knows how to create options in two stages: bootstrap, and then full options."""
+class OptionsBootstrapper(datatype([
+  ('env_tuples', tuple),
+  ('bootstrap_args', tuple),
+  ('args', tuple),
+  ('config', SubclassesOf(Config)),
+])):
+  """Holds the result of the first stage of options parsing, and assists with parsing full options."""
 
   @staticmethod
   def get_config_file_paths(env, args):
@@ -59,25 +69,42 @@ def get_config_file_paths(env, args):
 
     return ListValueComponent.merge(path_list_values).val
 
+  @staticmethod
+  def parse_bootstrap_options(env, args, config):
+    bootstrap_options = Options.create(
+      env=env,
+      config=config,
+      known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()],
+      args=args,
+    )
+
+    def register_global(*args, **kwargs):
+      ## Only use of Options.register?
+      bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
+
+    GlobalOptionsRegistrar.register_bootstrap_options(register_global)
+    return bootstrap_options
+
   @classmethod
   def from_options_parse_request(cls, parse_request):
-    inst = cls(env=dict(parse_request.env), args=parse_request.args)
-    inst.construct_and_set_bootstrap_options()
-    return inst
-
-  def __init__(self, env=None, args=None):
-    self._env = env if env is not None else os.environ.copy()
-    self._post_bootstrap_config = None  # Will be set later.
-    self._args = sys.argv if args is None else args
-    self._bootstrap_options = None  # We memoize the bootstrap options here.
-    self._full_options = {}  # We memoize the full options here.
-
-  def produce_and_set_bootstrap_options(self):
-    """Cooperatively populates the internal bootstrap_options cache with
-    a producer of `FileContent`."""
+    return cls.create(env=dict(parse_request.env), args=parse_request.args)
+
+  @classmethod
+  def create(cls, env=None, args=None):
+    """Parses the minimum amount of configuration necessary to create an OptionsBootstrapper.
+
+    :param env: An environment dictionary, or None to use `os.environ`.
+    :param args: An args array, or None to use `sys.argv`.
+    """
+    env = os.environ.copy() if env is None else env
+    args = tuple(sys.argv if args is None else args)
+
     flags = set()
     short_flags = set()
 
+    def filecontent_for(path):
+      return FileContent(ensure_text(path), read_file(path))
+
     def capture_the_flags(*args, **kwargs):
       for arg in args:
         flags.add(arg)
@@ -100,28 +127,13 @@ def is_bootstrap_option(arg):
     # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
     # Stop before '--' since args after that are pass-through and may have duplicate names to our
     # bootstrap options.
-    bargs = list(filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args)))
+    bargs = tuple(filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', args)))
 
-    config_file_paths = self.get_config_file_paths(env=self._env, args=self._args)
-    config_files_products = yield config_file_paths
+    config_file_paths = cls.get_config_file_paths(env=env, args=args)
+    config_files_products = [filecontent_for(p) for p in config_file_paths]
     pre_bootstrap_config = Config.load_file_contents(config_files_products)
 
-    def bootstrap_options_from_config(config):
-      bootstrap_options = Options.create(
-        env=self._env,
-        config=config,
-        known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()],
-        args=bargs,
-      )
-
-      def register_global(*args, **kwargs):
-        ## Only use of Options.register?
-        bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
-
-      GlobalOptionsRegistrar.register_bootstrap_options(register_global)
-      return bootstrap_options
-
-    initial_bootstrap_options = bootstrap_options_from_config(pre_bootstrap_config)
+    initial_bootstrap_options = cls.parse_bootstrap_options(env, bargs, pre_bootstrap_config)
     bootstrap_option_values = initial_bootstrap_options.for_global_scope()
 
     # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
@@ -132,41 +144,45 @@ def register_global(*args, **kwargs):
       existing_rcfiles = list(filter(os.path.exists, rcfiles))
       full_configpaths.extend(existing_rcfiles)
 
-    full_config_files_products = yield full_configpaths
-    self._post_bootstrap_config = Config.load_file_contents(
+    full_config_files_products = [filecontent_for(p) for p in full_configpaths]
+    post_bootstrap_config = Config.load_file_contents(
       full_config_files_products,
       seed_values=bootstrap_option_values
     )
 
-    # Now recompute the bootstrap options with the full config. This allows us to pick up
-    # bootstrap values (such as backends) from a config override file, for example.
-    self._bootstrap_options = bootstrap_options_from_config(self._post_bootstrap_config)
+    env_tuples = tuple(sorted(iteritems(env), key=lambda x: x[0]))
+    return cls(env_tuples=env_tuples, bootstrap_args=bargs, args=args, config=post_bootstrap_config)
 
-  def construct_and_set_bootstrap_options(self):
-    """Populates the internal bootstrap_options cache."""
-    def filecontent_for(path):
-      with open(path, 'rb') as fh:
-        path = ensure_text(path)
-        return FileContent(path, fh.read())
-
-    # N.B. This adaptor is meant to simulate how we would co-operatively invoke options bootstrap
-    # via an `@rule` after we have a solution in place for producing `FileContent` of abspaths.
-    producer = self.produce_and_set_bootstrap_options()
-    next_item = None
-    while 1:
-      try:
-        files = next_item or next(producer)
-        next_item = producer.send([filecontent_for(f) for f in files])
-      except StopIteration:
-        break
+  @memoized_property
+  def env(self):
+    return dict(self.env_tuples)
+
+  @memoized_property
+  def bootstrap_options(self):
+    """The post-bootstrap options, computed from the env, args, and fully discovered Config.
+
+    Re-computing options after Config has been fully expanded allows us to pick up bootstrap values
+    (such as backends) from a config override file, for example.
+
+    Because this can be computed from the in-memory representation of these values, it is not part
+    of the object's identity.
+    """
+    return self.parse_bootstrap_options(self.env, self.bootstrap_args, self.config)
 
   def get_bootstrap_options(self):
     """:returns: an Options instance that only knows about the bootstrap options.
     :rtype: :class:`Options`
     """
-    if not self._bootstrap_options:
-      self.construct_and_set_bootstrap_options()
-    return self._bootstrap_options
+    return self.bootstrap_options
+
+  @memoized_method
+  def _full_options(self, known_scope_infos):
+    bootstrap_option_values = self.get_bootstrap_options().for_global_scope()
+    return Options.create(self.env,
+                          self.config,
+                          known_scope_infos,
+                          args=self.args,
+                          bootstrap_option_values=bootstrap_option_values)
 
   def get_full_options(self, known_scope_infos):
     """Get the full Options instance bootstrapped by this object for the given known scopes.
@@ -176,17 +192,7 @@ def get_full_options(self, known_scope_infos):
               scopes.
     :rtype: :class:`Options`
     """
-    key = frozenset(sorted(known_scope_infos))
-    if key not in self._full_options:
-      # Note: Don't inline this into the Options() call, as this populates
-      # self._post_bootstrap_config, which is another argument to that call.
-      bootstrap_option_values = self.get_bootstrap_options().for_global_scope()
-      self._full_options[key] = Options.create(self._env,
-                                               self._post_bootstrap_config,
-                                               known_scope_infos,
-                                               args=self._args,
-                                               bootstrap_option_values=bootstrap_option_values)
-    return self._full_options[key]
+    return self._full_options(tuple(sorted(set(known_scope_infos))))
 
   def verify_configs_against_options(self, options):
     """Verify all loaded configs have correct scopes and options.
@@ -195,7 +201,7 @@ def verify_configs_against_options(self, options):
     :return: None.
     """
     error_log = []
-    for config in self._post_bootstrap_config.configs():
+    for config in self.config.configs():
       for section in config.sections():
         if section == GLOBAL_SCOPE_CONFIG_SECTION:
           scope = GLOBAL_SCOPE

src/python/pants/pantsd/pants_daemon.py

@@ -130,15 +130,16 @@ def create(cls, bootstrap_options=None, full_init=True):
                              initialize the engine). See the impl of `maybe_launch` for an example
                              of the intended usage.
       """
-      bootstrap_options = bootstrap_options or cls._parse_bootstrap_options()
+      # TODO: This method should receive an OptionsBootstrapper instead, to avoid re-parsing.
+      options_bootstrapper = OptionsBootstrapper.create() if full_init or not bootstrap_options else None
+      bootstrap_options = bootstrap_options or options_bootstrapper.bootstrap_options
       bootstrap_options_values = bootstrap_options.for_global_scope()
       # TODO: https://github.com/pantsbuild/pants/issues/3479
       watchman = WatchmanLauncher.create(bootstrap_options_values).watchman
 
       if full_init:
         build_root = get_buildroot()
         native = Native.create(bootstrap_options_values)
-        options_bootstrapper = OptionsBootstrapper()
         build_config = BuildConfigInitializer.get(options_bootstrapper)
         legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(native,
                                                                       bootstrap_options_values,
@@ -164,10 +165,6 @@ def create(cls, bootstrap_options=None, full_init=True):
         bootstrap_options=bootstrap_options
       )
 
-    @staticmethod
-    def _parse_bootstrap_options():
-      return OptionsBootstrapper().get_bootstrap_options()
-
     @staticmethod
     def _setup_services(build_root, bootstrap_options, legacy_graph_scheduler, watchman):
       """Initialize pantsd services.

tests/python/pants_test/base_test.py

@@ -466,7 +466,7 @@ def get_bootstrap_options(self, cli_options=()):
     """
     # Can't parse any options without a pants.ini.
     self.create_file('pants.ini')
-    return OptionsBootstrapper(args=cli_options).get_bootstrap_options().for_global_scope()
+    return OptionsBootstrapper.create(args=cli_options).bootstrap_options.for_global_scope()
 
   class LoggingRecorder(object):
     """Simple logging handler to record warnings."""

tests/python/pants_test/engine/legacy/test_graph.py

@@ -45,7 +45,8 @@ def _make_setup_args(self, specs):
   def _default_build_config(self, build_file_aliases=None):
     # TODO: Get default BuildFileAliases by extending BaseTest post
     #   https://github.com/pantsbuild/pants/issues/4401
-    build_config = BuildConfigInitializer.get(OptionsBootstrapper())
+    options_bootstrapper = OptionsBootstrapper.create()
+    build_config = BuildConfigInitializer.get(options_bootstrapper)
     if build_file_aliases:
       build_config.register_aliases(build_file_aliases)
     return build_config

tests/python/pants_test/init/test_options_initializer.py

@@ -14,15 +14,15 @@
 
 class OptionsInitializerTest(unittest.TestCase):
   def test_invalid_version(self):
-    options_bootstrapper = OptionsBootstrapper(args=['--pants-version=99.99.9999'])
+    options_bootstrapper = OptionsBootstrapper.create(args=['--pants-version=99.99.9999'])
     build_config = BuildConfigInitializer.get(options_bootstrapper)
 
     with self.assertRaises(BuildConfigurationError):
       OptionsInitializer.create(options_bootstrapper, build_config)
 
   def test_global_options_validation(self):
     # Specify an invalid combination of options.
-    ob = OptionsBootstrapper(args=['--loop', '--v1'])
+    ob = OptionsBootstrapper.create(args=['--loop', '--v1'])
     build_config = BuildConfigInitializer.get(ob)
     with self.assertRaises(OptionsError) as exc:
       OptionsInitializer.create(ob, build_config)

tests/python/pants_test/init/test_plugin_resolver.py

@@ -76,7 +76,7 @@ def provide_chroot(existing):
         touch(configpath)
       args = ["--pants-config-files=['{}']".format(configpath)]
 
-      options_bootstrapper = OptionsBootstrapper(env=env, args=args)
+      options_bootstrapper = OptionsBootstrapper.create(env=env, args=args)
       plugin_resolver = PluginResolver(options_bootstrapper)
       cache_dir = plugin_resolver.plugin_cache_dir
       yield plugin_resolver.resolve(WorkingSet(entries=[])), root_dir, repo_dir, cache_dir

tests/python/pants_test/option/test_options.py

@@ -696,8 +696,8 @@ def test_file_spec_args(self):
       cmdline = './pants --target-spec-file={filename} --pants-config-files="[]" ' \
                 'compile morx:tgt fleem:tgt'.format(
         filename=tmp.name)
-      bootstrapper = OptionsBootstrapper(args=shlex.split(cmdline))
-      bootstrap_options = bootstrapper.get_bootstrap_options().for_global_scope()
+      bootstrapper = OptionsBootstrapper.create(args=shlex.split(cmdline))
+      bootstrap_options = bootstrapper.bootstrap_options.for_global_scope()
       options = self._parse(cmdline, bootstrap_option_values=bootstrap_options)
       sorted_specs = sorted(options.target_specs)
       self.assertEqual(['bar', 'fleem:tgt', 'foo', 'morx:tgt'], sorted_specs)