Expose rules from language backends with an application to python_dist() creation (#5747)

### Problem

It's not possible to use rules defined outside of `src/python/pants/engine` in the rest of Pants.

### Solution

Modify [`load_backend()` in `extension_loader.py`](https://github.com/cosmicexplorer/pants/blob/c9dab55621863af87c3999c09f980b594f73ef78/src/python/pants/init/extension_loader.py#L113) to understand a `rules` entrypoint. If a backend has a method named `rules()` defined in its register.py, that method will be invoked to get a list of rules to make the scheduler with.

### Result

We can now use rules defined in language backends. [I created a `register.py` in the native backend](https://github.com/cosmicexplorer/pants/blob/c9dab55621863af87c3999c09f980b594f73ef78/src/python/pants/backend/native/register.py) so that we can test that this works ([it's consumed in BuildLocalPythonDistributions](https://github.com/cosmicexplorer/pants/blob/c9dab55621863af87c3999c09f980b594f73ef78/src/python/pants/backend/python/tasks/build_local_python_distributions.py#L116)). I have a diff that expands the native backend which depends on this functionality.

Note: Extending this to cover plugins as well is trivial (make the exact same change, but to `load_plugins()` instead).

Author: cosmicexplorer

build-support/bin/release.sh

@@ -128,6 +128,7 @@ function execute_packaged_pants_with_internal_backends() {
         'pants.backend.docgen',\
         'pants.backend.graph_info',\
         'pants.backend.jvm',\
+        'pants.backend.native',\
         'pants.backend.project_info',\
         'pants.backend.python',\
         'internal_backend.repositories',\

src/python/pants/backend/native/BUILD

@@ -0,0 +1,12 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(
+  dependencies=[
+    'src/python/pants/backend/native/subsystems',
+    'src/python/pants/engine:rules',
+    'src/python/pants/util:objects',
+    'src/python/pants/util:osutil',
+  ],
+)

src/python/pants/backend/native/register.py

@@ -0,0 +1,24 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.backend.native.subsystems.native_toolchain import NativeToolchain
+from pants.engine.rules import RootRule, SingletonRule
+from pants.util.objects import datatype
+from pants.util.osutil import get_normalized_os_name
+
+
+class Platform(datatype(['normalized_os_name'])):
+
+  def __new__(cls):
+    return super(Platform, cls).__new__(cls, get_normalized_os_name())
+
+
+def rules():
+  return [
+    RootRule(NativeToolchain),
+    SingletonRule(Platform, Platform()),
+  ]

src/python/pants/backend/python/register.py

@@ -25,7 +25,7 @@
 from pants.backend.python.tasks.python_run import PythonRun
 from pants.backend.python.tasks.resolve_requirements import ResolveRequirements
 from pants.backend.python.tasks.select_interpreter import SelectInterpreter
-from pants.backend.python.tasks.setup_py import SetupPy
+from pants.backend.python.tasks.setup_py import SetupPy, create_setup_py_rules
 from pants.build_graph.build_file_aliases import BuildFileAliases
 from pants.build_graph.resources import Resources
 from pants.goal.task_registrar import TaskRegistrar as task
@@ -65,3 +65,7 @@ def register_goals():
   task(name='setup-py', action=SetupPy).install()
   task(name='py', action=PythonBinaryCreate).install('binary')
   task(name='isort', action=IsortPythonTask).install('fmt')
+
+
+def rules():
+  return create_setup_py_rules()

src/python/pants/backend/python/tasks/BUILD

@@ -18,6 +18,8 @@ python_library(
     'src/python/pants/base:fingerprint_strategy',
     'src/python/pants/base:hash_utils',
     'src/python/pants/base:specs',
+    'src/python/pants/engine:rules',
+    'src/python/pants/engine:selectors',
     'src/python/pants/build_graph',
     'src/python/pants/invalidation',
     'src/python/pants/python',

src/python/pants/backend/python/tasks/build_local_python_distributions.py

@@ -16,13 +16,13 @@
 from pants.backend.python.python_requirement import PythonRequirement
 from pants.backend.python.targets.python_distribution import PythonDistribution
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
-from pants.backend.python.tasks.setup_py import SetupPyRunner
+from pants.backend.python.tasks.setup_py import SetupPyInvocationEnvironment, SetupPyRunner
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TargetDefinitionException, TaskError
 from pants.base.fingerprint_strategy import DefaultFingerprintStrategy
 from pants.build_graph.address import Address
 from pants.task.task import Task
-from pants.util.contextutil import environment_as, get_joined_path
+from pants.util.contextutil import environment_as
 from pants.util.dirutil import safe_mkdir
 from pants.util.memo import memoized_method
 
@@ -103,6 +103,11 @@ def _copy_sources(self, dist_tgt, dist_target_dir):
                                   src_relative_to_target_base)
       shutil.copyfile(abs_src_path, src_rel_to_results_dir)
 
+  def _request_single(self, product, subject):
+    # This is not supposed to be exposed to Tasks yet -- see #4769 to track the
+    # status of exposing v2 products in v1 tasks.
+    return self.context._scheduler.product_request(product, [subject])[0]
+
   # FIXME(cosmicexplorer): We should be isolating the path to just our provided
   # toolchain, but this causes errors in Travis because distutils looks for
   # "x86_64-linux-gnu-gcc" when linking native extensions. We almost definitely
@@ -111,11 +116,9 @@ def _copy_sources(self, dist_tgt, dist_target_dir):
   # compiler installed. Right now we just put our tools at the end of the PATH.
   @contextmanager
   def _setup_py_invocation_environment(self):
-    native_toolchain = self._native_toolchain_instance()
-    native_toolchain_path_entries = native_toolchain.path_entries()
-    appended_native_toolchain_path = get_joined_path(
-      native_toolchain_path_entries, os.environ.copy())
-    with environment_as(PATH=appended_native_toolchain_path):
+    setup_py_env = self._request_single(
+      SetupPyInvocationEnvironment, self._native_toolchain_instance())
+    with environment_as(**setup_py_env.as_env_dict()):
       yield
 
   def _create_dist(self, dist_tgt, dist_target_dir, interpreter):

src/python/pants/backend/python/tasks/setup_py.py

@@ -19,6 +19,7 @@
 from twitter.common.collections import OrderedSet
 from twitter.common.dirutil.chroot import Chroot
 
+from pants.backend.native.subsystems.native_toolchain import NativeToolchain
 from pants.backend.python.targets.python_binary import PythonBinary
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
 from pants.backend.python.targets.python_target import PythonTarget
@@ -29,10 +30,14 @@
 from pants.build_graph.address_lookup_error import AddressLookupError
 from pants.build_graph.build_graph import sort_targets
 from pants.build_graph.resources import Resources
+from pants.engine.rules import rule
+from pants.engine.selectors import Select
 from pants.task.task import Task
+from pants.util.contextutil import get_joined_path
 from pants.util.dirutil import safe_rmtree, safe_walk
 from pants.util.memo import memoized_property
 from pants.util.meta import AbstractClass
+from pants.util.objects import datatype
 
 
 SETUP_BOILERPLATE = """
@@ -74,6 +79,21 @@ def _setup_command(self):
     return self.__setup_command
 
 
+class SetupPyInvocationEnvironment(datatype(['joined_path'])):
+
+  def as_env_dict(self):
+    return {
+      'PATH': self.joined_path,
+    }
+
+
+@rule(SetupPyInvocationEnvironment, [Select(NativeToolchain)])
+def get_setup_py_env(native_toolchain):
+  joined_path = get_joined_path(
+    native_toolchain.path_entries(), os.environ.copy())
+  return SetupPyInvocationEnvironment(joined_path)
+
+
 class TargetAncestorIterator(object):
   """Supports iteration of target ancestor lineages."""
 
@@ -640,3 +660,9 @@ def create(exported_python_target):
           setup_runner = SetupPyRunner(setup_dir, self._run, interpreter=interpreter)
           setup_runner.run()
           python_dists[exported_python_target] = setup_dir
+
+
+def create_setup_py_rules():
+  return [
+    get_setup_py_env,
+  ]

src/python/pants/bin/engine_initializer.py

@@ -109,13 +109,19 @@ def create_build_graph(self, target_roots, build_root=None):
 class EngineInitializer(object):
   """Constructs the components necessary to run the v2 engine with v1 BuildGraph compatibility."""
 
+  @staticmethod
+  def get_default_build_file_aliases():
+    _, build_config = OptionsInitializer(OptionsBootstrapper()).setup(init_logging=False)
+    return build_config.registered_aliases()
+
   @staticmethod
   def setup_legacy_graph(pants_ignore_patterns,
                          workdir,
                          build_file_imports_behavior,
                          build_root=None,
                          native=None,
                          build_file_aliases=None,
+                         rules=None,
                          build_ignore_patterns=None,
                          exclude_target_regexps=None,
                          subproject_roots=None,
@@ -146,8 +152,10 @@ def setup_legacy_graph(pants_ignore_patterns,
     scm = get_scm()
 
     if not build_file_aliases:
-      _, build_config = OptionsInitializer(OptionsBootstrapper()).setup(init_logging=False)
-      build_file_aliases = build_config.registered_aliases()
+      build_file_aliases = EngineInitializer.get_default_build_file_aliases()
+
+    if not rules:
+      rules = []
 
     symbol_table = LegacySymbolTable(build_file_aliases)
 
@@ -173,7 +181,8 @@ def setup_legacy_graph(pants_ignore_patterns,
       create_legacy_graph_tasks(symbol_table) +
       create_fs_rules() +
       create_graph_rules(address_mapper, symbol_table) +
-      create_process_rules()
+      create_process_rules() +
+      rules
     )
 
     scheduler = LocalScheduler(workdir, dict(), tasks, project_tree, native, include_trace_on_error=include_trace_on_error)

src/python/pants/bin/goal_runner.py

@@ -107,6 +107,7 @@ def _init_graph(self,
         self._global_options.build_file_imports,
         native=native,
         build_file_aliases=self._build_config.registered_aliases(),
+        rules=self._build_config.rules(),
         build_ignore_patterns=build_ignore_patterns,
         exclude_target_regexps=exclude_target_regexps,
         subproject_roots=subproject_build_roots,

src/python/pants/build_graph/build_configuration.py

@@ -33,6 +33,7 @@ def __init__(self):
     self._exposed_object_by_alias = {}
     self._exposed_context_aware_object_factory_by_alias = {}
     self._subsystems = set()
+    self._rules = []
 
   def registered_aliases(self):
     """Return the registered aliases exposed in BUILD files.
@@ -129,6 +130,25 @@ def subsystems(self):
     """
     return self._subsystems
 
+  def register_rules(self, rules):
+    """Registers the given rules.
+
+    :param rules: The rules to register.
+    :type rules: :class:`collections.Iterable` containing
+                 :class:`pants.engine.rules.Rule` instances.
+    """
+
+    if not isinstance(rules, Iterable):
+      raise TypeError('The rules must be an iterable, given {!r}'.format(rules))
+    self._rules.extend(rules)
+
+  def rules(self):
+    """Returns the registered rules.
+
+    :rtype list
+    """
+    return self._rules
+
   @memoized_method
   def _get_addressable_factory(self, target_type, alias):
     return TargetAddressable.factory(target_type=target_type, alias=alias)

src/python/pants/init/BUILD

@@ -37,5 +37,6 @@ target(
     'src/python/pants/backend/jvm:plugin',
     'src/python/pants/backend/project_info:plugin',
     'src/python/pants/backend/python:plugin',
+    'src/python/pants/backend/native',
   ],
 )

src/python/pants/init/extension_loader.py

@@ -144,4 +144,8 @@ def invoke_entrypoint(name):
   if subsystems:
     build_configuration.register_subsystems(subsystems)
 
+  rules = invoke_entrypoint('rules')
+  if rules:
+    build_configuration.register_rules(rules)
+
   invoke_entrypoint('register_goals')

src/python/pants/option/global_options.py

@@ -71,6 +71,7 @@ def register_bootstrap_options(cls, register):
              default=['pants.backend.graph_info',
                       'pants.backend.python',
                       'pants.backend.jvm',
+                      'pants.backend.native',
                       'pants.backend.codegen.antlr.java',
                       'pants.backend.codegen.antlr.python',
                       'pants.backend.codegen.jaxb',

src/python/pants/util/osutil.py

@@ -59,5 +59,13 @@ def normalize_os_name(os_name):
   return os_name
 
 
+def get_normalized_os_name():
+  return normalize_os_name(get_os_name())
+
+
+def all_normalized_os_names():
+  return OS_ALIASES.keys()
+
+
 def known_os_names():
   return reduce(set.union, OS_ALIASES.values())

tests/python/pants_test/backend/python/tasks/BUILD

@@ -26,6 +26,8 @@ python_tests(
     '3rdparty/python:coverage',
     '3rdparty/python:mock',
     '3rdparty/python:pex',
+    'src/python/pants/backend/native',
+    'src/python/pants/backend/python:plugin',
     'src/python/pants/backend/python/subsystems',
     'src/python/pants/backend/python/targets',
     'src/python/pants/backend/python/tasks',
@@ -42,6 +44,7 @@ python_tests(
     'src/python/pants/util:contextutil',
     'src/python/pants/util:dirutil',
     'src/python/pants/util:process_handler',
+    'tests/python/pants_test/engine:scheduler_test_base',
     'tests/python/pants_test/subsystem:subsystem_utils',
     'tests/python/pants_test/tasks:task_test_base',
   ],

tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py

@@ -7,13 +7,16 @@
 
 from textwrap import dedent
 
+from pants.backend.native.register import rules as native_backend_rules
+from pants.backend.python.register import rules as python_backend_rules
 from pants.backend.python.targets.python_distribution import PythonDistribution
 from pants.backend.python.tasks.build_local_python_distributions import \
   BuildLocalPythonDistributions
 from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase
+from pants_test.engine.scheduler_test_base import SchedulerTestBase
 
 
-class TestBuildLocalPythonDistributions(PythonTaskTestBase):
+class TestBuildLocalPythonDistributions(PythonTaskTestBase, SchedulerTestBase):
   @classmethod
   def task_type(cls):
     return BuildLocalPythonDistributions
@@ -43,9 +46,18 @@ def setUp(self):
                                             target_type=PythonDistribution,
                                             sources=sources)
 
+  def _scheduling_context(self, **kwargs):
+    rules = (
+      native_backend_rules() +
+      python_backend_rules()
+    )
+    scheduler = self.mk_scheduler(rules=rules)
+    return self.context(scheduler=scheduler, **kwargs)
+
   def test_python_create_distributions(self):
-    context = self.context(target_roots=[self.python_dist_tgt],
-                           for_task_types=[BuildLocalPythonDistributions])
+    context = self._scheduling_context(
+      target_roots=[self.python_dist_tgt],
+      for_task_types=[BuildLocalPythonDistributions])
     self.assertEquals([self.python_dist_tgt], context.build_graph.targets())
     python_create_distributions_task = self.create_task(context)
     python_create_distributions_task.execute()

tests/python/pants_test/base_test.py

@@ -276,7 +276,7 @@ def set_options_for_scope(self, scope, **kwargs):
 
   def context(self, for_task_types=None, for_subsystems=None, options=None,
               target_roots=None, console_outstream=None, workspace=None,
-              **kwargs):
+              scheduler=None, **kwargs):
     """
     :API: public
 
@@ -330,7 +330,8 @@ def context(self, for_task_types=None, for_subsystems=None, options=None,
                                           build_file_parser=self.build_file_parser,
                                           address_mapper=self.address_mapper,
                                           console_outstream=console_outstream,
-                                          workspace=workspace)
+                                          workspace=workspace,
+                                          scheduler=scheduler)
     return context
 
   def tearDown(self):

tests/python/pants_test/engine/scheduler_test_base.py

@@ -60,6 +60,9 @@ def mk_scheduler(self,
                           self._native,
                           include_trace_on_error=include_trace_on_error)
 
+  def context_with_scheduler(self, scheduler, *args, **kwargs):
+    return self.context(*args, scheduler=scheduler, **kwargs)
+
   def execute(self, scheduler, product, *subjects):
     """Runs an ExecutionRequest for the given product and subjects, and returns the result value."""
     request = scheduler.execution_request([product], subjects)

tests/python/pants_test/init/BUILD

@@ -9,14 +9,19 @@ python_tests(
     '3rdparty/python:pex',
     '3rdparty/python:setuptools',
     'src/python/pants/base:exceptions',
+    'src/python/pants/bin',
     'src/python/pants/build_graph',
+    'src/python/pants/engine:rules',
+    'src/python/pants/engine:selectors',
     'src/python/pants/goal',
     'src/python/pants/goal:task_registrar',
     'src/python/pants/init',
     'src/python/pants/option',
+    'src/python/pants/pantsd:pants_daemon',
     'src/python/pants/subsystem',
     'src/python/pants/util:contextutil',
     'src/python/pants/util:dirutil',
+    'src/python/pants/util:objects',
     'tests/python/pants_test:base_test',
     'tests/python/pants_test/subsystem:subsystem_utils'
   ],