diff --git a/build-support/bin/release.sh b/build-support/bin/release.sh index 909aa653161..44e9b67eeaf 100755 --- a/build-support/bin/release.sh +++ b/build-support/bin/release.sh @@ -128,6 +128,7 @@ function execute_packaged_pants_with_internal_backends() { 'pants.backend.docgen',\ 'pants.backend.graph_info',\ 'pants.backend.jvm',\ + 'pants.backend.native',\ 'pants.backend.project_info',\ 'pants.backend.python',\ 'internal_backend.repositories',\ diff --git a/src/python/pants/backend/native/BUILD b/src/python/pants/backend/native/BUILD new file mode 100644 index 00000000000..1a7cb627e92 --- /dev/null +++ b/src/python/pants/backend/native/BUILD @@ -0,0 +1,12 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_library( + dependencies=[ + 'src/python/pants/backend/native/subsystems', + 'src/python/pants/engine:rules', + 'src/python/pants/util:objects', + 'src/python/pants/util:osutil', + ], +) diff --git a/src/python/pants/backend/native/register.py b/src/python/pants/backend/native/register.py new file mode 100644 index 00000000000..9d5659068b4 --- /dev/null +++ b/src/python/pants/backend/native/register.py @@ -0,0 +1,24 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.backend.native.subsystems.native_toolchain import NativeToolchain +from pants.engine.rules import RootRule, SingletonRule +from pants.util.objects import datatype +from pants.util.osutil import get_normalized_os_name + + +class Platform(datatype(['normalized_os_name'])): + + def __new__(cls): + return super(Platform, cls).__new__(cls, get_normalized_os_name()) + + +def rules(): + return [ + RootRule(NativeToolchain), + SingletonRule(Platform, Platform()), + ] diff --git a/src/python/pants/backend/python/register.py b/src/python/pants/backend/python/register.py index 3a84d143392..adddea4fe57 100644 --- a/src/python/pants/backend/python/register.py +++ b/src/python/pants/backend/python/register.py @@ -25,7 +25,7 @@ from pants.backend.python.tasks.python_run import PythonRun from pants.backend.python.tasks.resolve_requirements import ResolveRequirements from pants.backend.python.tasks.select_interpreter import SelectInterpreter -from pants.backend.python.tasks.setup_py import SetupPy +from pants.backend.python.tasks.setup_py import SetupPy, create_setup_py_rules from pants.build_graph.build_file_aliases import BuildFileAliases from pants.build_graph.resources import Resources from pants.goal.task_registrar import TaskRegistrar as task @@ -65,3 +65,7 @@ def register_goals(): task(name='setup-py', action=SetupPy).install() task(name='py', action=PythonBinaryCreate).install('binary') task(name='isort', action=IsortPythonTask).install('fmt') + + +def rules(): + return create_setup_py_rules() diff --git a/src/python/pants/backend/python/tasks/BUILD b/src/python/pants/backend/python/tasks/BUILD index b73a5379fec..6bf3f83183c 100644 --- a/src/python/pants/backend/python/tasks/BUILD +++ b/src/python/pants/backend/python/tasks/BUILD @@ -18,6 +18,8 @@ python_library( 'src/python/pants/base:fingerprint_strategy', 'src/python/pants/base:hash_utils', 'src/python/pants/base:specs', + 'src/python/pants/engine:rules', + 'src/python/pants/engine:selectors', 'src/python/pants/build_graph', 'src/python/pants/invalidation', 'src/python/pants/python', diff --git a/src/python/pants/backend/python/tasks/build_local_python_distributions.py b/src/python/pants/backend/python/tasks/build_local_python_distributions.py index 734aff9aa1d..faae95068f4 100644 --- a/src/python/pants/backend/python/tasks/build_local_python_distributions.py +++ b/src/python/pants/backend/python/tasks/build_local_python_distributions.py @@ -16,13 +16,13 @@ from pants.backend.python.python_requirement import PythonRequirement from pants.backend.python.targets.python_distribution import PythonDistribution from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary -from pants.backend.python.tasks.setup_py import SetupPyRunner +from pants.backend.python.tasks.setup_py import SetupPyInvocationEnvironment, SetupPyRunner from pants.base.build_environment import get_buildroot from pants.base.exceptions import TargetDefinitionException, TaskError from pants.base.fingerprint_strategy import DefaultFingerprintStrategy from pants.build_graph.address import Address from pants.task.task import Task -from pants.util.contextutil import environment_as, get_joined_path +from pants.util.contextutil import environment_as from pants.util.dirutil import safe_mkdir from pants.util.memo import memoized_method @@ -103,6 +103,11 @@ def _copy_sources(self, dist_tgt, dist_target_dir): src_relative_to_target_base) shutil.copyfile(abs_src_path, src_rel_to_results_dir) + def _request_single(self, product, subject): + # This is not supposed to be exposed to Tasks yet -- see #4769 to track the + # status of exposing v2 products in v1 tasks. + return self.context._scheduler.product_request(product, [subject])[0] + # FIXME(cosmicexplorer): We should be isolating the path to just our provided # toolchain, but this causes errors in Travis because distutils looks for # "x86_64-linux-gnu-gcc" when linking native extensions. We almost definitely @@ -111,11 +116,9 @@ def _copy_sources(self, dist_tgt, dist_target_dir): # compiler installed. Right now we just put our tools at the end of the PATH. @contextmanager def _setup_py_invocation_environment(self): - native_toolchain = self._native_toolchain_instance() - native_toolchain_path_entries = native_toolchain.path_entries() - appended_native_toolchain_path = get_joined_path( - native_toolchain_path_entries, os.environ.copy()) - with environment_as(PATH=appended_native_toolchain_path): + setup_py_env = self._request_single( + SetupPyInvocationEnvironment, self._native_toolchain_instance()) + with environment_as(**setup_py_env.as_env_dict()): yield def _create_dist(self, dist_tgt, dist_target_dir, interpreter): diff --git a/src/python/pants/backend/python/tasks/setup_py.py b/src/python/pants/backend/python/tasks/setup_py.py index 3f173669c02..a5582f59ddb 100644 --- a/src/python/pants/backend/python/tasks/setup_py.py +++ b/src/python/pants/backend/python/tasks/setup_py.py @@ -19,6 +19,7 @@ from twitter.common.collections import OrderedSet from twitter.common.dirutil.chroot import Chroot +from pants.backend.native.subsystems.native_toolchain import NativeToolchain from pants.backend.python.targets.python_binary import PythonBinary from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants.backend.python.targets.python_target import PythonTarget @@ -29,10 +30,14 @@ from pants.build_graph.address_lookup_error import AddressLookupError from pants.build_graph.build_graph import sort_targets from pants.build_graph.resources import Resources +from pants.engine.rules import rule +from pants.engine.selectors import Select from pants.task.task import Task +from pants.util.contextutil import get_joined_path from pants.util.dirutil import safe_rmtree, safe_walk from pants.util.memo import memoized_property from pants.util.meta import AbstractClass +from pants.util.objects import datatype SETUP_BOILERPLATE = """ @@ -74,6 +79,21 @@ def _setup_command(self): return self.__setup_command +class SetupPyInvocationEnvironment(datatype(['joined_path'])): + + def as_env_dict(self): + return { + 'PATH': self.joined_path, + } + + +@rule(SetupPyInvocationEnvironment, [Select(NativeToolchain)]) +def get_setup_py_env(native_toolchain): + joined_path = get_joined_path( + native_toolchain.path_entries(), os.environ.copy()) + return SetupPyInvocationEnvironment(joined_path) + + class TargetAncestorIterator(object): """Supports iteration of target ancestor lineages.""" @@ -640,3 +660,9 @@ def create(exported_python_target): setup_runner = SetupPyRunner(setup_dir, self._run, interpreter=interpreter) setup_runner.run() python_dists[exported_python_target] = setup_dir + + +def create_setup_py_rules(): + return [ + get_setup_py_env, + ] diff --git a/src/python/pants/bin/engine_initializer.py b/src/python/pants/bin/engine_initializer.py index 8ddbc460265..3b386736154 100644 --- a/src/python/pants/bin/engine_initializer.py +++ b/src/python/pants/bin/engine_initializer.py @@ -109,6 +109,11 @@ def create_build_graph(self, target_roots, build_root=None): class EngineInitializer(object): """Constructs the components necessary to run the v2 engine with v1 BuildGraph compatibility.""" + @staticmethod + def get_default_build_file_aliases(): + _, build_config = OptionsInitializer(OptionsBootstrapper()).setup(init_logging=False) + return build_config.registered_aliases() + @staticmethod def setup_legacy_graph(pants_ignore_patterns, workdir, @@ -116,6 +121,7 @@ def setup_legacy_graph(pants_ignore_patterns, build_root=None, native=None, build_file_aliases=None, + rules=None, build_ignore_patterns=None, exclude_target_regexps=None, subproject_roots=None, @@ -146,8 +152,10 @@ def setup_legacy_graph(pants_ignore_patterns, scm = get_scm() if not build_file_aliases: - _, build_config = OptionsInitializer(OptionsBootstrapper()).setup(init_logging=False) - build_file_aliases = build_config.registered_aliases() + build_file_aliases = EngineInitializer.get_default_build_file_aliases() + + if not rules: + rules = [] symbol_table = LegacySymbolTable(build_file_aliases) @@ -173,7 +181,8 @@ def setup_legacy_graph(pants_ignore_patterns, create_legacy_graph_tasks(symbol_table) + create_fs_rules() + create_graph_rules(address_mapper, symbol_table) + - create_process_rules() + create_process_rules() + + rules ) scheduler = LocalScheduler(workdir, dict(), tasks, project_tree, native, include_trace_on_error=include_trace_on_error) diff --git a/src/python/pants/bin/goal_runner.py b/src/python/pants/bin/goal_runner.py index c6cb1552685..2303099d557 100644 --- a/src/python/pants/bin/goal_runner.py +++ b/src/python/pants/bin/goal_runner.py @@ -107,6 +107,7 @@ def _init_graph(self, self._global_options.build_file_imports, native=native, build_file_aliases=self._build_config.registered_aliases(), + rules=self._build_config.rules(), build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_build_roots, diff --git a/src/python/pants/build_graph/build_configuration.py b/src/python/pants/build_graph/build_configuration.py index 985af317fe5..0c1d1d88b97 100644 --- a/src/python/pants/build_graph/build_configuration.py +++ b/src/python/pants/build_graph/build_configuration.py @@ -33,6 +33,7 @@ def __init__(self): self._exposed_object_by_alias = {} self._exposed_context_aware_object_factory_by_alias = {} self._subsystems = set() + self._rules = [] def registered_aliases(self): """Return the registered aliases exposed in BUILD files. @@ -129,6 +130,25 @@ def subsystems(self): """ return self._subsystems + def register_rules(self, rules): + """Registers the given rules. + + :param rules: The rules to register. + :type rules: :class:`collections.Iterable` containing + :class:`pants.engine.rules.Rule` instances. + """ + + if not isinstance(rules, Iterable): + raise TypeError('The rules must be an iterable, given {!r}'.format(rules)) + self._rules.extend(rules) + + def rules(self): + """Returns the registered rules. + + :rtype list + """ + return self._rules + @memoized_method def _get_addressable_factory(self, target_type, alias): return TargetAddressable.factory(target_type=target_type, alias=alias) diff --git a/src/python/pants/init/BUILD b/src/python/pants/init/BUILD index bc11d17f875..c8a7556e17c 100644 --- a/src/python/pants/init/BUILD +++ b/src/python/pants/init/BUILD @@ -37,5 +37,6 @@ target( 'src/python/pants/backend/jvm:plugin', 'src/python/pants/backend/project_info:plugin', 'src/python/pants/backend/python:plugin', + 'src/python/pants/backend/native', ], ) diff --git a/src/python/pants/init/extension_loader.py b/src/python/pants/init/extension_loader.py index 177aae345c1..0bdfa39fd3d 100644 --- a/src/python/pants/init/extension_loader.py +++ b/src/python/pants/init/extension_loader.py @@ -144,4 +144,8 @@ def invoke_entrypoint(name): if subsystems: build_configuration.register_subsystems(subsystems) + rules = invoke_entrypoint('rules') + if rules: + build_configuration.register_rules(rules) + invoke_entrypoint('register_goals') diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py index 684a127d83b..5ef6ec4d364 100644 --- a/src/python/pants/option/global_options.py +++ b/src/python/pants/option/global_options.py @@ -71,6 +71,7 @@ def register_bootstrap_options(cls, register): default=['pants.backend.graph_info', 'pants.backend.python', 'pants.backend.jvm', + 'pants.backend.native', 'pants.backend.codegen.antlr.java', 'pants.backend.codegen.antlr.python', 'pants.backend.codegen.jaxb', diff --git a/src/python/pants/util/osutil.py b/src/python/pants/util/osutil.py index 6901cfc8622..608942cd2eb 100644 --- a/src/python/pants/util/osutil.py +++ b/src/python/pants/util/osutil.py @@ -59,5 +59,13 @@ def normalize_os_name(os_name): return os_name +def get_normalized_os_name(): + return normalize_os_name(get_os_name()) + + +def all_normalized_os_names(): + return OS_ALIASES.keys() + + def known_os_names(): return reduce(set.union, OS_ALIASES.values()) diff --git a/tests/python/pants_test/backend/python/tasks/BUILD b/tests/python/pants_test/backend/python/tasks/BUILD index 5964ab56b6c..6207c43fe29 100644 --- a/tests/python/pants_test/backend/python/tasks/BUILD +++ b/tests/python/pants_test/backend/python/tasks/BUILD @@ -26,6 +26,8 @@ python_tests( '3rdparty/python:coverage', '3rdparty/python:mock', '3rdparty/python:pex', + 'src/python/pants/backend/native', + 'src/python/pants/backend/python:plugin', 'src/python/pants/backend/python/subsystems', 'src/python/pants/backend/python/targets', 'src/python/pants/backend/python/tasks', @@ -42,6 +44,7 @@ python_tests( 'src/python/pants/util:contextutil', 'src/python/pants/util:dirutil', 'src/python/pants/util:process_handler', + 'tests/python/pants_test/engine:scheduler_test_base', 'tests/python/pants_test/subsystem:subsystem_utils', 'tests/python/pants_test/tasks:task_test_base', ], diff --git a/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py b/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py index f676bc3d755..9b3cdd1b1f0 100644 --- a/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py +++ b/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py @@ -7,13 +7,16 @@ from textwrap import dedent +from pants.backend.native.register import rules as native_backend_rules +from pants.backend.python.register import rules as python_backend_rules from pants.backend.python.targets.python_distribution import PythonDistribution from pants.backend.python.tasks.build_local_python_distributions import \ BuildLocalPythonDistributions from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase +from pants_test.engine.scheduler_test_base import SchedulerTestBase -class TestBuildLocalPythonDistributions(PythonTaskTestBase): +class TestBuildLocalPythonDistributions(PythonTaskTestBase, SchedulerTestBase): @classmethod def task_type(cls): return BuildLocalPythonDistributions @@ -43,9 +46,18 @@ def setUp(self): target_type=PythonDistribution, sources=sources) + def _scheduling_context(self, **kwargs): + rules = ( + native_backend_rules() + + python_backend_rules() + ) + scheduler = self.mk_scheduler(rules=rules) + return self.context(scheduler=scheduler, **kwargs) + def test_python_create_distributions(self): - context = self.context(target_roots=[self.python_dist_tgt], - for_task_types=[BuildLocalPythonDistributions]) + context = self._scheduling_context( + target_roots=[self.python_dist_tgt], + for_task_types=[BuildLocalPythonDistributions]) self.assertEquals([self.python_dist_tgt], context.build_graph.targets()) python_create_distributions_task = self.create_task(context) python_create_distributions_task.execute() diff --git a/tests/python/pants_test/base_test.py b/tests/python/pants_test/base_test.py index f3dca325dd7..6fed162c55b 100644 --- a/tests/python/pants_test/base_test.py +++ b/tests/python/pants_test/base_test.py @@ -276,7 +276,7 @@ def set_options_for_scope(self, scope, **kwargs): def context(self, for_task_types=None, for_subsystems=None, options=None, target_roots=None, console_outstream=None, workspace=None, - **kwargs): + scheduler=None, **kwargs): """ :API: public @@ -330,7 +330,8 @@ def context(self, for_task_types=None, for_subsystems=None, options=None, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, - workspace=workspace) + workspace=workspace, + scheduler=scheduler) return context def tearDown(self): diff --git a/tests/python/pants_test/engine/scheduler_test_base.py b/tests/python/pants_test/engine/scheduler_test_base.py index 0c77990f967..24f1ac72803 100644 --- a/tests/python/pants_test/engine/scheduler_test_base.py +++ b/tests/python/pants_test/engine/scheduler_test_base.py @@ -60,6 +60,9 @@ def mk_scheduler(self, self._native, include_trace_on_error=include_trace_on_error) + def context_with_scheduler(self, scheduler, *args, **kwargs): + return self.context(*args, scheduler=scheduler, **kwargs) + def execute(self, scheduler, product, *subjects): """Runs an ExecutionRequest for the given product and subjects, and returns the result value.""" request = scheduler.execution_request([product], subjects) diff --git a/tests/python/pants_test/init/BUILD b/tests/python/pants_test/init/BUILD index 2f845a123d4..9e80be5ccce 100644 --- a/tests/python/pants_test/init/BUILD +++ b/tests/python/pants_test/init/BUILD @@ -9,14 +9,19 @@ python_tests( '3rdparty/python:pex', '3rdparty/python:setuptools', 'src/python/pants/base:exceptions', + 'src/python/pants/bin', 'src/python/pants/build_graph', + 'src/python/pants/engine:rules', + 'src/python/pants/engine:selectors', 'src/python/pants/goal', 'src/python/pants/goal:task_registrar', 'src/python/pants/init', 'src/python/pants/option', + 'src/python/pants/pantsd:pants_daemon', 'src/python/pants/subsystem', 'src/python/pants/util:contextutil', 'src/python/pants/util:dirutil', + 'src/python/pants/util:objects', 'tests/python/pants_test:base_test', 'tests/python/pants_test/subsystem:subsystem_utils' ], diff --git a/tests/python/pants_test/init/test_extension_loader.py b/tests/python/pants_test/init/test_extension_loader.py index c39b60da928..e9882094a9f 100644 --- a/tests/python/pants_test/init/test_extension_loader.py +++ b/tests/python/pants_test/init/test_extension_loader.py @@ -18,12 +18,15 @@ from pants.build_graph.build_configuration import BuildConfiguration from pants.build_graph.build_file_aliases import BuildFileAliases from pants.build_graph.target import Target +from pants.engine.rules import RootRule, rule +from pants.engine.selectors import Select from pants.goal.goal import Goal from pants.goal.task_registrar import TaskRegistrar from pants.init.extension_loader import (PluginLoadOrderError, PluginNotFound, load_backend, load_backends_and_plugins, load_plugins) from pants.subsystem.subsystem import Subsystem from pants.task.task import Task +from pants.util.objects import datatype class MockMetadata(EmptyProvider): @@ -80,6 +83,19 @@ class DummyTask(Task): def execute(self): return 42 +class RootType(datatype(['value'])): + pass + + +class WrapperType(datatype(['value'])): + pass + + +@rule(WrapperType, [Select(RootType)]) +def example_rule(root_type): + yield WrapperType(root_type.value) + + class LoaderTest(unittest.TestCase): def setUp(self): @@ -93,7 +109,7 @@ def tearDown(self): @contextmanager def create_register(self, build_file_aliases=None, register_goals=None, global_subsystems=None, - module_name='register'): + rules=None, module_name='register'): package_name = b'__test_package_{0}'.format(uuid.uuid4().hex) self.assertFalse(package_name in sys.modules) @@ -113,6 +129,7 @@ def register_entrypoint(function_name, function): register_entrypoint('build_file_aliases', build_file_aliases) register_entrypoint('global_subsystems', global_subsystems) register_entrypoint('register_goals', register_goals) + register_entrypoint('rules', rules) yield package_name finally: @@ -125,6 +142,7 @@ def assert_empty_aliases(self): self.assertEqual(0, len(registered_aliases.objects)) self.assertEqual(0, len(registered_aliases.context_aware_object_factories)) self.assertEqual(self.build_configuration.subsystems(), set()) + self.assertEqual(0, len(self.build_configuration.rules())) def test_load_valid_empty(self): with self.create_register() as backend_package: @@ -292,6 +310,14 @@ def global_subsystems(): self.assertEqual(self.build_configuration.subsystems(), {DummySubsystem1, DummySubsystem2}) + def test_rules(self): + def rules(): + return [example_rule, RootRule(RootType)] + with self.create_register(rules=rules) as backend_package: + load_backend(self.build_configuration, backend_package) + self.assertEqual(self.build_configuration.rules(), + [example_rule, RootRule(RootType)]) + def test_backend_plugin_ordering(self): def reg_alias(): return BuildFileAliases(targets={'override-alias': DummyTarget2})