diff --git a/src/python/pants/backend/native/tasks/c_compile.py b/src/python/pants/backend/native/tasks/c_compile.py index 6d0c47aba8a8..7ade44dadb60 100644 --- a/src/python/pants/backend/native/tasks/c_compile.py +++ b/src/python/pants/backend/native/tasks/c_compile.py @@ -15,6 +15,8 @@ class CCompile(NativeCompile): + options_scope = 'c-compile' + # Compile only C library targets. source_target_constraint = SubclassesOf(CLibrary) diff --git a/src/python/pants/backend/native/tasks/cpp_compile.py b/src/python/pants/backend/native/tasks/cpp_compile.py index 482c56084ace..96ab90b1b58c 100644 --- a/src/python/pants/backend/native/tasks/cpp_compile.py +++ b/src/python/pants/backend/native/tasks/cpp_compile.py @@ -15,6 +15,8 @@ class CppCompile(NativeCompile): + options_scope = 'cpp-compile' + # Compile only C++ library targets. source_target_constraint = SubclassesOf(CppLibrary) diff --git a/src/python/pants/backend/native/tasks/link_shared_libraries.py b/src/python/pants/backend/native/tasks/link_shared_libraries.py index 81948cc0fd12..394b9dfa3082 100644 --- a/src/python/pants/backend/native/tasks/link_shared_libraries.py +++ b/src/python/pants/backend/native/tasks/link_shared_libraries.py @@ -8,9 +8,10 @@ from pants.backend.native.config.environment import Linker, Platform from pants.backend.native.subsystems.native_toolchain import NativeToolchain +from pants.backend.native.targets.native_artifact import NativeArtifact from pants.backend.native.targets.native_library import NativeLibrary from pants.backend.native.tasks.native_compile import NativeTargetDependencies, ObjectFiles -from pants.backend.native.tasks.native_external_library_fetch import NativeExternalLibraryFetch +from pants.backend.native.tasks.native_external_library_fetch import NativeExternalLibraryFiles from pants.backend.native.tasks.native_task import NativeTask from pants.base.exceptions import TaskError from pants.base.workunit import WorkUnit, WorkUnitLabel @@ -24,16 +25,30 @@ class SharedLibrary(datatype(['name', 'path'])): pass class LinkSharedLibraryRequest(datatype([ - 'linker', - 'object_files', - 'native_artifact', + ('linker', Linker), + ('object_files', tuple), + ('native_artifact', NativeArtifact), 'output_dir', - 'external_libs_info' -])): pass + ('external_lib_dirs', tuple), + ('external_lib_names', tuple), +])): + + @classmethod + def with_external_libs_product(cls, external_libs_product=None, *args, **kwargs): + if external_libs_product is None: + lib_dirs = () + lib_names = () + else: + lib_dirs = (external_libs_product.lib_dir,) + lib_names = external_libs_product.lib_names + + return cls(*args, external_lib_dirs=lib_dirs, external_lib_names=lib_names, **kwargs) class LinkSharedLibraries(NativeTask): + options_scope = 'link-shared-libraries' + @classmethod def product_types(cls): return [SharedLibrary] @@ -43,7 +58,7 @@ def prepare(cls, options, round_manager): super(LinkSharedLibraries, cls).prepare(options, round_manager) round_manager.require(NativeTargetDependencies) round_manager.require(ObjectFiles) - round_manager.require(NativeExternalLibraryFetch.NativeExternalLibraryFiles) + round_manager.optional_product(NativeExternalLibraryFiles) @property def cache_target_dirs(self): @@ -85,7 +100,7 @@ def execute(self): native_target_deps_product = self.context.products.get(NativeTargetDependencies) compiled_objects_product = self.context.products.get(ObjectFiles) shared_libs_product = self.context.products.get(SharedLibrary) - external_libs_product = self.context.products.get_data(NativeExternalLibraryFetch.NativeExternalLibraryFiles) + external_libs_product = self.context.products.get_data(NativeExternalLibraryFiles) all_shared_libs_by_name = {} @@ -143,18 +158,25 @@ def _make_link_request(self, self.context.log.debug("object_file_paths: {}".format(object_file_paths)) all_compiled_object_files.extend(object_file_paths) - return LinkSharedLibraryRequest( + return LinkSharedLibraryRequest.with_external_libs_product( linker=self.linker, - object_files=all_compiled_object_files, + object_files=tuple(all_compiled_object_files), native_artifact=vt.target.ctypes_native_library, output_dir=vt.results_dir, - external_libs_info=external_libs_product) + external_libs_product=external_libs_product) _SHARED_CMDLINE_ARGS = { 'darwin': lambda: ['-mmacosx-version-min=10.11', '-Wl,-dylib'], 'linux': lambda: ['-shared'], } + def _get_third_party_lib_args(self, link_request): + ext_libs = link_request.external_libs_info + if not ext_libs: + return [] + + return ext_libs.get_third_party_lib_args() + def _execute_link_request(self, link_request): object_files = link_request.object_files @@ -167,11 +189,11 @@ def _execute_link_request(self, link_request): output_dir = link_request.output_dir resulting_shared_lib_path = os.path.join(output_dir, native_artifact.as_shared_lib(self.platform)) + self.context.log.debug("resulting_shared_lib_path: {}".format(resulting_shared_lib_path)) # We are executing in the results_dir, so get absolute paths for everything. cmd = ([linker.exe_filename] + self.platform.resolve_platform_specific(self._SHARED_CMDLINE_ARGS) + linker.extra_args + - link_request.external_libs_info.get_third_party_lib_args() + ['-o', os.path.abspath(resulting_shared_lib_path)] + [os.path.abspath(obj) for obj in object_files]) diff --git a/src/python/pants/backend/native/tasks/native_compile.py b/src/python/pants/backend/native/tasks/native_compile.py index f8daebc216a8..2ee90c5bdde8 100644 --- a/src/python/pants/backend/native/tasks/native_compile.py +++ b/src/python/pants/backend/native/tasks/native_compile.py @@ -11,7 +11,7 @@ from pants.backend.native.config.environment import Executable, Platform from pants.backend.native.targets.native_library import NativeLibrary -from pants.backend.native.tasks.native_external_library_fetch import NativeExternalLibraryFetch +from pants.backend.native.tasks.native_external_library_fetch import NativeExternalLibraryFiles from pants.backend.native.tasks.native_task import NativeTask from pants.base.build_environment import get_buildroot from pants.base.exceptions import TaskError @@ -64,7 +64,7 @@ def product_types(cls): @classmethod def prepare(cls, options, round_manager): super(NativeCompile, cls).prepare(options, round_manager) - round_manager.require(NativeExternalLibraryFetch.NativeExternalLibraryFiles) + round_manager.optional_data(NativeExternalLibraryFiles) @property def cache_target_dirs(self): @@ -124,9 +124,7 @@ def _add_product_at_target_base(product_mapping, target, value): def execute(self): object_files_product = self.context.products.get(ObjectFiles) native_deps_product = self.context.products.get(NativeTargetDependencies) - external_libs_product = self.context.products.get_data( - NativeExternalLibraryFetch.NativeExternalLibraryFiles - ) + external_libs_product = self.context.products.get_data(NativeExternalLibraryFiles) source_targets = self.context.targets(self.source_target_constraint.satisfied_by) with self.invalidated(source_targets, invalidate_dependents=True) as invalidation_check: @@ -195,6 +193,9 @@ def _compiler(self): return self.get_compiler() def _get_third_party_include_dirs(self, external_libs_product): + if not external_libs_product: + return [] + directory = external_libs_product.include_dir return [directory] if directory else [] @@ -232,14 +233,18 @@ def _make_compile_argv(self, compile_request): # We are going to execute in the target output, so get absolute paths for everything. # TODO: If we need to produce static libs, don't add -fPIC! (could use Variants -- see #5788). + buildroot = get_buildroot() argv = ( [compiler.exe_filename] + platform_specific_flags + self.extra_compile_args() + err_flags + ['-c', '-fPIC'] + - ['-I{}'.format(os.path.abspath(inc_dir)) for inc_dir in compile_request.include_dirs] + - [os.path.abspath(src) for src in compile_request.sources]) + [ + '-I{}'.format(os.path.join(buildroot, inc_dir)) + for inc_dir in compile_request.include_dirs + ] + + [os.path.join(buildroot, src) for src in compile_request.sources]) return argv diff --git a/src/python/pants/backend/native/tasks/native_external_library_fetch.py b/src/python/pants/backend/native/tasks/native_external_library_fetch.py index 9d7df8aa32be..7ac82dd2287f 100644 --- a/src/python/pants/backend/native/tasks/native_external_library_fetch.py +++ b/src/python/pants/backend/native/tasks/native_external_library_fetch.py @@ -61,6 +61,14 @@ def fetch_cmdline_args(self): return args +class NativeExternalLibraryFiles(datatype([ + 'include_dir', + # TODO: we shouldn't have any `lib_names` if `lib_dir` is not set! + 'lib_dir', + ('lib_names', tuple), +])): pass + + class NativeExternalLibraryFetch(Task): options_scope = 'native-external-library-fetch' native_library_constraint = Exactly(ExternalNativeLibrary) @@ -68,24 +76,6 @@ class NativeExternalLibraryFetch(Task): class NativeExternalLibraryFetchError(TaskError): pass - class NativeExternalLibraryFiles(object): - def __init__(self): - self.include_dir = None - self.lib_dir = None - self.lib_names = [] - - def add_lib_name(self, lib_name): - self.lib_names.append(lib_name) - - def get_third_party_lib_args(self): - lib_args = [] - if self.lib_names: - for lib_name in self.lib_names: - lib_args.append('-l{}'.format(lib_name)) - lib_dir_arg = '-L{}'.format(self.lib_dir) - lib_args.append(lib_dir_arg) - return lib_args - @classmethod def _parse_lib_name_from_library_filename(cls, filename): match_group = re.match(r"^lib(.*)\.(a|so|dylib)$", filename) @@ -105,7 +95,7 @@ def subsystem_dependencies(cls): @classmethod def product_types(cls): - return [cls.NativeExternalLibraryFiles] + return [NativeExternalLibraryFiles] @property def cache_target_dirs(self): @@ -116,9 +106,6 @@ def _conan_binary(self): return Conan.scoped_instance(self).bootstrap_conan() def execute(self): - task_product = self.context.products.get_data(self.NativeExternalLibraryFiles, - self.NativeExternalLibraryFiles) - native_lib_tgts = self.context.targets(self.native_library_constraint.satisfied_by) if native_lib_tgts: with self.invalidated(native_lib_tgts, @@ -128,7 +115,10 @@ def execute(self): if invalidation_check.invalid_vts or not resolve_vts.valid: for vt in invalidation_check.all_vts: self._fetch_packages(vt, vts_results_dir) - self._populate_task_product(vts_results_dir, task_product) + + native_external_libs_product = self._collect_external_libs(vts_results_dir) + self.context.products.register_data(NativeExternalLibraryFiles, + native_external_libs_product) def _prepare_vts_results_dir(self, vts): """ @@ -138,22 +128,23 @@ def _prepare_vts_results_dir(self, vts): safe_mkdir(vt_set_results_dir) return vt_set_results_dir - def _populate_task_product(self, results_dir, task_product): + def _collect_external_libs(self, results_dir): """ Sets the relevant properties of the task product (`NativeExternalLibraryFiles`) object. """ - lib = os.path.join(results_dir, 'lib') - include = os.path.join(results_dir, 'include') + lib_dir = os.path.join(results_dir, 'lib') + include_dir = os.path.join(results_dir, 'include') - if os.path.exists(lib): - task_product.lib_dir = lib - for filename in os.listdir(lib): + lib_names = [] + if os.path.isdir(lib_dir): + for filename in os.listdir(lib_dir): lib_name = self._parse_lib_name_from_library_filename(filename) if lib_name: - task_product.add_lib_name(lib_name) + lib_names.append(lib_name) - if os.path.exists(include): - task_product.include_dir = include + return NativeExternalLibraryFiles(include_dir=include_dir, + lib_dir=lib_dir, + lib_names=tuple(lib_names)) def _get_conan_data_dir_path_for_package(self, pkg_dir_path, pkg_sha): return os.path.join(self.workdir, diff --git a/src/python/pants/backend/python/subsystems/python_native_code.py b/src/python/pants/backend/python/subsystems/python_native_code.py index 01b3175e63da..6c51ea600f99 100644 --- a/src/python/pants/backend/python/subsystems/python_native_code.py +++ b/src/python/pants/backend/python/subsystems/python_native_code.py @@ -15,7 +15,7 @@ from pants.base.exceptions import IncompatiblePlatformsError from pants.subsystem.subsystem import Subsystem from pants.util.memo import memoized_property -from pants.util.objects import Exactly +from pants.util.objects import SubclassesOf class PythonNativeCode(Subsystem): @@ -63,8 +63,8 @@ def native_target_has_native_sources(self, target): @memoized_property def _native_target_matchers(self): return { - Exactly(PythonDistribution): self.pydist_has_native_sources, - Exactly(NativeLibrary): self.native_target_has_native_sources, + SubclassesOf(PythonDistribution): self.pydist_has_native_sources, + SubclassesOf(NativeLibrary): self.native_target_has_native_sources, } def _any_targets_have_native_sources(self, targets): @@ -97,7 +97,7 @@ def get_targets_by_declared_platform(self, targets): '--platforms option or a pants.ini file.'] return targets_by_platforms - _PYTHON_PLATFORM_TARGETS_CONSTRAINT = Exactly(PythonBinary, PythonDistribution) + _PYTHON_PLATFORM_TARGETS_CONSTRAINT = SubclassesOf(PythonBinary, PythonDistribution) def check_build_for_current_platform_only(self, targets): """ diff --git a/testprojects/src/python/python_distribution/ctypes/BUILD b/testprojects/src/python/python_distribution/ctypes/BUILD index 0444eec0892e..5bccab8a025b 100644 --- a/testprojects/src/python/python_distribution/ctypes/BUILD +++ b/testprojects/src/python/python_distribution/ctypes/BUILD @@ -31,4 +31,5 @@ python_binary( dependencies=[ ':ctypes', ], + platforms=['current'], ) diff --git a/tests/python/pants_test/backend/python/tasks/BUILD b/tests/python/pants_test/backend/python/tasks/BUILD index f7a3313a891a..361d335f31fd 100644 --- a/tests/python/pants_test/backend/python/tasks/BUILD +++ b/tests/python/pants_test/backend/python/tasks/BUILD @@ -24,6 +24,7 @@ python_library( python_native_code_test_files = [ 'test_build_local_python_distributions.py', 'test_python_distribution_integration.py', + 'test_ctypes.py', 'test_ctypes_integration.py', ] @@ -31,15 +32,17 @@ python_tests( name='python_native_code_testing', sources=python_native_code_test_files, dependencies=[ - ':python_task_test_base', '3rdparty/python:future', + '3rdparty/python/twitter/commons:twitter.common.collections', 'src/python/pants/backend/native', + 'src/python/pants/backend/native/targets', 'src/python/pants/backend/python:plugin', 'src/python/pants/backend/python/targets', 'src/python/pants/backend/python/tasks', 'src/python/pants/util:contextutil', 'src/python/pants/util:process_handler', 'tests/python/pants_test:int-test', + 'tests/python/pants_test/backend/python/tasks/util', 'tests/python/pants_test/engine:scheduler_test_base', ], tags={'platform_specific_behavior', 'integration'}, diff --git a/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py b/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py index 5c2e600383cd..81f1628bc57c 100644 --- a/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py +++ b/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py @@ -4,30 +4,27 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import re -from builtins import next, str +from builtins import str from textwrap import dedent -from pants.backend.native.register import rules as native_backend_rules -from pants.backend.python.register import rules as python_backend_rules +from twitter.common.collections import OrderedDict + from pants.backend.python.targets.python_distribution import PythonDistribution -from pants.backend.python.tasks.build_local_python_distributions import \ - BuildLocalPythonDistributions -from pants.util.collections import assert_single_element -from pants_test.backend.python.tasks.python_task_test_base import (PythonTaskTestBase, - check_wheel_platform_matches_host, +from pants_test.backend.python.tasks.python_task_test_base import (check_wheel_platform_matches_host, name_and_platform) -from pants_test.engine.scheduler_test_base import SchedulerTestBase +from pants_test.backend.python.tasks.util.build_local_dists_test_base import \ + BuildLocalPythonDistributionsTestBase + + +class TestBuildLocalDistsNativeSources(BuildLocalPythonDistributionsTestBase): + _extra_relevant_task_types = [] -class TestBuildLocalPythonDistributions(PythonTaskTestBase, SchedulerTestBase): - @classmethod - def task_type(cls): - return BuildLocalPythonDistributions + _dist_specs = OrderedDict([ - _dist_specs = { - 'src/python/dist:universal_dist': { + ('src/python/dist:universal_dist', { 'key': 'universal', + 'target_type': PythonDistribution, 'sources': ['foo.py', 'bar.py', '__init__.py', 'setup.py'], 'filemap': { 'src/python/dist/__init__.py': '', @@ -42,9 +39,11 @@ def task_type(cls): ) """) } - }, - 'src/python/plat_specific_dist:plat_specific_dist': { + }), + + ('src/python/plat_specific_dist:plat_specific_dist', { 'key': 'platform_specific', + 'target_type': PythonDistribution, 'sources': ['__init__.py', 'setup.py', 'native_source.c'], 'filemap': { 'src/python/plat_specific_dist/__init__.py': '', @@ -75,84 +74,9 @@ def task_type(cls): } """), } - }, - } - - def setUp(self): - super(TestBuildLocalPythonDistributions, self).setUp() - - self.target_dict = {} - - # Create a python_dist() target from each specification and insert it into `self.target_dict`. - for target_spec, file_spec in self._dist_specs.items(): - filemap = file_spec['filemap'] - for rel_path, content in filemap.items(): - self.create_file(rel_path, content) - - sources = file_spec['sources'] - python_dist_tgt = self.make_target(spec=target_spec, - target_type=PythonDistribution, - sources=sources) - key = file_spec['key'] - self.target_dict[key] = python_dist_tgt - - def _all_dist_targets(self): - return list(self.target_dict.values()) - - def _scheduling_context(self, **kwargs): - rules = ( - native_backend_rules() + - python_backend_rules() - ) - scheduler = self.mk_scheduler(rules=rules) - return self.context(scheduler=scheduler, **kwargs) - - def _retrieve_single_product_at_target_base(self, product_mapping, target): - product = product_mapping.get(target) - base_dirs = list(product.keys()) - self.assertEqual(1, len(base_dirs)) - single_base_dir = base_dirs[0] - all_products = product[single_base_dir] - self.assertEqual(1, len(all_products)) - single_product = all_products[0] - return single_product - - def _get_dist_snapshot_version(self, task, python_dist_target): - """Get the target's fingerprint, and guess the resulting version string of the built dist. - - Local python_dist() builds are tagged with the versioned target's fingerprint using the - --tag-build option in the egg_info command. This fingerprint string is slightly modified by - distutils to ensure a valid version string, and this method finds what that modified version - string is so we can verify that the produced local dist is being tagged with the correct - snapshot version. - - The argument we pass to that option begins with a +, which is unchanged. See - https://www.python.org/dev/peps/pep-0440/ for further information. - """ - with task.invalidated([python_dist_target], invalidate_dependents=True) as invalidation_check: - versioned_dist_target = assert_single_element(invalidation_check.all_vts) - - versioned_target_fingerprint = versioned_dist_target.cache_key.hash - - # This performs the normalization that distutils performs to the version string passed to the - # --tag-build option. - return re.sub(r'[^a-zA-Z0-9]', '.', versioned_target_fingerprint.lower()) - - def _create_distribution_synthetic_target(self, python_dist_target): - context = self._scheduling_context( - target_roots=[python_dist_target], - for_task_types=[BuildLocalPythonDistributions]) - self.assertEquals(set(self._all_dist_targets()), set(context.build_graph.targets())) - python_create_distributions_task = self.create_task(context) - python_create_distributions_task.execute() - synthetic_tgts = set(context.build_graph.targets()) - set(self._all_dist_targets()) - self.assertEquals(1, len(synthetic_tgts)) - synthetic_target = next(iter(synthetic_tgts)) - - snapshot_version = self._get_dist_snapshot_version( - python_create_distributions_task, python_dist_target) - - return context, synthetic_target, snapshot_version + }), + + ]) def test_python_create_universal_distribution(self): universal_dist = self.target_dict['universal'] diff --git a/tests/python/pants_test/backend/python/tasks/test_ctypes.py b/tests/python/pants_test/backend/python/tasks/test_ctypes.py new file mode 100644 index 000000000000..51e9cd14d5d4 --- /dev/null +++ b/tests/python/pants_test/backend/python/tasks/test_ctypes.py @@ -0,0 +1,117 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from builtins import str +from textwrap import dedent + +from twitter.common.collections import OrderedDict + +from pants.backend.native.targets.native_artifact import NativeArtifact +from pants.backend.native.targets.native_library import CLibrary, CppLibrary +from pants.backend.native.tasks.c_compile import CCompile +from pants.backend.native.tasks.cpp_compile import CppCompile +from pants.backend.native.tasks.link_shared_libraries import LinkSharedLibraries +from pants.backend.python.targets.python_distribution import PythonDistribution +from pants_test.backend.python.tasks.python_task_test_base import check_wheel_platform_matches_host +from pants_test.backend.python.tasks.util.build_local_dists_test_base import \ + BuildLocalPythonDistributionsTestBase + + +class TestBuildLocalDistsWithCtypesNativeSources(BuildLocalPythonDistributionsTestBase): + + _extra_relevant_task_types = [CCompile, CppCompile, LinkSharedLibraries] + + _dist_specs = OrderedDict([ + + ('src/python/plat_specific_c_dist:ctypes_c_library', { + 'key': 'ctypes_c_library', + 'target_type': CLibrary, + 'ctypes_native_library': NativeArtifact(lib_name='c-math-lib'), + 'sources': ['c_math_lib.c', 'c_math_lib.h'], + 'filemap': { + 'src/python/plat_specific_c_dist/c_math_lib.c': dedent(""" + #include "c_math_lib.h" + int add_two(int x) { return x + 2; } +"""), + 'src/python/plat_specific_c_dist/c_math_lib.h': dedent(""" + int add_two(int); +"""), + } + }), + + ('src/python/plat_specific_c_dist:plat_specific_ctypes_c_dist', { + 'key': 'platform_specific_ctypes_c_dist', + 'target_type': PythonDistribution, + 'sources': ['__init__.py', 'setup.py'], + 'dependencies': ['src/python/plat_specific_c_dist:ctypes_c_library'], + 'filemap': { + 'src/python/plat_specific_c_dist/__init__.py': '', + 'src/python/plat_specific_c_dist/setup.py': dedent(""" + from setuptools import setup, find_packages + setup( + name='platform_specific_ctypes_c_dist', + version='0.0.0', + packages=find_packages(), + data_files=[('', ['libc-math-lib.so'])], + ) + """), + } + }), + + ('src/python/plat_specific_cpp_dist:ctypes_cpp_library', { + 'key': 'ctypes_cpp_library', + 'target_type': CppLibrary, + 'ctypes_native_library': NativeArtifact(lib_name='cpp-math-lib'), + 'sources': ['cpp_math_lib.cpp', 'cpp_math_lib.hpp'], + 'filemap': { + 'src/python/plat_specific_cpp_dist/cpp_math_lib.cpp': '', + 'src/python/plat_specific_cpp_dist/cpp_math_lib.hpp': '', + }, + }), + + ('src/python/plat_specific_cpp_dist:plat_specific_ctypes_cpp_dist', { + 'key': 'platform_specific_ctypes_cpp_dist', + 'target_type': PythonDistribution, + 'sources': ['__init__.py', 'setup.py'], + 'dependencies': ['src/python/plat_specific_cpp_dist:ctypes_cpp_library'], + 'filemap': { + 'src/python/plat_specific_cpp_dist/__init__.py': '', + 'src/python/plat_specific_cpp_dist/setup.py': dedent(""" + from setuptools import setup, find_packages + setup( + name='platform_specific_ctypes_cpp_dist', + version='0.0.0', + packages=find_packages(), + data_files=[('', ['libcpp-math-lib.so'])], + ) + """), + } + }), + + ]) + + def test_ctypes_c_dist(self): + platform_specific_dist = self.target_dict['platform_specific_ctypes_c_dist'] + context, synthetic_target, snapshot_version = self._create_distribution_synthetic_target( + platform_specific_dist, extra_targets=[self.target_dict['ctypes_c_library']]) + self.assertEquals(['platform_specific_ctypes_c_dist==0.0.0+{}'.format(snapshot_version)], + [str(x.requirement) for x in synthetic_target.requirements.value]) + local_wheel_products = context.products.get('local_wheels') + local_wheel = self._retrieve_single_product_at_target_base( + local_wheel_products, platform_specific_dist) + self.assertTrue(check_wheel_platform_matches_host(local_wheel)) + + def test_ctypes_cpp_dist(self): + platform_specific_dist = self.target_dict['platform_specific_ctypes_cpp_dist'] + context, synthetic_target, snapshot_version = self._create_distribution_synthetic_target( + platform_specific_dist, extra_targets=[self.target_dict['ctypes_cpp_library']]) + self.assertEquals(['platform_specific_ctypes_cpp_dist==0.0.0+{}'.format(snapshot_version)], + [str(x.requirement) for x in synthetic_target.requirements.value]) + + local_wheel_products = context.products.get('local_wheels') + local_wheel = self._retrieve_single_product_at_target_base( + local_wheel_products, platform_specific_dist) + self.assertTrue(check_wheel_platform_matches_host(local_wheel)) diff --git a/tests/python/pants_test/backend/python/tasks/test_ctypes_integration.py b/tests/python/pants_test/backend/python/tasks/test_ctypes_integration.py index 3841f4cb1d60..f30cc8cd3962 100644 --- a/tests/python/pants_test/backend/python/tasks/test_ctypes_integration.py +++ b/tests/python/pants_test/backend/python/tasks/test_ctypes_integration.py @@ -30,14 +30,14 @@ class CTypesIntegrationTest(PantsRunIntegrationTest): 'testprojects/src/python/python_distribution/ctypes_with_third_party:bin_with_third_party' ) - def test_run(self): + def test_ctypes_run(self): pants_run = self.run_pants(command=['run', self._binary_target]) self.assert_success(pants_run) # This is the entire output from main.py. self.assertIn('x=3, f(x)=17', pants_run.stdout_data) - def test_binary(self): + def test_ctypes_binary(self): with temporary_dir() as tmp_dir: pants_run = self.run_pants(command=['binary', self._binary_target], config={ GLOBAL_SCOPE_CONFIG_SECTION: { @@ -75,7 +75,7 @@ def test_binary(self): # Execute the binary and ensure its output is correct. binary_run_output = invoke_pex_for_output(pex) - self.assertIn('x=3, f(x)=17', binary_run_output) + self.assertEqual('x=3, f(x)=17\n', binary_run_output) def test_ctypes_third_party_integration(self): pants_binary = self.run_pants( @@ -95,3 +95,24 @@ def test_ctypes_third_party_integration(self): ) self.assert_success(pants_run) self.assertIn('Test worked!', pants_run.stdout_data) + + def test_pants_native_source_detection_for_local_ctypes_dists_for_current_platform_only(self): + """Test that `./pants run` respects platforms when the closure contains native sources. + + To do this, we need to setup a pants.ini that contains two platform defauts: (1) "current" and + (2) a different platform than the one we are currently running on. The python_binary() target + below is declared with `platforms="current"`. + """ + # Clean all to rebuild requirements pex. + command = [ + 'clean-all', + 'run', + 'testprojects/src/python/python_distribution/ctypes:bin' + ] + pants_run = self.run_pants(command=command, config={ + 'python-setup': { + 'platforms': ['current', 'this-platform-does_not-exist'] + }, + }) + self.assert_success(pants_run) + self.assertIn('x=3, f(x)=17', pants_run.stdout_data) diff --git a/tests/python/pants_test/backend/python/tasks/test_python_distribution_integration.py b/tests/python/pants_test/backend/python/tasks/test_python_distribution_integration.py index e27f6a8cae21..2c1c9bf99eb6 100644 --- a/tests/python/pants_test/backend/python/tasks/test_python_distribution_integration.py +++ b/tests/python/pants_test/backend/python/tasks/test_python_distribution_integration.py @@ -153,8 +153,11 @@ def test_pants_resolves_local_dists_for_current_platform_only(self): # or running a target that depends on native (c or cpp) sources. with temporary_dir() as tmp_dir: pex = os.path.join(tmp_dir, 'main.pex') - pants_ini_config = {'python-setup': {'platforms': ['current', 'linux-x86_64']}} - # Clean all to rebuild requirements pex. + pants_ini_config = { + 'python-setup': { + 'platforms': ['current', 'this-platform-does_not-exist'], + }, + } command=[ '--pants-distdir={}'.format(tmp_dir), 'run', @@ -171,11 +174,14 @@ def test_pants_resolves_local_dists_for_current_platform_only(self): output = subprocess.check_output(pex) self._assert_native_greeting(output) - def test_pants_tests_local_dists_for_current_platform_only(self): - platform_string = Platform.create().resolve_platform_specific({ + def _get_current_platform_string(self): + return Platform.create().resolve_platform_specific({ 'darwin': lambda: 'macosx-10.12-x86_64', 'linux': lambda: 'linux-x86_64', }) + + def test_pants_tests_local_dists_for_current_platform_only(self): + platform_string = self._get_current_platform_string() # Use a platform-specific string for testing because the test goal # requires the coverage package and the pex resolver raises an Untranslatable error when # attempting to translate the coverage sdist for incompatible platforms. diff --git a/tests/python/pants_test/backend/python/tasks/util/BUILD b/tests/python/pants_test/backend/python/tasks/util/BUILD new file mode 100644 index 000000000000..b2e6214c9b65 --- /dev/null +++ b/tests/python/pants_test/backend/python/tasks/util/BUILD @@ -0,0 +1,8 @@ +python_library( + dependencies=[ + 'src/python/pants/backend/native', + 'src/python/pants/backend/python/tasks', + 'tests/python/pants_test/backend/python/tasks:python_task_test_base', + 'tests/python/pants_test/engine:scheduler_test_base', + ] +) diff --git a/tests/python/pants_test/backend/python/tasks/util/__init__.py b/tests/python/pants_test/backend/python/tasks/util/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py b/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py new file mode 100644 index 000000000000..c500f0b2e794 --- /dev/null +++ b/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py @@ -0,0 +1,119 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import re +from builtins import next + +from pants.backend.native.register import rules as native_backend_rules +from pants.backend.python.tasks.build_local_python_distributions import \ + BuildLocalPythonDistributions +from pants.util.collections import assert_single_element +from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase +from pants_test.engine.scheduler_test_base import SchedulerTestBase + + +class BuildLocalPythonDistributionsTestBase(PythonTaskTestBase, SchedulerTestBase): + + @classmethod + def task_type(cls): + return BuildLocalPythonDistributions + + # This is an informally-specified nested dict -- see ../test_ctypes.py for an example. Special + # keys are 'key' (used to index into `self.target_dict`) and 'filemap' (creates files at the + # specified relative paths). The rest of the keys are fed into `self.make_target()`. An + # `OrderedDict` of 2-tuples may be used if targets need to be created in a specific order (e.g. if + # they have dependencies on each other). + _dist_specs = None + # By default, we just use a `BuildLocalPythonDistributions` task. When testing with C/C++ targets, + # we want to compile and link them as well to get the resulting dist to build, so we add those + # task types here and execute them beforehand. + _extra_relevant_task_types = None + + def setUp(self): + super(BuildLocalPythonDistributionsTestBase, self).setUp() + + self.target_dict = {} + + # Create a python_dist() target from each specification and insert it into `self.target_dict`. + for target_spec, file_spec in self._dist_specs.items(): + file_spec = file_spec.copy() + filemap = file_spec.pop('filemap') + for rel_path, content in filemap.items(): + self.create_file(rel_path, content) + + key = file_spec.pop('key') + dep_targets = [] + for dep_spec in file_spec.pop('dependencies', []): + existing_tgt_key = self._dist_specs[dep_spec]['key'] + dep_targets.append(self.target_dict[existing_tgt_key]) + python_dist_tgt = self.make_target(spec=target_spec, dependencies=dep_targets, **file_spec) + self.target_dict[key] = python_dist_tgt + + def _all_specified_targets(self): + return list(self.target_dict.values()) + + def _scheduling_context(self, **kwargs): + scheduler = self.mk_scheduler(rules=native_backend_rules()) + return self.context(scheduler=scheduler, **kwargs) + + def _retrieve_single_product_at_target_base(self, product_mapping, target): + product = product_mapping.get(target) + base_dirs = list(product.keys()) + self.assertEqual(1, len(base_dirs)) + single_base_dir = base_dirs[0] + all_products = product[single_base_dir] + self.assertEqual(1, len(all_products)) + single_product = all_products[0] + return single_product + + def _get_dist_snapshot_version(self, task, python_dist_target): + """Get the target's fingerprint, and guess the resulting version string of the built dist. + + Local python_dist() builds are tagged with the versioned target's fingerprint using the + --tag-build option in the egg_info command. This fingerprint string is slightly modified by + distutils to ensure a valid version string, and this method finds what that modified version + string is so we can verify that the produced local dist is being tagged with the correct + snapshot version. + + The argument we pass to that option begins with a +, which is unchanged. See + https://www.python.org/dev/peps/pep-0440/ for further information. + """ + with task.invalidated([python_dist_target], invalidate_dependents=True) as invalidation_check: + versioned_dist_target = assert_single_element(invalidation_check.all_vts) + + versioned_target_fingerprint = versioned_dist_target.cache_key.hash + + # This performs the normalization that distutils performs to the version string passed to the + # --tag-build option. + return re.sub(r'[^a-zA-Z0-9]', '.', versioned_target_fingerprint.lower()) + + def _create_task(self, task_type, context): + return task_type(context, self.test_workdir) + + def _create_distribution_synthetic_target(self, python_dist_target, extra_targets=[]): + context = self._scheduling_context( + target_roots=([python_dist_target] + extra_targets), + for_task_types=([self.task_type()] + self._extra_relevant_task_types)) + self.assertEquals(set(self._all_specified_targets()), set(context.build_graph.targets())) + + python_create_distributions_task = self.create_task(context) + extra_tasks = [ + self._create_task(task_type, context) + for task_type in self._extra_relevant_task_types + ] + for tsk in extra_tasks: + tsk.execute() + + python_create_distributions_task.execute() + + synthetic_tgts = set(context.build_graph.targets()) - set(self._all_specified_targets()) + self.assertEquals(1, len(synthetic_tgts)) + synthetic_target = next(iter(synthetic_tgts)) + + snapshot_version = self._get_dist_snapshot_version( + python_create_distributions_task, python_dist_target) + + return context, synthetic_target, snapshot_version