diff --git a/examples/src/python/example/python_distribution/hello/pants_setup_requires/BUILD b/examples/src/python/example/python_distribution/hello/pants_setup_requires/BUILD index 89fe1c253310..de071b8bd2c6 100644 --- a/examples/src/python/example/python_distribution/hello/pants_setup_requires/BUILD +++ b/examples/src/python/example/python_distribution/hello/pants_setup_requires/BUILD @@ -1,10 +1,18 @@ # Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +main_file = 'main.py' + python_dist( - sources=globs('*.py'), + sources=rglobs('*.py', exclude=[main_file]), setup_requires=[ # FIXME: this currently fails because setup_requires doesn't fetch transitive deps! 'testprojects/pants-plugins/3rdparty/python/pants', ], ) + +python_binary( + name='bin', + sources=[main_file], + dependencies=[':pants_setup_requires'], +) diff --git a/examples/src/python/example/python_distribution/hello/pants_setup_requires/hello/__init__.py b/examples/src/python/example/python_distribution/hello/pants_setup_requires/hello/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/examples/src/python/example/python_distribution/hello/pants_setup_requires/main.py b/examples/src/python/example/python_distribution/hello/pants_setup_requires/main.py new file mode 100644 index 000000000000..8e6d8dc3f488 --- /dev/null +++ b/examples/src/python/example/python_distribution/hello/pants_setup_requires/main.py @@ -0,0 +1,13 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import pkg_resources + + +hello_module_version = pkg_resources.get_distribution('hello_again').version + +if __name__ == '__main__': + print(hello_module_version) diff --git a/examples/src/python/example/python_distribution/hello/pants_setup_requires/setup.py b/examples/src/python/example/python_distribution/hello/pants_setup_requires/setup.py index b7b6207f2ea3..1e4e9630e211 100644 --- a/examples/src/python/example/python_distribution/hello/pants_setup_requires/setup.py +++ b/examples/src/python/example/python_distribution/hello/pants_setup_requires/setup.py @@ -4,13 +4,12 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import os from setuptools import setup, find_packages from pants.version import VERSION as pants_version setup( - name='hello', + name='hello_again', # FIXME: test the wheel version in a unit test! version=pants_version, packages=find_packages(), diff --git a/src/python/pants/backend/native/subsystems/conan.py b/src/python/pants/backend/native/subsystems/conan.py index d7eb6e7a81bc..90bcba411355 100644 --- a/src/python/pants/backend/native/subsystems/conan.py +++ b/src/python/pants/backend/native/subsystems/conan.py @@ -5,20 +5,9 @@ from __future__ import absolute_import, division, print_function, unicode_literals import logging -import os - -from pex.interpreter import PythonInterpreter -from pex.pex import PEX -from pex.pex_builder import PEXBuilder -from pex.pex_info import PexInfo from pants.backend.python.python_requirement import PythonRequirement -from pants.backend.python.subsystems.python_repos import PythonRepos -from pants.backend.python.subsystems.python_setup import PythonSetup -from pants.backend.python.tasks.pex_build_util import dump_requirements -from pants.base.build_environment import get_pants_cachedir from pants.binaries.executable_pex_tool import ExecutablePexTool -from pants.util.dirutil import safe_concurrent_creation from pants.util.memo import memoized_property @@ -49,20 +38,16 @@ class Conan(ExecutablePexTool): 'deprecation>=2.0, <2.1' ) - @classmethod - def implementation_version(cls): - return super(Conan, cls).implementation_version() + [('Conan', 0)] - - @classmethod - def subsystem_dependencies(cls): - return super(Conan, cls).subsystem_dependencies() + (PythonRepos, PythonSetup) - @classmethod def register_options(cls, register): super(Conan, cls).register_options(register) register('--conan-requirements', type=list, default=cls.default_conan_requirements, advanced=True, help='The requirements used to build the conan client pex.') + @classmethod + def implementation_version(cls): + return super(Conan, cls).implementation_version() + [('Conan', 0)] + @memoized_property - def pex_tool_requirements(self): + def base_requirements(self): return [PythonRequirement(req) for req in self.get_options().conan_requirements] diff --git a/src/python/pants/backend/python/subsystems/python_native_code.py b/src/python/pants/backend/python/subsystems/python_native_code.py index cfa772679ed9..236dee10a914 100644 --- a/src/python/pants/backend/python/subsystems/python_native_code.py +++ b/src/python/pants/backend/python/subsystems/python_native_code.py @@ -4,21 +4,21 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import os from builtins import str from collections import defaultdict -from wheel.install import WheelFile +from pex.pex import PEX from pants.backend.native.config.environment import CppToolchain, CToolchain, Platform from pants.backend.native.subsystems.native_toolchain import NativeToolchain from pants.backend.native.subsystems.xcode_cli_tools import MIN_OSX_VERSION_ARG from pants.backend.native.targets.native_library import NativeLibrary +from pants.backend.python.python_requirement import PythonRequirement from pants.backend.python.subsystems.python_setup import PythonSetup from pants.backend.python.targets.python_binary import PythonBinary from pants.backend.python.targets.python_distribution import PythonDistribution -from pants.backend.python.tasks.pex_build_util import resolve_multi from pants.base.exceptions import IncompatiblePlatformsError +from pants.binaries.executable_pex_tool import ExecutablePexTool from pants.subsystem.subsystem import Subsystem from pants.util.memo import memoized_property from pants.util.objects import SubclassesOf, datatype @@ -138,6 +138,25 @@ def check_build_for_current_platform_only(self, targets): .format(str(platforms_with_sources))) +class BuildSetupRequiresPex(ExecutablePexTool): + options_scope = 'build-setup-requires-pex' + + @classmethod + def subsystem_dependencies(cls): + return super(BuildSetupRequiresPex, cls).subsystem_dependencies() + (PythonSetup.scoped(cls),) + + @property + def base_requirements(self): + return [ + PythonRequirement('setuptools'), + PythonRequirement('wheel'), + ] + + @memoized_property + def _python_setup(self): + return PythonSetup.scoped_instance(self) + + class SetupPyNativeTools(datatype([ ('c_toolchain', CToolchain), ('cpp_toolchain', CppToolchain), @@ -149,44 +168,16 @@ class SetupPyNativeTools(datatype([ """ -class SetupRequiresSiteDir(datatype(['site_dir'])): pass - - -# TODO: This could be formulated as an @rule if targets and `PythonInterpreter` are made available -# to the v2 engine. -def ensure_setup_requires_site_dir(reqs_to_resolve, interpreter, site_dir, - platforms=None): - if not reqs_to_resolve: - return None - - setup_requires_dists = resolve_multi(interpreter, reqs_to_resolve, platforms, None) - - # FIXME: there's no description of what this does or why it's necessary. - overrides = { - 'purelib': site_dir, - 'headers': os.path.join(site_dir, 'headers'), - 'scripts': os.path.join(site_dir, 'bin'), - 'platlib': site_dir, - 'data': site_dir - } - - # The `python_dist` target builds for the current platform only. - # FIXME: why does it build for the current platform only? - for obj in setup_requires_dists['current']: - wf = WheelFile(obj.location) - wf.install(overrides=overrides, force=True) - - return SetupRequiresSiteDir(site_dir) - - # TODO: It might be pretty useful to have an Optional TypeConstraint. class SetupPyExecutionEnvironment(datatype([ # If None, don't set PYTHONPATH in the setup.py environment. - 'setup_requires_site_dir', + ('setup_requires_pex', PEX), # If None, don't execute in the toolchain environment. 'setup_py_native_tools', ])): + + _SHARED_CMDLINE_ARGS = { 'darwin': lambda: [ MIN_OSX_VERSION_ARG, @@ -200,9 +191,6 @@ class SetupPyExecutionEnvironment(datatype([ def as_environment(self): ret = {} - if self.setup_requires_site_dir: - ret['PYTHONPATH'] = self.setup_requires_site_dir.site_dir - # FIXME(#5951): the below is a lot of error-prone repeated logic -- we need a way to compose # executables more hygienically. We should probably be composing each datatype's members, and # only creating an environment at the very end. diff --git a/src/python/pants/backend/python/tasks/build_local_python_distributions.py b/src/python/pants/backend/python/tasks/build_local_python_distributions.py index f2eaf38bc2f3..0be551f16740 100644 --- a/src/python/pants/backend/python/tasks/build_local_python_distributions.py +++ b/src/python/pants/backend/python/tasks/build_local_python_distributions.py @@ -16,10 +16,10 @@ from pants.backend.native.targets.native_library import NativeLibrary from pants.backend.native.tasks.link_shared_libraries import SharedLibrary from pants.backend.python.python_requirement import PythonRequirement -from pants.backend.python.subsystems.python_native_code import (PythonNativeCode, +from pants.backend.python.subsystems.python_native_code import (BuildSetupRequiresPex, + PythonNativeCode, SetupPyExecutionEnvironment, - SetupPyNativeTools, - ensure_setup_requires_site_dir) + SetupPyNativeTools) from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants.backend.python.tasks.pex_build_util import is_local_python_dist from pants.backend.python.tasks.setup_py import SetupPyRunner @@ -44,6 +44,8 @@ class BuildLocalPythonDistributions(Task): # This will contain the sources used to build the python_dist(). _DIST_SOURCE_SUBDIR = 'python_dist_subdir' + setup_requires_pex_filename = 'setup-requires.pex' + # This defines the output directory when building the dist, so we know where the output wheel is # located. It is a subdirectory of `_DIST_SOURCE_SUBDIR`. _DIST_OUTPUT_DIR = 'dist' @@ -67,6 +69,7 @@ def implementation_version(cls): @classmethod def subsystem_dependencies(cls): return super(BuildLocalPythonDistributions, cls).subsystem_dependencies() + ( + BuildSetupRequiresPex.scoped(cls), PythonNativeCode.scoped(cls), ) @@ -80,6 +83,10 @@ def _platform(cls): def _python_native_code_settings(self): return PythonNativeCode.scoped_instance(self) + @memoized_property + def _build_setup_requires_pex_settings(self): + return BuildSetupRequiresPex.scoped_instance(self) + # FIXME(#5869): delete this and get Subsystems from options, when that is possible. def _request_single(self, product, subject): # NB: This is not supposed to be exposed to Tasks yet -- see #4769 to track the status of @@ -222,6 +229,8 @@ def _prepare_and_create_dist(self, interpreter, shared_libs_product, versioned_t # We are including a platform-specific shared lib in this dist, so mark it as such. is_platform_specific = True + versioned_target_fingerprint = versioned_target.cache_key.hash + setup_requires_dir = os.path.join(results_dir, self._SETUP_REQUIRES_SITE_SUBDIR) setup_reqs_to_resolve = self._get_setup_requires_to_resolve(dist_target) if setup_reqs_to_resolve: @@ -229,18 +238,18 @@ def _prepare_and_create_dist(self, interpreter, shared_libs_product, versioned_t 'Installing setup requirements: {}\n\n' .format([req.key for req in setup_reqs_to_resolve])) - setup_requires_site_dir = ensure_setup_requires_site_dir( - setup_reqs_to_resolve, interpreter, setup_requires_dir, platforms=['current']) - if setup_requires_site_dir: - self.context.log.debug('Setting PYTHONPATH with setup_requires site directory: {}' - .format(setup_requires_site_dir)) + setup_reqs_pex_path = os.path.join( + setup_requires_dir, + 'setup-requires-{}.pex'.format(versioned_target_fingerprint)) + setup_requires_pex = self._build_setup_requires_pex_settings.bootstrap( + interpreter, setup_reqs_pex_path, extra_reqs=setup_reqs_to_resolve) + self.context.log.debug('Using pex file as setup.py interpreter: {}' + .format(setup_requires_pex)) setup_py_execution_environment = SetupPyExecutionEnvironment( - setup_requires_site_dir=setup_requires_site_dir, + setup_requires_pex=setup_requires_pex, setup_py_native_tools=native_tools) - versioned_target_fingerprint = versioned_target.cache_key.hash - self._create_dist( dist_target, dist_output_dir, @@ -280,29 +289,36 @@ def _create_dist(self, dist_tgt, dist_target_dir, interpreter, setup_py_snapshot_version_argv = self._generate_snapshot_bdist_wheel_argv( snapshot_fingerprint, is_platform_specific) + setup_requires_interpreter = PythonInterpreter( + binary=setup_py_execution_environment.setup_requires_pex.path(), + identity=interpreter.identity, + extras=interpreter.extras) + setup_runner = SetupPyRunner( source_dir=dist_target_dir, setup_command=setup_py_snapshot_version_argv, - interpreter=interpreter) + file_input='setup.py', + interpreter=setup_requires_interpreter) setup_py_env = setup_py_execution_environment.as_environment() with environment_as(**setup_py_env): # Build a whl using SetupPyRunner and return its absolute path. - was_installed_successfully = setup_runner.run() - # FIXME: Make a run_raising_error() method in SetupPyRunner that doesn't print directly to - # stderr like pex does (better: put this in pex itself). - if not was_installed_successfully: + try: + setup_runner.run() + except SetupPyRunner.SetupPyRunnerError as e: raise self.BuildLocalPythonDistributionsError( "Installation of python distribution from target {target} into directory {into_dir} " "failed.\n" "The chosen interpreter was: {interpreter}.\n" "The execution environment was: {env}.\n" - "The setup command was: {command}." + "The setup command was: {command}.\n{err}" .format(target=dist_tgt, into_dir=dist_target_dir, - interpreter=interpreter, + interpreter=setup_requires_interpreter, env=setup_py_env, - command=setup_py_snapshot_version_argv)) + command=setup_py_snapshot_version_argv, + err=e), + e) def _inject_synthetic_dist_requirements(self, dist, req_lib_addr): """Inject a synthetic requirements library that references a local wheel. diff --git a/src/python/pants/backend/python/tasks/setup_py.py b/src/python/pants/backend/python/tasks/setup_py.py index 8c76453a2538..59a213664942 100644 --- a/src/python/pants/backend/python/tasks/setup_py.py +++ b/src/python/pants/backend/python/tasks/setup_py.py @@ -14,8 +14,10 @@ from collections import OrderedDict, defaultdict from pex.compatibility import string, to_bytes +from pex.executor import Executor from pex.installer import InstallerBase, Packager from pex.interpreter import PythonInterpreter +from pex.tracer import TRACER from twitter.common.collections import OrderedSet from twitter.common.dirutil.chroot import Chroot @@ -51,8 +53,11 @@ class SetupPyRunner(InstallerBase): _EXTRAS = ('setuptools', 'wheel') - def __init__(self, source_dir, setup_command, **kw): + class SetupPyRunnerError(Exception): pass + + def __init__(self, source_dir, setup_command, file_input=None, **kw): self.__setup_command = setup_command + self._file_input = file_input super(SetupPyRunner, self).__init__(source_dir, **kw) def mixins(self): @@ -74,6 +79,42 @@ def mixins(self): def _setup_command(self): return self.__setup_command + def run(self): + """???/ripped from the pex installer class + + Used so we can control the command line, and raise an exception on failure. + """ + if self._installed is not None: + return self._installed + + if self._file_input is None: + stdin_payload = self.bootstrap_script.encode('ascii') + init_args = [self._interpreter.binary, '-'] + else: + stdin_payload = None + init_args = [self._interpreter.binary, self._file_input, '--'] + + full_command = init_args + self._setup_command() + + with TRACER.timed('Installing %s' % self._install_tmp, V=2): + try: + Executor.execute(full_command, + env=self._interpreter.sanitized_environment(), + cwd=self._source_dir, + stdin_payload=stdin_payload) + self._installed = True + except Executor.NonZeroExit as e: + self._installed = False + name = os.path.basename(self._source_dir) + raise self.SetupPyRunnerError( + '**** Failed to install {name} (caused by: {error}\n):' + 'stdout:\n{stdout}\nstderr:\n{stderr}\n' + .format(name=name, + error=e, + stdout=e.stdout, + stderr=e.stderr)) + return self._installed + class TargetAncestorIterator(object): """Supports iteration of target ancestor lineages.""" diff --git a/src/python/pants/binaries/executable_pex_tool.py b/src/python/pants/binaries/executable_pex_tool.py index 1483a12d06f9..057786e46272 100644 --- a/src/python/pants/binaries/executable_pex_tool.py +++ b/src/python/pants/binaries/executable_pex_tool.py @@ -5,23 +5,14 @@ from __future__ import absolute_import, division, print_function, unicode_literals import logging -import os -from abc import abstractproperty -from builtins import str -from future.utils import text_type -from pex.interpreter import PythonInterpreter from pex.pex import PEX from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo from pants.backend.python.tasks.pex_build_util import dump_requirements -from pants.binaries.binary_util import BinaryRequest, BinaryUtil -from pants.engine.fs import PathGlobs, PathGlobsAndRoot -from pants.fs.archive import XZCompressedTarArchiver, create_archiver from pants.subsystem.subsystem import Subsystem -from pants.util.dirutil import safe_concurrent_creation -from pants.util.memo import memoized_method, memoized_property +from pants.util.dirutil import is_executable, safe_concurrent_creation logger = logging.getLogger(__name__) @@ -31,19 +22,18 @@ class ExecutablePexTool(Subsystem): entry_point = None - @abstractproperty - def pex_tool_requirements(self): pass + base_requirements = [] - def bootstrap(self, interpreter, pex_file_path): + def bootstrap(self, interpreter, pex_file_path, extra_reqs=None): pex_info = PexInfo.default() if self.entry_point is not None: pex_info.entry_point = self.entry_point - if os.path.exists(pex_file_path): + if is_executable(pex_file_path): return PEX(pex_file_path, interpreter) else: with safe_concurrent_creation(pex_file_path) as safe_path: - builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info) - reqs = self.pex_tool_requirements - dump_requirements(builder, interpreter, reqs, logger) - builder.freeze() + builder = PEXBuilder(interpreter=interpreter, pex_info=pex_info) + all_reqs = list(self.base_requirements) + list(extra_reqs or []) + dump_requirements(builder, interpreter, all_reqs, logger) + builder.build(safe_path) return PEX(pex_file_path, interpreter)