From 38ccf42c49f31e26462a266e8c63fb7875bcec14 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 8 Oct 2018 09:19:37 -0400 Subject: [PATCH] Upgrade to pex 1.4.8 and eliminate workarounds. (#6594) Pex now allows us to float setuptools high and run on python 3.7 and it supports sane platform expansion as well as graceful handling of non-standard `setuptools` platform reporting for Apple system interpreters allowing us to eliminate several workarounds. We also pickup a fix for concurrent pex extraction which is particularly useful when running pants from a pex using `PEX_FORCE_LOCAL`. Fixes #5922 Fixes #6363 --- .travis.yml | 6 -- 3rdparty/python/requirements.txt | 6 +- build-support/bin/ci.sh | 4 +- src/python/pants/backend/python/BUILD | 8 -- .../pants/backend/python/interpreter_cache.py | 9 +-- src/python/pants/backend/python/pex_util.py | 73 ------------------- .../backend/python/subsystems/python_setup.py | 4 +- src/python/pants/backend/python/tasks/BUILD | 1 - .../backend/python/tasks/pex_build_util.py | 3 +- .../pants/backend/python/tasks/pytest_prep.py | 18 ++++- 10 files changed, 25 insertions(+), 107 deletions(-) delete mode 100644 src/python/pants/backend/python/pex_util.py diff --git a/.travis.yml b/.travis.yml index b307cf9808d..06db876274d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -241,12 +241,6 @@ matrix: osx_image: xcode8.3 stage: Test Pants language: generic - env: - # Specifically avoid the OSX provided 2.7.10 under xcode8.3 since it returns a platform - # of `macosx-*-intel` where the `intel` suffix is bogus but pex has not yet been taught to - # deal with this. Can be removed when this issue is resolved: - # https://github.com/pantsbuild/pex/issues/523 - - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>2.7.10,<3']" before_install: - brew tap caskroom/cask && brew update && brew cask install osxfuse before_script: diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index c89ce00757e..183268df2e3 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -14,7 +14,7 @@ mock==2.0.0 packaging==16.8 parameterized==0.6.1 pathspec==0.5.0 -pex==1.4.5 +pex==1.4.8 psutil==4.3.0 pycodestyle==2.4.0 pyflakes==2.0.0 @@ -27,9 +27,9 @@ pywatchman==1.4.1 requests[security]>=2.5.0,<2.19 scandir==1.2 setproctitle==1.1.10 -setuptools==33.1.1 +setuptools==40.4.3 six>=1.9.0,<2 subprocess32==3.2.7 ; python_version<'3' thrift>=0.9.1 -wheel==0.29.0 +wheel==0.31.1 www-authenticate==0.9.2 diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index 0d1d8fcc134..524e65b1793 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -130,9 +130,7 @@ fi if [[ "${python_three:-false}" == "true" ]]; then # The 3.4 end of this constraint is necessary to jive with the travis ubuntu trusty image. banner "Setting interpreter constraints for 3!" - # TODO(John Sirois): Allow `<4` when the issues with `3.7` are fixed. See: - # https://github.com/pantsbuild/pants/issues/6363 - export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.4,<3.7"]' + export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.4,<4"]' # TODO: Clear interpreters, otherwise this constraint does not end up applying due to a cache # bug between the `./pants binary` and further runs. ./pants.pex clean-all diff --git a/src/python/pants/backend/python/BUILD b/src/python/pants/backend/python/BUILD index ef8ca6bcb2e..b2d036c019e 100644 --- a/src/python/pants/backend/python/BUILD +++ b/src/python/pants/backend/python/BUILD @@ -41,7 +41,6 @@ python_library( dependencies = [ '3rdparty/python:future', '3rdparty/python:pex', - ':pex_util', 'src/python/pants/backend/python/targets', 'src/python/pants/base:exceptions', 'src/python/pants/process', @@ -88,10 +87,3 @@ python_library( ] ) -python_library( - name = 'pex_util', - sources = ['pex_util.py'], - dependencies = [ - '3rdparty/python:pex', - ] -) diff --git a/src/python/pants/backend/python/interpreter_cache.py b/src/python/pants/backend/python/interpreter_cache.py index 670c36ba595..2a112b8f171 100644 --- a/src/python/pants/backend/python/interpreter_cache.py +++ b/src/python/pants/backend/python/interpreter_cache.py @@ -14,7 +14,6 @@ from pex.resolver import resolve from pex.variables import Variables -from pants.backend.python.pex_util import expand_and_maybe_adjust_platform from pants.backend.python.targets.python_target import PythonTarget from pants.base.exceptions import TaskError from pants.process.lock import OwnerPrintingInterProcessFileLock @@ -225,11 +224,9 @@ def _resolve_and_link(self, interpreter, requirement, target_link): distributions = resolve(requirements=[requirement], fetchers=self._python_repos.get_fetchers(), interpreter=interpreter, - platform=expand_and_maybe_adjust_platform( - interpreter=interpreter, - # The local interpreter cache is, by definition, composed of - # interpreters for the 'current' platform. - platform='current'), + # The local interpreter cache is, by definition, composed of + # interpreters for the 'current' platform. + platform='current', context=self._python_repos.get_network_context(), precedence=precedence) if not distributions: diff --git a/src/python/pants/backend/python/pex_util.py b/src/python/pants/backend/python/pex_util.py deleted file mode 100644 index 4d77e7af941..00000000000 --- a/src/python/pants/backend/python/pex_util.py +++ /dev/null @@ -1,73 +0,0 @@ -# coding=utf-8 -# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -import logging - -from pex.interpreter import PythonInterpreter -from pex.platforms import Platform - - -logger = logging.getLogger(__name__) - - -def _interpreter_str(interp): - ident = interp.identity - return ('PythonInterpreter({binary!r}, {identity!r} with extended info: ' - '(abbr_impl: {abbr_impl!r}, impl_ver: {impl_ver!r}, abi_tag: {abi_tag!r}))' - .format(binary=interp.binary, - identity=ident, - abbr_impl=ident.abbr_impl, - impl_ver=ident.impl_ver, - abi_tag=ident.abi_tag)) - - -def expand_and_maybe_adjust_platform(interpreter, platform): - """Adjusts `platform` if it is 'current' and does not match the given `interpreter` platform. - - :param interpreter: The target interpreter for the given `platform`. - :type interpreter: :class:`pex.interpreter.PythonInterpreter` - :param platform: The platform name to expand and maybe adjust. - :type platform: text - :returns: The `platform`, potentially adjusted. - :rtype: :class:`pex.platforms.Platform` - """ - # TODO(John Sirois): Kill all usages when https://github.com/pantsbuild/pex/issues/511 is fixed. - cur_plat = Platform.current() - - if cur_plat.platform != Platform.create(platform).platform: - # IE: Say we're on OSX and platform was 'linux-x86_64' or 'linux_x86_64-cp-27-cp27mu'. - return Platform.create(platform) - - ii = interpreter.identity - if (ii.abbr_impl, ii.impl_ver, ii.abi_tag) == (cur_plat.impl, cur_plat.version, cur_plat.abi): - # IE: Say we're on Linux and platform was 'current' or 'linux-x86_64' or - # 'linux_x86_64-cp-27-cp27mu'and the current extended platform info matches the given - # interpreter exactly. - return cur_plat - - # Otherwise we need to adjust the platform to match a local interpreter different from the - # currently executing interpreter. - interpreter_platform = Platform(platform=cur_plat.platform, - impl=ii.abbr_impl, - version=ii.impl_ver, - abi=ii.abi_tag) - - logger.debug(""" -Modifying given platform of {given_platform!r}: -Using the current platform of {current_platform!r} -Under current interpreter {current_interpreter!r} - -To match given interpreter {given_interpreter!r}. - -Calculated platform: {calculated_platform!r}""".format( - given_platform=platform, - current_platform=cur_plat, - current_interpreter=_interpreter_str(PythonInterpreter.get()), - given_interpreter=_interpreter_str(interpreter), - calculated_platform=interpreter_platform) - ) - - return interpreter_platform diff --git a/src/python/pants/backend/python/subsystems/python_setup.py b/src/python/pants/backend/python/subsystems/python_setup.py index 4bcaebe95cd..c26164405bf 100644 --- a/src/python/pants/backend/python/subsystems/python_setup.py +++ b/src/python/pants/backend/python/subsystems/python_setup.py @@ -26,9 +26,9 @@ def register_options(cls, register): "or 'PyPy' (A pypy interpreter of any version). Multiple constraint strings will " "be ORed together. These constraints are applied in addition to any " "compatibilities required by the relevant targets.") - register('--setuptools-version', advanced=True, default='33.1.1', + register('--setuptools-version', advanced=True, default='40.4.3', help='The setuptools version for this python environment.') - register('--wheel-version', advanced=True, default='0.29.0', + register('--wheel-version', advanced=True, default='0.31.1', help='The wheel version for this python environment.') register('--platforms', advanced=True, type=list, metavar='', default=['current'], help='A list of platforms to be supported by this python environment. Each platform' diff --git a/src/python/pants/backend/python/tasks/BUILD b/src/python/pants/backend/python/tasks/BUILD index c4bea86f136..64c8aab501e 100644 --- a/src/python/pants/backend/python/tasks/BUILD +++ b/src/python/pants/backend/python/tasks/BUILD @@ -13,7 +13,6 @@ python_library( 'src/python/pants/backend/native/targets', 'src/python/pants/backend/native/tasks', 'src/python/pants/backend/python:interpreter_cache', - 'src/python/pants/backend/python:pex_util', 'src/python/pants/backend/python:python_requirement', 'src/python/pants/backend/python/subsystems', 'src/python/pants/backend/python/targets', diff --git a/src/python/pants/backend/python/tasks/pex_build_util.py b/src/python/pants/backend/python/tasks/pex_build_util.py index d61d4da6a41..4c739b202da 100644 --- a/src/python/pants/backend/python/tasks/pex_build_util.py +++ b/src/python/pants/backend/python/tasks/pex_build_util.py @@ -12,7 +12,6 @@ from pex.resolver import resolve from twitter.common.collections import OrderedSet -from pants.backend.python.pex_util import expand_and_maybe_adjust_platform from pants.backend.python.subsystems.python_repos import PythonRepos from pants.backend.python.subsystems.python_setup import PythonSetup from pants.backend.python.targets.python_binary import PythonBinary @@ -160,7 +159,7 @@ def resolve_multi(interpreter, requirements, platforms, find_links): requirements=[req.requirement for req in requirements], interpreter=interpreter, fetchers=fetchers, - platform=expand_and_maybe_adjust_platform(interpreter=interpreter, platform=platform), + platform=platform, context=python_repos.get_network_context(), cache=requirements_cache_dir, cache_ttl=python_setup.resolver_cache_ttl, diff --git a/src/python/pants/backend/python/tasks/pytest_prep.py b/src/python/pants/backend/python/tasks/pytest_prep.py index a325a809691..25b0724beea 100644 --- a/src/python/pants/backend/python/tasks/pytest_prep.py +++ b/src/python/pants/backend/python/tasks/pytest_prep.py @@ -8,6 +8,9 @@ from builtins import object import pkg_resources +from pex.interpreter import PythonInterpreter +from pex.pex import PEX +from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo from pants.backend.python.subsystems.pytest import PyTest @@ -23,8 +26,15 @@ class PytestBinary(object): _COVERAGE_PLUGIN_MODULE_NAME = '__{}__'.format(__name__.replace('.', '_')) - def __init__(self, pex): - self._pex = pex + def __init__(self, interpreter, pex): + # Here we hack around `coverage.cmdline` nuking the 0th element of `sys.path` (our root pex) + # by ensuring, the root pex is on the sys.path twice. + # See: https://github.com/nedbat/coveragepy/issues/715 + pex_path = pex.path() + pex_info = PexInfo.from_pex(pex_path) + pex_info.merge_pex_path(pex_path) # We're now on the sys.path twice. + PEXBuilder(pex_path, interpreter=interpreter, pex_info=pex_info).freeze() + self._pex = PEX(pex=pex_path, interpreter=interpreter) @property def pex(self): @@ -76,4 +86,6 @@ def execute(self): pex_info = PexInfo.default() pex_info.entry_point = 'pytest' pytest_binary = self.create_pex(pex_info) - self.context.products.register_data(self.PytestBinary, self.PytestBinary(pytest_binary)) + interpreter = self.context.products.get_data(PythonInterpreter) + self.context.products.register_data(self.PytestBinary, + self.PytestBinary(interpreter, pytest_binary))