From bd154f2aa4955d6105d4fefc76901edfb9532ec1 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Thu, 4 Oct 2018 16:58:03 -0400 Subject: [PATCH] Upgrade to pex 1.4.8 and eliminate workarounds. Pex now allows us to float setuptools high and run on python 3.7 and it supports sane platform expansion as well as graceful handling of non-standard `setuptools` platform reporting for Apple system interpreters allowing us to eliminate several workarounds. We also pickup a fix for concurrent pex extraction which is particularly useful when running pants from a pex using `PEX_FORCE_LOCAL`. Fixes #5922 Fixes #6363 --- .travis.yml | 6 -- 3rdparty/python/requirements.txt | 6 +- build-support/bin/ci.sh | 4 +- src/python/pants/backend/python/BUILD | 8 -- .../pants/backend/python/interpreter_cache.py | 9 +-- src/python/pants/backend/python/pex_util.py | 73 ------------------- .../backend/python/subsystems/python_setup.py | 4 +- src/python/pants/backend/python/tasks/BUILD | 1 - .../backend/python/tasks/pex_build_util.py | 3 +- 9 files changed, 10 insertions(+), 104 deletions(-) delete mode 100644 src/python/pants/backend/python/pex_util.py diff --git a/.travis.yml b/.travis.yml index b307cf9808d3..06db876274da 100644 --- a/.travis.yml +++ b/.travis.yml @@ -241,12 +241,6 @@ matrix: osx_image: xcode8.3 stage: Test Pants language: generic - env: - # Specifically avoid the OSX provided 2.7.10 under xcode8.3 since it returns a platform - # of `macosx-*-intel` where the `intel` suffix is bogus but pex has not yet been taught to - # deal with this. Can be removed when this issue is resolved: - # https://github.com/pantsbuild/pex/issues/523 - - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>2.7.10,<3']" before_install: - brew tap caskroom/cask && brew update && brew cask install osxfuse before_script: diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index c89ce00757ed..183268df2e3c 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -14,7 +14,7 @@ mock==2.0.0 packaging==16.8 parameterized==0.6.1 pathspec==0.5.0 -pex==1.4.5 +pex==1.4.8 psutil==4.3.0 pycodestyle==2.4.0 pyflakes==2.0.0 @@ -27,9 +27,9 @@ pywatchman==1.4.1 requests[security]>=2.5.0,<2.19 scandir==1.2 setproctitle==1.1.10 -setuptools==33.1.1 +setuptools==40.4.3 six>=1.9.0,<2 subprocess32==3.2.7 ; python_version<'3' thrift>=0.9.1 -wheel==0.29.0 +wheel==0.31.1 www-authenticate==0.9.2 diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index 0d1d8fcc1342..524e65b1793f 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -130,9 +130,7 @@ fi if [[ "${python_three:-false}" == "true" ]]; then # The 3.4 end of this constraint is necessary to jive with the travis ubuntu trusty image. banner "Setting interpreter constraints for 3!" - # TODO(John Sirois): Allow `<4` when the issues with `3.7` are fixed. See: - # https://github.com/pantsbuild/pants/issues/6363 - export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.4,<3.7"]' + export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.4,<4"]' # TODO: Clear interpreters, otherwise this constraint does not end up applying due to a cache # bug between the `./pants binary` and further runs. ./pants.pex clean-all diff --git a/src/python/pants/backend/python/BUILD b/src/python/pants/backend/python/BUILD index ef8ca6bcb2ed..b2d036c019e6 100644 --- a/src/python/pants/backend/python/BUILD +++ b/src/python/pants/backend/python/BUILD @@ -41,7 +41,6 @@ python_library( dependencies = [ '3rdparty/python:future', '3rdparty/python:pex', - ':pex_util', 'src/python/pants/backend/python/targets', 'src/python/pants/base:exceptions', 'src/python/pants/process', @@ -88,10 +87,3 @@ python_library( ] ) -python_library( - name = 'pex_util', - sources = ['pex_util.py'], - dependencies = [ - '3rdparty/python:pex', - ] -) diff --git a/src/python/pants/backend/python/interpreter_cache.py b/src/python/pants/backend/python/interpreter_cache.py index 670c36ba5951..2a112b8f1711 100644 --- a/src/python/pants/backend/python/interpreter_cache.py +++ b/src/python/pants/backend/python/interpreter_cache.py @@ -14,7 +14,6 @@ from pex.resolver import resolve from pex.variables import Variables -from pants.backend.python.pex_util import expand_and_maybe_adjust_platform from pants.backend.python.targets.python_target import PythonTarget from pants.base.exceptions import TaskError from pants.process.lock import OwnerPrintingInterProcessFileLock @@ -225,11 +224,9 @@ def _resolve_and_link(self, interpreter, requirement, target_link): distributions = resolve(requirements=[requirement], fetchers=self._python_repos.get_fetchers(), interpreter=interpreter, - platform=expand_and_maybe_adjust_platform( - interpreter=interpreter, - # The local interpreter cache is, by definition, composed of - # interpreters for the 'current' platform. - platform='current'), + # The local interpreter cache is, by definition, composed of + # interpreters for the 'current' platform. + platform='current', context=self._python_repos.get_network_context(), precedence=precedence) if not distributions: diff --git a/src/python/pants/backend/python/pex_util.py b/src/python/pants/backend/python/pex_util.py deleted file mode 100644 index 4d77e7af941e..000000000000 --- a/src/python/pants/backend/python/pex_util.py +++ /dev/null @@ -1,73 +0,0 @@ -# coding=utf-8 -# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -import logging - -from pex.interpreter import PythonInterpreter -from pex.platforms import Platform - - -logger = logging.getLogger(__name__) - - -def _interpreter_str(interp): - ident = interp.identity - return ('PythonInterpreter({binary!r}, {identity!r} with extended info: ' - '(abbr_impl: {abbr_impl!r}, impl_ver: {impl_ver!r}, abi_tag: {abi_tag!r}))' - .format(binary=interp.binary, - identity=ident, - abbr_impl=ident.abbr_impl, - impl_ver=ident.impl_ver, - abi_tag=ident.abi_tag)) - - -def expand_and_maybe_adjust_platform(interpreter, platform): - """Adjusts `platform` if it is 'current' and does not match the given `interpreter` platform. - - :param interpreter: The target interpreter for the given `platform`. - :type interpreter: :class:`pex.interpreter.PythonInterpreter` - :param platform: The platform name to expand and maybe adjust. - :type platform: text - :returns: The `platform`, potentially adjusted. - :rtype: :class:`pex.platforms.Platform` - """ - # TODO(John Sirois): Kill all usages when https://github.com/pantsbuild/pex/issues/511 is fixed. - cur_plat = Platform.current() - - if cur_plat.platform != Platform.create(platform).platform: - # IE: Say we're on OSX and platform was 'linux-x86_64' or 'linux_x86_64-cp-27-cp27mu'. - return Platform.create(platform) - - ii = interpreter.identity - if (ii.abbr_impl, ii.impl_ver, ii.abi_tag) == (cur_plat.impl, cur_plat.version, cur_plat.abi): - # IE: Say we're on Linux and platform was 'current' or 'linux-x86_64' or - # 'linux_x86_64-cp-27-cp27mu'and the current extended platform info matches the given - # interpreter exactly. - return cur_plat - - # Otherwise we need to adjust the platform to match a local interpreter different from the - # currently executing interpreter. - interpreter_platform = Platform(platform=cur_plat.platform, - impl=ii.abbr_impl, - version=ii.impl_ver, - abi=ii.abi_tag) - - logger.debug(""" -Modifying given platform of {given_platform!r}: -Using the current platform of {current_platform!r} -Under current interpreter {current_interpreter!r} - -To match given interpreter {given_interpreter!r}. - -Calculated platform: {calculated_platform!r}""".format( - given_platform=platform, - current_platform=cur_plat, - current_interpreter=_interpreter_str(PythonInterpreter.get()), - given_interpreter=_interpreter_str(interpreter), - calculated_platform=interpreter_platform) - ) - - return interpreter_platform diff --git a/src/python/pants/backend/python/subsystems/python_setup.py b/src/python/pants/backend/python/subsystems/python_setup.py index 4bcaebe95cd9..c26164405bfb 100644 --- a/src/python/pants/backend/python/subsystems/python_setup.py +++ b/src/python/pants/backend/python/subsystems/python_setup.py @@ -26,9 +26,9 @@ def register_options(cls, register): "or 'PyPy' (A pypy interpreter of any version). Multiple constraint strings will " "be ORed together. These constraints are applied in addition to any " "compatibilities required by the relevant targets.") - register('--setuptools-version', advanced=True, default='33.1.1', + register('--setuptools-version', advanced=True, default='40.4.3', help='The setuptools version for this python environment.') - register('--wheel-version', advanced=True, default='0.29.0', + register('--wheel-version', advanced=True, default='0.31.1', help='The wheel version for this python environment.') register('--platforms', advanced=True, type=list, metavar='', default=['current'], help='A list of platforms to be supported by this python environment. Each platform' diff --git a/src/python/pants/backend/python/tasks/BUILD b/src/python/pants/backend/python/tasks/BUILD index c4bea86f1360..64c8aab501e6 100644 --- a/src/python/pants/backend/python/tasks/BUILD +++ b/src/python/pants/backend/python/tasks/BUILD @@ -13,7 +13,6 @@ python_library( 'src/python/pants/backend/native/targets', 'src/python/pants/backend/native/tasks', 'src/python/pants/backend/python:interpreter_cache', - 'src/python/pants/backend/python:pex_util', 'src/python/pants/backend/python:python_requirement', 'src/python/pants/backend/python/subsystems', 'src/python/pants/backend/python/targets', diff --git a/src/python/pants/backend/python/tasks/pex_build_util.py b/src/python/pants/backend/python/tasks/pex_build_util.py index d61d4da6a411..4c739b202da5 100644 --- a/src/python/pants/backend/python/tasks/pex_build_util.py +++ b/src/python/pants/backend/python/tasks/pex_build_util.py @@ -12,7 +12,6 @@ from pex.resolver import resolve from twitter.common.collections import OrderedSet -from pants.backend.python.pex_util import expand_and_maybe_adjust_platform from pants.backend.python.subsystems.python_repos import PythonRepos from pants.backend.python.subsystems.python_setup import PythonSetup from pants.backend.python.targets.python_binary import PythonBinary @@ -160,7 +159,7 @@ def resolve_multi(interpreter, requirements, platforms, find_links): requirements=[req.requirement for req in requirements], interpreter=interpreter, fetchers=fetchers, - platform=expand_and_maybe_adjust_platform(interpreter=interpreter, platform=platform), + platform=platform, context=python_repos.get_network_context(), cache=requirements_cache_dir, cache_ttl=python_setup.resolver_cache_ttl,