Upgrade to Pex 2.1.4 (#8881)

The primary changes to adapt to were:
1. The `pex.resolver.resolve` function signature changed.
2. The results of resolutions are now installed wheel chroots instead
   of zipped wheels.
3. Some packaging code once used by Pex and our plugin tests is now
   deleted.

Fixes #8786

This is built on top of:

Revert "Revert "Upgrade to Pex 2.0.3. (#8704)" (#8787)"
    
    This reverts commit 91d4af0.

3rdparty/python/requirements.txt

@@ -3,16 +3,14 @@ beautifulsoup4>=4.6.0,<4.7
 cffi==1.13.2
 contextlib2==0.5.5
 coverage>=4.5,<4.6
-# TODO remove this pin once we resolve https://github.com/pantsbuild/pants/issues/8502
-cryptography==2.7
 dataclasses==0.6
 docutils==0.14
 fasteners==0.15.0
 Markdown==2.1.1
 packaging==16.8
 parameterized==0.6.1
 pathspec==0.5.9
-pex==1.6.12
+pex==2.1.4
 psutil==5.6.3
 Pygments==2.3.1
 pyopenssl==17.3.0
@@ -24,8 +22,8 @@ py_zipkin==0.18.4
 requests[security]>=2.20.1
 responses==0.10.4
 setproctitle==1.1.10
-setuptools==40.6.3
+setuptools==44.0.0
 toml==0.10.0
 typing-extensions==3.7.4
-wheel==0.31.1
+wheel==0.33.6
 www-authenticate==0.9.2

contrib/awslambda/python/src/python/pants/contrib/awslambda/python/subsystems/lambdex.py

@@ -9,5 +9,5 @@ class Lambdex(PythonToolBase):
     default_version = "lambdex==0.1.3"
     # TODO(John Sirois): Remove when we can upgrade to a version of lambdex with
     # https://github.com/wickman/lambdex/issues/6 fixed.
-    default_extra_requirements = ["setuptools==40.8.0"]
+    default_extra_requirements = ["setuptools==44.0.0"]
     default_entry_point = "lambdex.bin.lambdex"

contrib/python/tests/python/pants_test/contrib/python/checks/tasks/checkstyle/BUILD

@@ -5,7 +5,6 @@ python_tests(
   dependencies=[
     '3rdparty/python:parameterized',
     '3rdparty/python:pex',
-    '3rdparty/python:wheel',
     '//:build_root',
     '//:pants_pex',
     'build-support/regexes',
@@ -20,4 +19,5 @@ python_tests(
     'tests/python/pants_test/backend/python/tasks:python_task_test_base',
   ],
   tags = {'partially_type_checked'},
+  timeout=480,
 )

contrib/python/tests/python/pants_test/contrib/python/checks/tasks/checkstyle/test_checkstyle.py

@@ -17,18 +17,21 @@
 from pants.util.dirutil import safe_mkdtemp, safe_rmtree
 from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase
 from parameterized import parameterized
+from pex import resolver
 from pex.interpreter import PythonInterpreter
-from wheel.install import WheelFile
 
 from pants.contrib.python.checks.tasks.checkstyle.checkstyle import Checkstyle
 
 CHECKER_RESOLVE_METHOD = [("sys.path", True), ("resolve", False)]
 
 
+# IMPORTANT NOTE: This test fails if run in a chroot.
+# To run it, use `./pants test.pytest --no-chroot`.
+# One more reason to kill this plugin...
 class CheckstyleTest(PythonTaskTestBase):
 
     py2_constraint = "CPython>=2.7,<3"
-    py3_constraint = "CPython>=3.4,<3.6"
+    py3_constraint = "CPython>=3.6,<3.8"
 
     @staticmethod
     def build_checker_wheel(root_dir: str) -> str:
@@ -48,13 +51,11 @@ def build_checker_wheel(root_dir: str) -> str:
         return str(next(wheel_files))
 
     @staticmethod
-    def install_wheel(wheel, root_dir):
-        importable_path = os.path.join(root_dir, "install", os.path.basename(wheel))
-        overrides = {
-            path: importable_path for path in ("purelib", "platlib", "headers", "scripts", "data")
-        }
-        WheelFile(wheel).install(force=True, overrides=overrides)
-        return importable_path
+    def install_wheel(wheel):
+        return [
+            resolved_dist.distribution.location
+            for resolved_dist in resolver.resolve(requirements=[wheel])
+        ]
 
     _distdir = None
     _checker_dist = None
@@ -64,7 +65,7 @@ def install_wheel(wheel, root_dir):
     def setUpClass(cls):
         cls._distdir = safe_mkdtemp()
         cls._checker_dist = cls.build_checker_wheel(cls._distdir)
-        cls._checker_dist_importable_path = cls.install_wheel(cls._checker_dist, cls._distdir)
+        cls._checker_dist_importable_path = cls.install_wheel(cls._checker_dist)
 
     @classmethod
     def tearDownClass(cls):
@@ -87,7 +88,7 @@ def resolve_configuration(self, resolve_local=False):
             )
 
             prior = sys.path[:]
-            sys.path.append(self._checker_dist_importable_path)
+            sys.path.extend(self._checker_dist_importable_path)
             try:
                 yield
             finally:

pants.remote.toml

@@ -36,12 +36,17 @@ process_execution_speculation_delay = 15
 # TODO(#7735): This config is not ideal, that we must specify the PATH for both local and remote
 # platforms. This should be replaced by a proper mechanism to differentiate between the two.
 interpreter_search_paths = [
+  # These are the interpreter paths we set up on the remote container, plus `/usr/bin`, so that
+  # pip can find `ld` if necessary.
+  "/pyenv-docker-build/versions/3.7.3/bin:/pyenv-docker-build/versions/3.6.8/bin:/pyenv-docker-build/versions/2.7.15/bin:/usr/bin",
   # We include the host PATH and PEXRC values so that speculation still works.
-  "<PATH>",
-  "<PEXRC>",
-  # This is the $PATH of the docker container, obtained by locally running `$ docker run --tag
-  # rbe-remote-execution sh -c 'echo $PATH'`.
-  "/pyenv-docker-build/versions/3.7.3/bin:/pyenv-docker-build/versions/3.6.8/bin:/pyenv-docker-build/versions/2.7.15/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin",
+  # NOTE: These come after the remote paths. Putting them before the remote paths means generic
+  # bin dirs like /usr/bin will be on the PATH ahead of the pyenv dirs we actually want to use
+  # on the remote side.  The /pyenv-docker-build/ paths are unlikely to exist on local systems,
+  # and so will not interfere with interpreter discovery there. This emphasizes
+  # that we should fix #7735, and not commingle the paths of two unrelated systems.
+  '<PATH>',
+  '<PEXRC>',
 ]
 
 [python-native-code]

pants.toml

@@ -407,7 +407,7 @@ pytest_plugins.add = [
   "pytest-rerunfailures",
   # TODO(#8651): We need this until we switch to implicit namespace packages so that pytest-cov
   #  understands our __init__ files. NB: this version matches 3rdparty/python/requirements.txt.
-  "setuptools==40.6.3",
+  "setuptools==44.0.0",
   "ipdb",
 ]
 

pants.travis-ci.toml

@@ -3,11 +3,16 @@
 # Turn off all nailgun use.
 execution_strategy = "subprocess"
 
+# If we use typical default process parallelism tied to core count, we see too many cores under
+# travis and either get oomkilled from launching too many processes with too much total memory
+# overhead or else just generally thrash the container and slow things down.
+travis_parallelism = 4
+
 [compile.rsc]
-# If we use the default of 1 worker per core, we see too many cores under travis
-# and get oomkilled from launching too many workers with too much total memory
-# overhead.
-worker_count = 4
+worker_count = "%(travis_parallelism)s"
+
+[python-setup]
+resolver_jobs = "%(travis_parallelism)s"
 
 [test.pytest]
 # NB: We set a maximum timeout of 9.8 minutes to fail before hitting Travis' 10 minute timeout (which

src/python/pants/backend/awslambda/python/lambdex.py

@@ -9,5 +9,5 @@ class Lambdex(PythonToolBase):
     default_version = "lambdex==0.1.3"
     # TODO(John Sirois): Remove when we can upgrade to a version of lambdex with
     # https://github.com/wickman/lambdex/issues/6 fixed.
-    default_extra_requirements = ["setuptools==40.8.0"]
+    default_extra_requirements = ["setuptools==44.0.0"]
     default_entry_point = "lambdex.bin.lambdex"

src/python/pants/backend/python/rules/download_pex_bin.py

@@ -37,14 +37,16 @@ def directory_digest(self) -> Digest:
     class Factory(Script):
         options_scope = "download-pex-bin"
         name = "pex"
-        default_version = "v1.6.12"
+        default_version = "v2.1.4"
 
+        # Note: You can compute the digest and size using:
+        # curl -L $URL | tee >(wc -c) >(shasum -a 256) >/dev/null
         default_versions_and_digests = {
             PlatformConstraint.none: ToolForPlatform(
                 digest=Digest(
-                    "ce64cb72cd23d2123dd48126af54ccf2b718d9ecb98c2ed3045ed1802e89e7e1", 1842359
+                    "6c5ae1f6b9aa40c97bd26a154849044b49f4d698a6abb9ac58ce006bda9cbd4a", 2614246
                 ),
-                version=ToolVersion("v1.6.12"),
+                version=ToolVersion("v2.1.4"),
             ),
         }
 

src/python/pants/backend/python/rules/pex.py

@@ -102,10 +102,16 @@ async def create_pex(
     constraints."""
 
     argv = ["--output-file", request.output_filename]
+    if python_setup.resolver_jobs:
+        argv.extend(["--jobs", python_setup.resolver_jobs])
     if request.entry_point is not None:
         argv.extend(["--entry-point", request.entry_point])
     argv.extend(request.interpreter_constraints.generate_pex_arg_list())
     argv.extend(request.additional_args)
+    if python_setup.manylinux:
+        argv.extend(["--manylinux", python_setup.manylinux])
+    else:
+        argv.append("--no-manylinux")
 
     source_dir_name = "source_files"
     argv.append(f"--sources-directory={source_dir_name}")
@@ -125,8 +131,8 @@ async def create_pex(
     merged_digest = await Get[Digest](DirectoriesToMerge(directories=all_inputs))
 
     # NB: PEX outputs are platform dependent so in order to get a PEX that we can use locally, without
-    # cross-building, we specify that out PEX command be run on the current local platform. When we
-    # support cross-building through CLI flags we can configure requests that build a PEX for out
+    # cross-building, we specify that our PEX command be run on the current local platform. When we
+    # support cross-building through CLI flags we can configure requests that build a PEX for our
     # local platform that are able to execute on a different platform, but for now in order to
     # guarantee correct build we need to restrict this command to execute on the same platform type
     # that the output is intended for. The correct way to interpret the keys

src/python/pants/backend/python/subsystems/pex_build_util.py

@@ -25,14 +25,14 @@ def has_python_sources(tgt: Target) -> bool:
     return is_python_target(tgt) and tgt.has_sources()
 
 
-def is_local_python_dist(tgt: Target) -> bool:
-    return isinstance(tgt, PythonDistribution)
-
-
 def has_resources(tgt: Target) -> bool:
     return isinstance(tgt, Files) and tgt.has_sources()
 
 
+def is_local_python_dist(tgt: Target) -> bool:
+    return isinstance(tgt, PythonDistribution)
+
+
 def has_python_requirements(tgt: Target) -> bool:
     return isinstance(tgt, PythonRequirementLibrary)
 

src/python/pants/backend/python/subsystems/python_native_code.py

@@ -10,11 +10,9 @@
 from pants.backend.native.subsystems.native_toolchain import NativeToolchain
 from pants.backend.native.targets.native_library import NativeLibrary
 from pants.backend.python.subsystems import pex_build_util
-from pants.backend.python.subsystems.executable_pex_tool import ExecutablePexTool
 from pants.backend.python.targets.python_distribution import PythonDistribution
 from pants.base.exceptions import IncompatiblePlatformsError
 from pants.engine.rules import rule, subsystem_rule
-from pants.python.python_requirement import PythonRequirement
 from pants.python.python_setup import PythonSetup
 from pants.subsystem.subsystem import Subsystem
 from pants.util.memo import memoized_property
@@ -139,35 +137,6 @@ def check_build_for_current_platform_only(self, targets):
         )
 
 
-class BuildSetupRequiresPex(ExecutablePexTool):
-    options_scope = "build-setup-requires-pex"
-
-    @classmethod
-    def register_options(cls, register):
-        super().register_options(register)
-        register(
-            "--setuptools-version",
-            advanced=True,
-            fingerprint=True,
-            default="40.6.3",
-            help="The setuptools version to use when executing `setup.py` scripts.",
-        )
-        register(
-            "--wheel-version",
-            advanced=True,
-            fingerprint=True,
-            default="0.32.3",
-            help="The wheel version to use when executing `setup.py` scripts.",
-        )
-
-    @property
-    def base_requirements(self):
-        return [
-            PythonRequirement("setuptools=={}".format(self.get_options().setuptools_version)),
-            PythonRequirement("wheel=={}".format(self.get_options().wheel_version)),
-        ]
-
-
 @dataclass(frozen=True)
 class PexBuildEnvironment:
     cpp_flags: Tuple[str, ...]

src/python/pants/backend/python/subsystems/python_setup.py

@@ -2,7 +2,7 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from pants.base.deprecated import deprecated_module
-from pants.python.python_setup import PythonSetup
+from pants.python.python_setup import PythonSetup as PythonSetup
 
 deprecated_module("1.27.0.dev0", "Import from pants.python.python_setup instead.")
 

src/python/pants/backend/python/targets/unpacked_whls.py

@@ -6,6 +6,7 @@
 from twitter.common.collections import maybe_list
 
 from pants.backend.python.targets.import_wheels_mixin import ImportWheelsMixin
+from pants.base.deprecated import deprecated_conditional
 from pants.base.payload import Payload
 from pants.base.payload_field import PrimitiveField
 from pants.build_graph.target import Target
@@ -57,15 +58,21 @@ def __init__(
         :param compatibility: Python interpreter constraints used to create the pex for the requirement
                               target. If unset, the default interpreter constraints are used. This
                               argument is unnecessary unless the native code depends on libpython.
-        :param str within_data_subdir: If provided, descend into '<name>-<version>.data/<subdir>' when
-                                       matching `include_patterns`. For python wheels which declare any
-                                       non-code data, this is usually needed to extract that without
-                                       manually specifying the relative path, including the package
-                                       version. For example, when `data_files` is used in a setup.py,
-                                       `within_data_subdir='data'` will allow specifying
-                                       `include_patterns` matching exactly what is specified in the
-                                       setup.py.
+        :param bool within_data_subdir: If True, descend into '<name>-<version>.data/' when matching
+                                        `include_patterns`. For python wheels which declare any non-code
+                                        data, this is usually needed to extract that without manually
+                                        specifying the relative path, including the package version. For
+                                        example, when `data_files` is used in a setup.py,
+                                        `within_data_subdir=True` will allow specifying
+                                        `include_patterns` matching exactly what is specified in the
+                                        setup.py.
         """
+        deprecated_conditional(
+            lambda: type(within_data_subdir) not in (bool, type(None)),
+            removal_version="1.28.0.dev2",
+            entity_description="A non-boolean value for `within_data_subdir`",
+            hint_message="The location of the .data subdirectory will be inferred from the module name!",
+        )
         payload = payload or Payload()
         payload.add_fields(
             {

src/python/pants/backend/python/tasks/BUILD

@@ -5,6 +5,7 @@ python_library(
   dependencies=[
     '3rdparty/python:dataclasses',
     '3rdparty/python:pex',
+    '3rdparty/python:wheel',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     '3rdparty/python/twitter/commons:twitter.common.dirutil',
     'src/python/pants/backend/native/config',