Upgrade to Pex 2.0.3. (#8704)

The primary changes to adapt to were:
1. The `pex.resolver.resolve` function signature changed.
2. The results of resolutions are now installed wheel chroots instead
   of zipped wheels.
3. Some packaging code once used by Pex and our plugin tests is now
   deleted.

3rdparty/python/requirements.txt

@@ -12,7 +12,7 @@ Markdown==2.1.1
 packaging==16.8
 parameterized==0.6.1
 pathspec==0.5.9
-pex==1.6.12
+pex==2.0.3
 psutil==5.6.3
 Pygments==2.3.1
 pyopenssl==17.3.0

pants.travis-ci.ini

@@ -4,18 +4,22 @@
 # Turn off all nailgun use.
 execution_strategy: subprocess
 
+# If we use typical default process parallelism tied to core count, we see too many cores under
+# travis and either get oomkilled from launching too many processes with too much total memory
+# overhead or else just generally thrash the container and slow things down.
+travis_parallelism: 4
+
 [compile.rsc]
-# If we use the default of 1 worker per core, we see too many cores under travis
-# and get oomkilled from launching too many workers with too much total memory
-# overhead.
-worker_count: 4
+worker_count: %(travis_parallelism)s
+
+[python-setup]
+resolver_jobs: %(travis_parallelism)s
 
 [test.pytest]
 # NB: We set a maximum timeout of 9.8 minutes to fail before hitting Travis' 10 minute timeout (which
 # doesn't give us useful debug info).
 timeout_maximum: 590
 
-
 [test.junit]
 # NB: See `test.pytest`.
 timeout_maximum: 540

src/python/pants/backend/python/rules/download_pex_bin.py

@@ -59,8 +59,8 @@ def create_execute_request(self,
 @rule
 async def download_pex_bin() -> DownloadedPexBin:
   # TODO: Inject versions and digests here through some option, rather than hard-coding it.
-  url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.12/pex'
-  digest = Digest('ce64cb72cd23d2123dd48126af54ccf2b718d9ecb98c2ed3045ed1802e89e7e1', 1842359)
+  url = 'https://github.com/pantsbuild/pex/releases/download/v2.0.3/pex'
+  digest = Digest('183a14145553186ca1c0f2877e5eb3a1d7504501f711bb7b84b281342ffbd5ce', 2427459)
   snapshot = await Get(Snapshot, UrlToFetch(url, digest))
   return DownloadedPexBin(executable=snapshot.files[0], directory_digest=snapshot.directory_digest)
 

src/python/pants/backend/python/rules/pex.py

@@ -97,6 +97,8 @@ async def create_pex(
   interpreter constraints."""
 
   argv = ["--output-file", request.output_filename]
+  if python_setup.resolver_jobs:
+    argv.extend(["--jobs", python_setup.resolver_jobs])
   if request.entry_point is not None:
     argv.extend(["--entry-point", request.entry_point])
   argv.extend(request.interpreter_constraints.generate_pex_arg_list())

src/python/pants/backend/python/subsystems/pex_build_util.py

@@ -7,9 +7,8 @@
 from pathlib import Path
 from typing import Callable, Sequence, Set
 
-from pex.fetcher import Fetcher
 from pex.pex_builder import PEXBuilder
-from pex.resolver import resolve
+from pex.resolver import resolve_multi
 from pex.util import DistributionHelper
 from twitter.common.collections import OrderedSet
 
@@ -26,7 +25,6 @@
 from pants.build_graph.files import Files
 from pants.build_graph.target import Target
 from pants.subsystem.subsystem import Subsystem
-from pants.util.collections import assert_single_element
 from pants.util.contextutil import temporary_file
 
 
@@ -177,112 +175,62 @@ def __init__(self,
   def add_requirement_libs_from(self, req_libs, platforms=None):
     """Multi-platform dependency resolution for PEX files.
 
-    :param builder: Dump the requirements into this builder.
-    :param interpreter: The :class:`PythonInterpreter` to resolve requirements for.
     :param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve.
-    :param log: Use this logger.
     :param platforms: A list of :class:`Platform`s to resolve requirements for.
                       Defaults to the platforms specified by PythonSetup.
     """
     reqs = [req for req_lib in req_libs for req in req_lib.requirements]
     self.add_resolved_requirements(reqs, platforms=platforms)
 
-  class SingleDistExtractionError(Exception): pass
+  def resolve_distributions(self, reqs, platforms=None):
+    """Multi-platform dependency resolution.
 
-  def extract_single_dist_for_current_platform(self, reqs, dist_key):
-    """Resolve a specific distribution from a set of requirements matching the current platform.
-
-    :param list reqs: A list of :class:`PythonRequirement` to resolve.
-    :param str dist_key: The value of `distribution.key` to match for a `distribution` from the
-                         resolved requirements.
-    :return: The single :class:`pkg_resources.Distribution` matching `dist_key`.
-    :raises: :class:`self.SingleDistExtractionError` if no dists or multiple dists matched the given
-             `dist_key`.
+    :param reqs: A list of :class:`PythonRequirement` to resolve.
+    :param platforms: A list of platform strings to resolve requirements for.
+                      Defaults to the platforms specified by PythonSetup.
+    :returns: List of :class:`pex.resolver.ResolvedDistribution` instances meeting requirements for
+              the given platforms.
     """
-    distributions = self._resolve_distributions_by_platform(reqs, platforms=['current'])
-    try:
-      matched_dist = assert_single_element(list(
-        dist
-        for _, dists in distributions.items()
-        for dist in dists
-        if dist.key == dist_key
-      ))
-    except (StopIteration, ValueError) as e:
-      raise self.SingleDistExtractionError(
-        f"Exactly one dist was expected to match name {dist_key} in requirements {reqs}: {e!r}"
-      )
-    return matched_dist
-
-  def _resolve_distributions_by_platform(self, reqs, platforms):
     deduped_reqs = OrderedSet(reqs)
     find_links = OrderedSet()
     for req in deduped_reqs:
-      self._log.debug(f'  Dumping requirement: {req}')
-      self._builder.add_requirement(str(req.requirement))
       if req.repository:
         find_links.add(req.repository)
 
-    # Resolve the requirements into distributions.
-    distributions = self._resolve_multi(self._builder.interpreter, deduped_reqs, platforms,
-      find_links)
-    return distributions
+    return self._resolve_multi(deduped_reqs, platforms=platforms, find_links=find_links)
 
   def add_resolved_requirements(self, reqs, platforms=None):
     """Multi-platform dependency resolution for PEX files.
 
-    :param builder: Dump the requirements into this builder.
-    :param interpreter: The :class:`PythonInterpreter` to resolve requirements for.
-    :param reqs: A list of :class:`PythonRequirement` to resolve.
-    :param log: Use this logger.
-    :param platforms: A list of :class:`Platform`s to resolve requirements for.
+    :param reqs: A list of :class:`PythonRequirement`s to resolve.
+    :param platforms: A list of platform strings to resolve requirements for.
                       Defaults to the platforms specified by PythonSetup.
     """
-    distributions = self._resolve_distributions_by_platform(reqs, platforms=platforms)
-    locations = set()
-    for platform, dists in distributions.items():
-      for dist in dists:
-        if dist.location not in locations:
-          self._log.debug(f'  Dumping distribution: .../{os.path.basename(dist.location)}')
-          self.add_distribution(dist)
-        locations.add(dist.location)
-
-  def _resolve_multi(self, interpreter, requirements, platforms, find_links):
-    """Multi-platform dependency resolution for PEX files.
+    for resolved_dist in self.resolve_distributions(reqs, platforms=platforms):
+      requirement = resolved_dist.requirement
+      self._log.debug(f'  Dumping requirement: {requirement}')
+      self._builder.add_requirement(str(requirement))
 
-    Returns a list of distributions that must be included in order to satisfy a set of requirements.
-    That may involve distributions for multiple platforms.
+      distribution = resolved_dist.distribution
+      self._log.debug(f'  Dumping distribution: .../{os.path.basename(distribution.location)}')
+      self.add_distribution(distribution)
 
-    :param interpreter: The :class:`PythonInterpreter` to resolve for.
-    :param requirements: A list of :class:`PythonRequirement` objects to resolve.
-    :param platforms: A list of :class:`Platform`s to resolve for.
-    :param find_links: Additional paths to search for source packages during resolution.
-    :return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed
-             to satisfy the requirements on that platform.
-    """
+  def _resolve_multi(self, requirements, platforms=None, find_links=None):
     python_setup = self._python_setup_subsystem
     python_repos = self._python_repos_subsystem
     platforms = platforms or python_setup.platforms
-    find_links = find_links or []
-    distributions = {}
-    fetchers = python_repos.get_fetchers()
-    fetchers.extend(Fetcher([path]) for path in find_links)
-
-    for platform in platforms:
-      requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir,
-        str(interpreter.identity))
-      resolved_dists = resolve(
-        requirements=[str(req.requirement) for req in requirements],
-        interpreter=interpreter,
-        fetchers=fetchers,
-        platform=platform,
-        context=python_repos.get_network_context(),
-        cache=requirements_cache_dir,
-        cache_ttl=python_setup.resolver_cache_ttl,
-        allow_prereleases=python_setup.resolver_allow_prereleases,
-        use_manylinux=python_setup.use_manylinux)
-      distributions[platform] = [resolved_dist.distribution for resolved_dist in resolved_dists]
-
-    return distributions
+    find_links = list(find_links) if find_links else []
+    find_links.extend(python_repos.repos)
+
+    return resolve_multi(
+      requirements=[str(req.requirement) for req in requirements],
+      interpreters=[self._builder.interpreter],
+      indexes=python_repos.indexes,
+      find_links=find_links,
+      platforms=platforms,
+      cache=python_setup.resolver_cache_dir,
+      allow_prereleases=python_setup.resolver_allow_prereleases,
+      max_parallel_jobs=python_setup.resolver_jobs)
 
   def add_sources_from(self, tgt: Target) -> None:
     dump_source = _create_source_dumper(self._builder, tgt)

src/python/pants/backend/python/subsystems/python_native_code.py

@@ -9,12 +9,10 @@
 
 from pants.backend.native.subsystems.native_toolchain import NativeToolchain
 from pants.backend.native.targets.native_library import NativeLibrary
-from pants.backend.python.python_requirement import PythonRequirement
 from pants.backend.python.subsystems import pex_build_util
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_distribution import PythonDistribution
 from pants.base.exceptions import IncompatiblePlatformsError
-from pants.binaries.executable_pex_tool import ExecutablePexTool
 from pants.engine.rules import optionable_rule, rule
 from pants.subsystem.subsystem import Subsystem
 from pants.util.memo import memoized_property
@@ -125,25 +123,6 @@ def check_build_for_current_platform_only(self, targets):
     ))
 
 
-class BuildSetupRequiresPex(ExecutablePexTool):
-  options_scope = 'build-setup-requires-pex'
-
-  @classmethod
-  def register_options(cls, register):
-    super().register_options(register)
-    register('--setuptools-version', advanced=True, fingerprint=True, default='40.6.3',
-             help='The setuptools version to use when executing `setup.py` scripts.')
-    register('--wheel-version', advanced=True, fingerprint=True, default='0.32.3',
-             help='The wheel version to use when executing `setup.py` scripts.')
-
-  @property
-  def base_requirements(self):
-    return [
-      PythonRequirement('setuptools=={}'.format(self.get_options().setuptools_version)),
-      PythonRequirement('wheel=={}'.format(self.get_options().wheel_version)),
-    ]
-
-
 @dataclass(frozen=True)
 class PexBuildEnvironment:
   cpp_flags: Tuple[str, ...]

src/python/pants/backend/python/subsystems/python_repos.py

@@ -1,59 +1,7 @@
 # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-import logging
-
-from pex.fetcher import Fetcher, PyPIFetcher
-from pex.http import RequestsContext, StreamFilelike, requests
-
 from pants.subsystem.subsystem import Subsystem
-from pants.util.memo import memoized_method
-
-
-logger = logging.getLogger(__name__)
-
-
-# TODO: These methods of RequestsContext are monkey-patched out to work around
-# https://github.com/pantsbuild/pex/issues/26: we should upstream a fix for this.
-_REQUESTS_TIMEOUTS = (15, 30)
-
-
-def _open_monkey(self, link):
-  # requests does not support file:// -- so we must short-circuit manually
-  if link.local:
-    return open(link.local_path, 'rb')  # noqa: T802
-  for attempt in range(self._max_retries + 1):
-    try:
-      return StreamFilelike(self._session.get(
-          link.url, verify=self._verify, stream=True, headers={'User-Agent': self.USER_AGENT},
-          timeout=_REQUESTS_TIMEOUTS),
-          link)
-    except requests.exceptions.ReadTimeout:
-      # Connect timeouts are handled by the HTTPAdapter, unfortunately read timeouts are not
-      # so we'll retry them ourselves.
-      logger.log('Read timeout trying to fetch %s, retrying. %d retries remain.' % (
-          link.url,
-          self._max_retries - attempt))
-    except requests.exceptions.RequestException as e:
-      raise self.Error(e)
-
-  raise self.Error(
-      requests.packages.urllib3.exceptions.MaxRetryError(
-          None,
-          link,
-          'Exceeded max retries of %d' % self._max_retries))
-
-
-def _resolve_monkey(self, link):
-  return link.wrap(self._session.head(
-      link.url, verify=self._verify, allow_redirects=True,
-      headers={'User-Agent': self.USER_AGENT},
-      timeout=_REQUESTS_TIMEOUTS,
-  ).url)
-
-
-RequestsContext.open = _open_monkey
-RequestsContext.resolve = _resolve_monkey
 
 
 class PythonRepos(Subsystem):
@@ -75,15 +23,3 @@ def repos(self):
   @property
   def indexes(self):
     return self.get_options().indexes
-
-  @memoized_method
-  def get_fetchers(self):
-    fetchers = []
-    fetchers.extend(Fetcher([url]) for url in self.repos)
-    fetchers.extend(PyPIFetcher(url) for url in self.indexes)
-    return fetchers
-
-  @memoized_method
-  def get_network_context(self):
-    # TODO(wickman): Add retry, conn_timeout, threads, etc configuration here.
-    return RequestsContext()

src/python/pants/backend/python/subsystems/python_setup.py

@@ -44,10 +44,6 @@ def register_options(cls, register):
     register('--resolver-cache-dir', advanced=True, default=None, metavar='<dir>',
              help='The parent directory for the requirement resolver cache. '
                   'If unspecified, a standard path under the workdir is used.')
-    register('--resolver-cache-ttl', advanced=True, type=int, metavar='<seconds>',
-             default=10 * 365 * 86400,  # 10 years.
-             help='The time in seconds before we consider re-resolving an open-ended requirement, '
-                  'e.g. "flask>=0.2" if a matching distribution is available on disk.')
     register('--resolver-allow-prereleases', advanced=True, type=bool, default=UnsetBool,
              fingerprint=True, help='Whether to include pre-releases when resolving requirements.')
     register('--artifact-cache-dir', advanced=True, default=None, metavar='<dir>',
@@ -60,9 +56,8 @@ def register_options(cls, register):
                   '"<PATH>" (the contents of the PATH env var), '
                   '"<PEXRC>" (paths in the PEX_PYTHON_PATH variable in a pexrc file), '
                   '"<PYENV>" (all python versions under $(pyenv root)/versions).')
-    register('--resolver-use-manylinux', advanced=True, type=bool, default=True, fingerprint=True,
-             help='Whether to consider manylinux wheels when resolving requirements for linux '
-                  'platforms.')
+    register('--resolver-jobs', type=int, default=None, advanced=True, fingerprint=True,
+             help='The maximum number of concurrent jobs to resolve wheels with.')
 
   @property
   def interpreter_constraints(self):
@@ -105,17 +100,13 @@ def resolver_cache_dir(self):
     return (self.get_options().resolver_cache_dir or
             os.path.join(self.scratch_dir, 'resolved_requirements'))
 
-  @property
-  def resolver_cache_ttl(self):
-    return self.get_options().resolver_cache_ttl
-
   @property
   def resolver_allow_prereleases(self):
     return self.get_options().resolver_allow_prereleases
 
   @property
-  def use_manylinux(self):
-    return self.get_options().resolver_use_manylinux
+  def resolver_jobs(self):
+    return self.get_options().resolver_jobs
 
   @property
   def artifact_cache_dir(self):