From 31af708a9914f6d0590c88011dd44f4b4a167e4e Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 13 Dec 2019 20:51:46 -0800 Subject: [PATCH] Fix pex resolution to respect --ignore-errors. (#828) Previously, pip might warn in red that versions were mismatched but we would get no failure until pex runtime resolution. Improve pex runtime resolution error messages and implement pex buildtime resolution sanity checks when --no-ignore-errors. --- pex/bin/pex.py | 7 +- pex/environment.py | 59 +++-- pex/package.py | 12 +- pex/pex_builder.py | 6 +- pex/pip.py | 378 ++++++++++++++++------------ pex/resolver.py | 52 +++- pex/testing.py | 6 +- pex/variables.py | 5 + tests/test_finders.py | 4 +- tests/test_integration.py | 6 +- tests/test_unified_install_cache.py | 4 +- 11 files changed, 338 insertions(+), 201 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index aeb4f440b..dc34a517f 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -234,8 +234,8 @@ def configure_clp_pex_options(parser): dest='ignore_errors', default=False, action='store_true', - help='Ignore run-time requirement resolution errors when invoking the pex. ' - '[Default: %default]') + help='Ignore requirement resolution solver errors when building pexes and later invoking ' + 'them. [Default: %default]') group.add_option( '--inherit-path', @@ -565,7 +565,8 @@ def walk_and_do(fn, src_dir): build=options.build, use_wheel=options.use_wheel, compile=options.compile, - max_parallel_jobs=options.max_parallel_jobs) + max_parallel_jobs=options.max_parallel_jobs, + ignore_errors=options.ignore_errors) for resolved_dist in resolveds: log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution), diff --git a/pex/environment.py b/pex/environment.py index ded45f9b4..40b818ca2 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -9,17 +9,18 @@ import site import sys import zipfile -from collections import OrderedDict +from collections import OrderedDict, defaultdict from pex import pex_builder, pex_warnings from pex.bootstrap import Bootstrap from pex.common import atomic_directory, die, open_zip from pex.interpreter import PythonInterpreter +from pex.orderedset import OrderedSet from pex.package import distribution_compatible from pex.platforms import Platform from pex.third_party.pkg_resources import DistributionNotFound, Environment, Requirement, WorkingSet from pex.tracer import TRACER -from pex.util import CacheHelper +from pex.util import CacheHelper, DistributionHelper def _import_pkg_resources(): @@ -226,16 +227,16 @@ def activate(self): return self._working_set def _resolve(self, working_set, reqs): - reqs = reqs[:] - unresolved_reqs = set() - resolveds = set() + reqs_by_key = OrderedDict((req.key, req) for req in reqs) + unresolved_reqs = OrderedDict() + resolveds = OrderedSet() environment = self._target_interpreter_env.copy() environment['extra'] = list(set(itertools.chain(*(req.extras for req in reqs)))) # Resolve them one at a time so that we can figure out which ones we need to elide should # there be an interpreter incompatibility. - for req in reqs: + for req in reqs_by_key.values(): if req.marker and not req.marker.evaluate(environment=environment): TRACER.log('Skipping activation of `%s` due to environment marker de-selection' % req) continue @@ -244,27 +245,55 @@ def _resolve(self, working_set, reqs): resolveds.update(working_set.resolve([req], env=self)) except DistributionNotFound as e: TRACER.log('Failed to resolve a requirement: %s' % e) - unresolved_reqs.add(e.req.project_name) + requirers = unresolved_reqs.setdefault(e.req, OrderedSet()) if e.requirers: - unresolved_reqs.update(e.requirers) - - unresolved_reqs = set([req.lower() for req in unresolved_reqs]) + requirers.update(reqs_by_key[requirer] for requirer in e.requirers) if unresolved_reqs: TRACER.log('Unresolved requirements:') for req in unresolved_reqs: TRACER.log(' - %s' % req) + TRACER.log('Distributions contained within this pex:') + distributions_by_key = defaultdict(list) if not self._pex_info.distributions: TRACER.log(' None') else: - for dist in self._pex_info.distributions: - TRACER.log(' - %s' % dist) + for dist_name, dist_digest in self._pex_info.distributions.items(): + TRACER.log(' - %s' % dist_name) + distribution = DistributionHelper.distribution_from_path( + path=os.path.join(self._pex_info.install_cache, dist_digest, dist_name) + ) + distributions_by_key[distribution.as_requirement().key].append(distribution) + if not self._pex_info.ignore_errors: + items = [] + for index, (requirement, requirers) in enumerate(unresolved_reqs.items()): + rendered_requirers = '' + if requirers: + rendered_requirers = ( + '\n Required by:' + '\n {requirers}' + ).format(requirers='\n '.join(map(str, requirers))) + + items.append( + '{index: 2d}: {requirement}' + '{rendered_requirers}' + '\n But this pex only contains:' + '\n {distributions}'.format( + index=index + 1, + requirement=requirement, + rendered_requirers=rendered_requirers, + distributions='\n '.join(os.path.basename(d.location) + for d in distributions_by_key[requirement.key]) + ) + ) + die( - 'Failed to execute PEX file, missing %s compatible dependencies for:\n%s' % ( - Platform.current(), - '\n'.join(str(r) for r in unresolved_reqs) + 'Failed to execute PEX file. Needed {platform} compatible dependencies for:\n{items}' + .format( + platform=Platform.of_interpreter(self._interpreter), + items='\n\t'.join(items) ) ) diff --git a/pex/package.py b/pex/package.py index 2090226cd..b5a650b7a 100644 --- a/pex/package.py +++ b/pex/package.py @@ -15,12 +15,18 @@ def distribution_compatible(dist, supported_tags=None): by the platform in question; defaults to the current interpreter's supported tags. :returns: True if the distribution is compatible, False if it is unrecognized or incompatible. """ - if supported_tags is None: - supported_tags = get_supported() filename, ext = os.path.splitext(os.path.basename(dist.location)) if ext.lower() != '.whl': - return False + # This supports resolving pex's own vendored distributions which are vendored in directory + # directory with the project name (`pip/` for pip) and not the corresponding wheel name + # (`pip-19.3.1-py2.py3-none-any.whl/` for pip). Pex only vendors universal wheels for all + # platforms it supports at buildtime and runtime so this is always safe. + return True + + if supported_tags is None: + supported_tags = get_supported() + try: name_, raw_version_, py_tag, abi_tag, arch_tag = filename.rsplit('-', 4) except ValueError: diff --git a/pex/pex_builder.py b/pex/pex_builder.py index 36877b3a4..78645a020 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -13,7 +13,7 @@ from pex.finders import get_entry_point_from_console_script, get_script_from_distributions from pex.interpreter import PythonInterpreter from pex.pex_info import PexInfo -from pex.pip import spawn_install_wheel +from pex.pip import get_pip from pex.third_party.pkg_resources import DefaultProvider, ZipProvider, get_provider from pex.tracer import TRACER from pex.util import CacheHelper, DistributionHelper @@ -278,7 +278,7 @@ def _add_dist_dir(self, path, dist_name): def _add_dist_wheel_file(self, path, dist_name): with temporary_dir() as install_dir: - spawn_install_wheel( + get_pip().spawn_install_wheel( wheel=path, install_dir=install_dir, target=DistributionTarget.for_interpreter(self.interpreter) @@ -327,7 +327,7 @@ def add_dist_location(self, dist, name=None): dist_path = dist if os.path.isfile(dist_path) and dist_path.endswith('.whl'): dist_path = os.path.join(safe_mkdtemp(), os.path.basename(dist)) - spawn_install_wheel( + get_pip().spawn_install_wheel( wheel=dist, install_dir=dist_path, target=DistributionTarget.for_interpreter(self.interpreter) diff --git a/pex/pip.py b/pex/pip.py index f6357ec02..944caf30c 100644 --- a/pex/pip.py +++ b/pex/pip.py @@ -4,176 +4,238 @@ from __future__ import absolute_import, print_function +import os from collections import deque +from textwrap import dedent +from pex import third_party from pex.compatibility import urlparse from pex.distribution_target import DistributionTarget -from pex.jobs import spawn_python_job +from pex.jobs import Job from pex.variables import ENV -def _spawn_pip_isolated(args, cache=None, interpreter=None): - pip_args = ['-m', 'pip', '--disable-pip-version-check', '--isolated', '--exists-action', 'i'] +class Pip(object): + @classmethod + def create(cls, path=None): + """Creates a pip tool with PEX isolation at path. - # The max pip verbosity is -vvv and for pex it's -vvvvvvvvv; so we scale down by a factor of 3. - verbosity = ENV.PEX_VERBOSE // 3 - if verbosity > 0: - pip_args.append('-{}'.format('v' * verbosity)) - else: - pip_args.append('-q') + :param str path: The path to build the pip tool pex at; a temporary directory by default. + """ + from pex.pex_builder import PEXBuilder - if cache: - pip_args.extend(['--cache-dir', cache]) - else: - pip_args.append('--no-cache-dir') + isolated_pip_builder = PEXBuilder(path=path) + pythonpath = third_party.expose(['pip', 'setuptools', 'wheel']) + isolated_pip_environment = third_party.pkg_resources.Environment(search_path=pythonpath) + for dist_name in isolated_pip_environment: + for dist in isolated_pip_environment[dist_name]: + isolated_pip_builder.add_dist_location(dist=dist.location) + with open(os.path.join(isolated_pip_builder.path(), 'run_pip.py'), 'w') as fp: + fp.write(dedent("""\ + import os + import runpy + import sys - return spawn_python_job( - args=pip_args + args, - interpreter=interpreter, - expose=['pip', 'setuptools', 'wheel'] - ) + # Propagate un-vendored setuptools and wheel to pip for any legacy setup.py builds it needs + # to perform. + os.environ['__PEX_UNVENDORED__'] = '1' + os.environ['PYTHONPATH'] = os.pathsep.join(sys.path) -def _calculate_package_index_options(indexes=None, find_links=None): - trusted_hosts = [] + runpy.run_module('pip', run_name='__main__') + """)) + isolated_pip_builder.set_executable(fp.name) + isolated_pip_builder.freeze() - def maybe_trust_insecure_host(url): - url_info = urlparse.urlparse(url) - if 'http' == url_info.scheme: - # Implicitly trust explicitly asked for http indexes and find_links repos instead of requiring - # seperate trust configuration. - trusted_hosts.append(url_info.netloc) - return url + return cls(isolated_pip_builder.path()) - # N.B.: We interpret None to mean accept pip index defaults, [] to mean turn off all index use. - if indexes is not None: - if len(indexes) == 0: - yield '--no-index' + def __init__(self, pip_pex_path): + self._pip_pex_path = pip_pex_path + + def _spawn_pip_isolated(self, args, cache=None, interpreter=None): + pip_args = ['--disable-pip-version-check', '--isolated', '--exists-action', 'i'] + + # The max pip verbosity is -vvv and for pex it's -vvvvvvvvv; so we scale down by a factor of 3. + pex_verbosity = ENV.PEX_VERBOSE + pip_verbosity = pex_verbosity // 3 + if pip_verbosity > 0: + pip_args.append('-{}'.format('v' * pip_verbosity)) + else: + pip_args.append('-q') + + if cache: + pip_args.extend(['--cache-dir', cache]) else: - all_indexes = deque(indexes) - yield '--index-url' - yield maybe_trust_insecure_host(all_indexes.popleft()) - if all_indexes: - for extra_index in all_indexes: - yield '--extra-index-url' - yield maybe_trust_insecure_host(extra_index) - - if find_links: - for find_link_url in find_links: - yield '--find-links' - yield maybe_trust_insecure_host(find_link_url) - - for trusted_host in trusted_hosts: - yield '--trusted-host' - yield trusted_host - - -def spawn_download_distributions(download_dir, - requirements=None, - requirement_files=None, - constraint_files=None, - allow_prereleases=False, - transitive=True, - target=None, - indexes=None, - find_links=None, - cache=None, - build=True, - use_wheel=True): - - target = target or DistributionTarget.current() - - platform = target.get_platform() - if not use_wheel: - if not build: - raise ValueError('Cannot both ignore wheels (use_wheel=False) and refrain from building ' - 'distributions (build=False).') - elif target.is_foreign: - raise ValueError('Cannot ignore wheels (use_wheel=False) when resolving for a foreign ' - 'platform: {}'.format(platform)) - - download_cmd = ['download', '--dest', download_dir] - download_cmd.extend(_calculate_package_index_options(indexes=indexes, find_links=find_links)) - - if target.is_foreign: - # We're either resolving for a different host / platform or a different interpreter for the - # current platform that we have no access to; so we need to let pip know and not otherwise - # pickup platform info from the interpreter we execute pip with. - download_cmd.extend(['--platform', platform.platform]) - download_cmd.extend(['--implementation', platform.impl]) - download_cmd.extend(['--python-version', platform.version]) - download_cmd.extend(['--abi', platform.abi]) - - if target.is_foreign or not build: - download_cmd.extend(['--only-binary', ':all:']) - - if not use_wheel: - download_cmd.extend(['--no-binary', ':all:']) - - if allow_prereleases: - download_cmd.append('--pre') - - if not transitive: - download_cmd.append('--no-deps') - - if requirement_files: - for requirement_file in requirement_files: - download_cmd.extend(['--requirement', requirement_file]) - - if constraint_files: - for constraint_file in constraint_files: - download_cmd.extend(['--constraint', constraint_file]) - - download_cmd.extend(requirements) - - return _spawn_pip_isolated(download_cmd, cache=cache, interpreter=target.get_interpreter()) - - -def spawn_build_wheels(distributions, - wheel_dir, - interpreter=None, - indexes=None, - find_links=None, - cache=None): - - wheel_cmd = ['wheel', '--no-deps', '--wheel-dir', wheel_dir] - - # If the build is PEP-517 compliant it may need to resolve build requirements. - wheel_cmd.extend(_calculate_package_index_options(indexes=indexes, find_links=find_links)) - - wheel_cmd.extend(distributions) - return _spawn_pip_isolated(wheel_cmd, cache=cache, interpreter=interpreter) - - -def spawn_install_wheel(wheel, - install_dir, - compile=False, - overwrite=False, - cache=None, - target=None): - - target = target or DistributionTarget.current() - - install_cmd = [ - 'install', - '--no-deps', - '--no-index', - '--only-binary', ':all:', - '--target', install_dir - ] - - interpreter = target.get_interpreter() - if target.is_foreign: - if compile: - raise ValueError('Cannot compile bytecode for {} using {} because the wheel has a foreign ' - 'platform.'.format(wheel, interpreter)) - - # We're installing a wheel for a foreign platform. This is just an unpacking operation though; - # so we don't actually need to perform it with a target platform compatible interpreter. - install_cmd.append('--ignore-requires-python') - - install_cmd.append('--compile' if compile else '--no-compile') - if overwrite: - install_cmd.extend(['--upgrade', '--force-reinstall']) - install_cmd.append(wheel) - return _spawn_pip_isolated(install_cmd, cache=cache, interpreter=interpreter) + pip_args.append('--no-cache-dir') + + command = pip_args + args + with ENV.strip().patch(PEX_ROOT=ENV.PEX_ROOT, PEX_VERBOSE=str(pex_verbosity)) as env: + from pex.pex import PEX + pip = PEX(pex=self._pip_pex_path, interpreter=interpreter) + return Job( + command=pip.cmdline(command), + process=pip.run( + args=command, + env=env, + blocking=False + ) + ) + + def _calculate_package_index_options(self, indexes=None, find_links=None): + trusted_hosts = [] + + def maybe_trust_insecure_host(url): + url_info = urlparse.urlparse(url) + if 'http' == url_info.scheme: + # Implicitly trust explicitly asked for http indexes and find_links repos instead of + # requiring seperate trust configuration. + trusted_hosts.append(url_info.netloc) + return url + + # N.B.: We interpret None to mean accept pip index defaults, [] to mean turn off all index use. + if indexes is not None: + if len(indexes) == 0: + yield '--no-index' + else: + all_indexes = deque(indexes) + yield '--index-url' + yield maybe_trust_insecure_host(all_indexes.popleft()) + if all_indexes: + for extra_index in all_indexes: + yield '--extra-index-url' + yield maybe_trust_insecure_host(extra_index) + + if find_links: + for find_link_url in find_links: + yield '--find-links' + yield maybe_trust_insecure_host(find_link_url) + + for trusted_host in trusted_hosts: + yield '--trusted-host' + yield trusted_host + + def spawn_download_distributions(self, + download_dir, + requirements=None, + requirement_files=None, + constraint_files=None, + allow_prereleases=False, + transitive=True, + target=None, + indexes=None, + find_links=None, + cache=None, + build=True, + use_wheel=True): + + target = target or DistributionTarget.current() + + platform = target.get_platform() + if not use_wheel: + if not build: + raise ValueError('Cannot both ignore wheels (use_wheel=False) and refrain from building ' + 'distributions (build=False).') + elif target.is_foreign: + raise ValueError('Cannot ignore wheels (use_wheel=False) when resolving for a foreign ' + 'platform: {}'.format(platform)) + + download_cmd = ['download', '--dest', download_dir] + package_index_options = self._calculate_package_index_options( + indexes=indexes, + find_links=find_links + ) + download_cmd.extend(package_index_options) + + if target.is_foreign: + # We're either resolving for a different host / platform or a different interpreter for the + # current platform that we have no access to; so we need to let pip know and not otherwise + # pickup platform info from the interpreter we execute pip with. + download_cmd.extend(['--platform', platform.platform]) + download_cmd.extend(['--implementation', platform.impl]) + download_cmd.extend(['--python-version', platform.version]) + download_cmd.extend(['--abi', platform.abi]) + + if target.is_foreign or not build: + download_cmd.extend(['--only-binary', ':all:']) + + if not use_wheel: + download_cmd.extend(['--no-binary', ':all:']) + + if allow_prereleases: + download_cmd.append('--pre') + + if not transitive: + download_cmd.append('--no-deps') + + if requirement_files: + for requirement_file in requirement_files: + download_cmd.extend(['--requirement', requirement_file]) + + if constraint_files: + for constraint_file in constraint_files: + download_cmd.extend(['--constraint', constraint_file]) + + download_cmd.extend(requirements) + + return self._spawn_pip_isolated(download_cmd, cache=cache, interpreter=target.get_interpreter()) + + def spawn_build_wheels(self, + distributions, + wheel_dir, + interpreter=None, + indexes=None, + find_links=None, + cache=None): + + wheel_cmd = ['wheel', '--no-deps', '--wheel-dir', wheel_dir] + + # If the build is PEP-517 compliant it may need to resolve build requirements. + wheel_cmd.extend(self._calculate_package_index_options(indexes=indexes, find_links=find_links)) + + wheel_cmd.extend(distributions) + return self._spawn_pip_isolated(wheel_cmd, cache=cache, interpreter=interpreter) + + def spawn_install_wheel(self, + wheel, + install_dir, + compile=False, + overwrite=False, + cache=None, + target=None): + + target = target or DistributionTarget.current() + + install_cmd = [ + 'install', + '--no-deps', + '--no-index', + '--only-binary', ':all:', + '--target', install_dir + ] + + interpreter = target.get_interpreter() + if target.is_foreign: + if compile: + raise ValueError('Cannot compile bytecode for {} using {} because the wheel has a foreign ' + 'platform.'.format(wheel, interpreter)) + + # We're installing a wheel for a foreign platform. This is just an unpacking operation though; + # so we don't actually need to perform it with a target platform compatible interpreter. + install_cmd.append('--ignore-requires-python') + + install_cmd.append('--compile' if compile else '--no-compile') + if overwrite: + install_cmd.extend(['--upgrade', '--force-reinstall']) + install_cmd.append(wheel) + return self._spawn_pip_isolated(install_cmd, cache=cache, interpreter=interpreter) + + +_PIP = None + + +def get_pip(): + """Returns a lazily instantiated global Pip object that is safe for un-coordinated use.""" + global _PIP + if _PIP is None: + _PIP = Pip.create() + return _PIP diff --git a/pex/resolver.py b/pex/resolver.py index c39f27c29..3c7b9c01f 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -16,7 +16,7 @@ from pex.jobs import SpawnedJob, execute_parallel, spawn_python_job from pex.orderedset import OrderedSet from pex.pex_info import PexInfo -from pex.pip import spawn_build_wheels, spawn_download_distributions, spawn_install_wheel +from pex.pip import get_pip from pex.platforms import Platform from pex.requirements import local_project_from_requirement, local_projects_from_requirement_file from pex.third_party.pkg_resources import Distribution, Environment, Requirement @@ -363,7 +363,7 @@ def _run_parallel(self, inputs, spawn_func, raise_type): def _spawn_resolve(self, resolved_dists_dir, target): download_dir = os.path.join(resolved_dists_dir, target.id) - download_job = spawn_download_distributions( + download_job = get_pip().spawn_download_distributions( download_dir=download_dir, requirements=self._requirements, requirement_files=self._requirement_files, @@ -395,7 +395,7 @@ def _categorize_build_requests(self, build_requests, dist_root): def _spawn_wheel_build(self, built_wheels_dir, build_request): build_result = build_request.result(built_wheels_dir) - build_job = spawn_build_wheels( + build_job = get_pip().spawn_build_wheels( distributions=[build_request.source_path], wheel_dir=build_result.build_dir, cache=self._cache, @@ -420,7 +420,7 @@ def _categorize_install_requests(self, install_requests, installed_wheels_dir): def _spawn_install(self, installed_wheels_dir, install_request): install_result = install_request.result(installed_wheels_dir) - install_job = spawn_install_wheel( + install_job = get_pip().spawn_install_wheel( wheel=install_request.wheel_path, install_dir=install_result.build_chroot, compile=self._compile, @@ -430,7 +430,7 @@ def _spawn_install(self, installed_wheels_dir, install_request): ) return SpawnedJob.wait(job=install_job, result=install_result) - def resolve_distributions(self): + def resolve_distributions(self, ignore_errors=False): # This method has four stages: # 1. Resolve sdists and wheels. # 2. Build local projects and sdists. @@ -560,8 +560,38 @@ def add_requirements_requests(install_result): distribution=distribution ) ) + + if not ignore_errors and self._transitive: + self._check_resolve(resolved_distributions) return resolved_distributions + def _check_resolve(self, resolved_distributions): + dist_by_key = OrderedDict( + (resolved_distribution.requirement.key, resolved_distribution.distribution) + for resolved_distribution in resolved_distributions + ) + + unsatisfied = [] + for dist in dist_by_key.values(): + for requirement in dist.requires(): + resolved_dist = dist_by_key.get(requirement.key) + if not resolved_dist or resolved_dist not in requirement: + unsatisfied.append( + '{dist} requires {requirement} but {resolved_dist} was resolved'.format( + dist=dist.as_requirement(), + requirement=requirement, + resolved_dist=resolved_dist.as_requirement() if resolved_dist else None + ) + ) + + if unsatisfied: + raise Unsatisfiable( + 'Failed to resolve compatible distributions:\n{failures}'.format( + failures='\n'.join('{index}: {failure}'.format(index=index + 1, failure=failure) + for index, failure in enumerate(unsatisfied)) + ) + ) + def resolve(requirements=None, requirement_files=None, @@ -576,7 +606,8 @@ def resolve(requirements=None, build=True, use_wheel=True, compile=False, - max_parallel_jobs=None): + max_parallel_jobs=None, + ignore_errors=False): """Produce all distributions needed to meet all specified requirements. :keyword requirements: A sequence of requirement strings. @@ -612,6 +643,7 @@ def resolve(requirements=None, Defaults to ``False``. :keyword int max_parallel_jobs: The maximum number of parallel jobs to use when resolving, building and installing distributions in a resolve. Defaults to the number of CPUs available. + :keyword bool ignore_errors: Whether to ignore resolution solver errors. Defaults to ``False``. :returns: List of :class:`ResolvedDistribution` instances meeting ``requirements``. :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. :raises Untranslateable: If no compatible distributions could be acquired for @@ -634,7 +666,7 @@ def resolve(requirements=None, compile=compile, max_parallel_jobs=max_parallel_jobs) - return list(resolve_request.resolve_distributions()) + return list(resolve_request.resolve_distributions(ignore_errors=ignore_errors)) def resolve_multi(requirements=None, @@ -650,7 +682,8 @@ def resolve_multi(requirements=None, build=True, use_wheel=True, compile=False, - max_parallel_jobs=None): + max_parallel_jobs=None, + ignore_errors=False): """A generator function that produces all distributions needed to meet `requirements` for multiple interpreters and/or platforms. @@ -689,6 +722,7 @@ def resolve_multi(requirements=None, Defaults to ``False``. :keyword int max_parallel_jobs: The maximum number of parallel jobs to use when resolving, building and installing distributions in a resolve. Defaults to the number of CPUs available. + :keyword bool ignore_errors: Whether to ignore resolution solver errors. Defaults to ``False``. :returns: List of :class:`ResolvedDistribution` instances meeting ``requirements``. :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. :raises Untranslateable: If no compatible distributions could be acquired for @@ -730,4 +764,4 @@ def iter_targets(): compile=compile, max_parallel_jobs=max_parallel_jobs) - return list(resolve_request.resolve_distributions()) + return list(resolve_request.resolve_distributions(ignore_errors=ignore_errors)) diff --git a/pex/testing.py b/pex/testing.py index 8b4414b14..da83c64ed 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -21,7 +21,7 @@ from pex.interpreter import PythonInterpreter from pex.pex import PEX from pex.pex_builder import PEXBuilder -from pex.pip import spawn_build_wheels, spawn_install_wheel +from pex.pip import get_pip from pex.third_party.pkg_resources import Distribution from pex.util import DistributionHelper, named_temporary_file @@ -161,7 +161,7 @@ def __init__(self, source_dir, interpreter=None, wheel_dir=None): self._interpreter = interpreter or PythonInterpreter.get() def bdist(self): - spawn_build_wheels( + get_pip().spawn_build_wheels( distributions=[self._source_dir], wheel_dir=self._wheel_dir, interpreter=self._interpreter @@ -211,7 +211,7 @@ def make_bdist(name='my_project', version='0.0.0', zip_safe=True, interpreter=No **kwargs) as dist_location: install_dir = os.path.join(safe_mkdtemp(), os.path.basename(dist_location)) - spawn_install_wheel( + get_pip().spawn_install_wheel( wheel=dist_location, install_dir=install_dir, target=DistributionTarget.for_interpreter(interpreter) diff --git a/pex/variables.py b/pex/variables.py index fe14b6e98..821507987 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -106,6 +106,11 @@ def _get_int(self, variable, default=None): except KeyError: return self._defaulted(default) + def strip(self): + stripped_environ = {k: v + for k, v in self.copy().items() if not k.startswith(('PEX_', '__PEX_'))} + return Variables(environ=stripped_environ) + def strip_defaults(self): """Returns a copy of these variables but with defaults stripped. diff --git a/tests/test_finders.py b/tests/test_finders.py index 376c9187f..0beebefb5 100644 --- a/tests/test_finders.py +++ b/tests/test_finders.py @@ -6,7 +6,7 @@ import pytest from pex.finders import get_entry_point_from_console_script, get_script_from_distributions -from pex.pip import spawn_install_wheel +from pex.pip import get_pip from pex.util import DistributionHelper @@ -16,7 +16,7 @@ def test_get_script_from_distributions(tmpdir): whl_path = './tests/example_packages/aws_cfn_bootstrap-1.4-py2-none-any.whl' install_dir = os.path.join(str(tmpdir), os.path.basename(whl_path)) - spawn_install_wheel(wheel=whl_path, install_dir=install_dir).wait() + get_pip().spawn_install_wheel(wheel=whl_path, install_dir=install_dir).wait() dist = DistributionHelper.distribution_from_path(install_dir) assert 'aws-cfn-bootstrap' == dist.project_name diff --git a/tests/test_integration.py b/tests/test_integration.py index 8c36ce8b8..5789ccd70 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -17,7 +17,7 @@ from pex.common import safe_copy, safe_open, safe_sleep, temporary_dir from pex.compatibility import WINDOWS, nested, to_bytes from pex.pex_info import PexInfo -from pex.pip import spawn_build_wheels, spawn_download_distributions +from pex.pip import get_pip from pex.testing import ( NOT_CPYTHON27, NOT_CPYTHON27_OR_OSX, @@ -1341,12 +1341,12 @@ def test_issues_539_abi3_resolution(): # sdist. Since we want to test in --no-build, we pre-resolve/build the pycparser wheel here and # add the resulting wheelhouse to the --no-build pex command. download_dir = os.path.join(td, '.downloads') - spawn_download_distributions( + get_pip().spawn_download_distributions( download_dir=download_dir, requirements=['pycparser'] ).wait() wheel_dir = os.path.join(td, '.wheels') - spawn_build_wheels( + get_pip().spawn_build_wheels( wheel_dir=wheel_dir, distributions=glob.glob(os.path.join(download_dir, '*')) ).wait() diff --git a/tests/test_unified_install_cache.py b/tests/test_unified_install_cache.py index 0058aadbb..07582bbda 100644 --- a/tests/test_unified_install_cache.py +++ b/tests/test_unified_install_cache.py @@ -9,9 +9,9 @@ import pytest -from pex import pip from pex.common import safe_mkdir, safe_mkdtemp, safe_open, safe_rmtree from pex.pex_info import PexInfo +from pex.pip import get_pip from pex.testing import run_pex_command @@ -31,7 +31,7 @@ def test_issues_789_demo(pex_project_dir): ] wheelhouse = os.path.join(tmpdir, 'wheelhouse') - pip.spawn_download_distributions( + get_pip().spawn_download_distributions( download_dir=wheelhouse, requirements=requirements ).wait()