diff --git a/contrib/python/src/python/pants/contrib/python/checks/tasks/checkstyle/checkstyle.py b/contrib/python/src/python/pants/contrib/python/checks/tasks/checkstyle/checkstyle.py index 631b971896d7..1444edbd71af 100644 --- a/contrib/python/src/python/pants/contrib/python/checks/tasks/checkstyle/checkstyle.py +++ b/contrib/python/src/python/pants/contrib/python/checks/tasks/checkstyle/checkstyle.py @@ -7,7 +7,7 @@ from packaging import version from pants.backend.python.interpreter_cache import PythonInterpreterCache from pants.backend.python.python_requirement import PythonRequirement -from pants.backend.python.subsystems.pex_build_util import PexBuilderWrapper +from pants.python.pex_build_util import PexBuilderWrapper from pants.backend.python.subsystems.python_setup import PythonSetup from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants.backend.python.targets.python_target import PythonTarget diff --git a/contrib/python/src/python/pants/contrib/python/checks/tasks/python_eval.py b/contrib/python/src/python/pants/contrib/python/checks/tasks/python_eval.py index 5308f2d42f2f..cd3b287eee25 100644 --- a/contrib/python/src/python/pants/contrib/python/checks/tasks/python_eval.py +++ b/contrib/python/src/python/pants/contrib/python/checks/tasks/python_eval.py @@ -6,9 +6,6 @@ import pkgutil from pants.backend.python.interpreter_cache import PythonInterpreterCache -from pants.backend.python.subsystems.pex_build_util import (PexBuilderWrapper, - has_python_requirements, - has_python_sources) from pants.backend.python.targets.python_binary import PythonBinary from pants.backend.python.targets.python_library import PythonLibrary from pants.backend.python.targets.python_target import PythonTarget @@ -17,6 +14,11 @@ from pants.base.exceptions import TaskError from pants.base.generator import Generator, TemplateData from pants.base.workunit import WorkUnit, WorkUnitLabel +from pants.python.pex_build_util import ( + PexBuilderWrapper, + has_python_requirements, + has_python_sources, +) from pants.task.lint_task_mixin import LintTaskMixin from pants.util.dirutil import safe_concurrent_creation, safe_mkdir from pants.util.memo import memoized_property diff --git a/contrib/python/tests/python/pants_test/contrib/python/checks/tasks/checkstyle/test_checkstyle.py b/contrib/python/tests/python/pants_test/contrib/python/checks/tasks/checkstyle/test_checkstyle.py index 2f474e344463..3c0a2a0b2f15 100644 --- a/contrib/python/tests/python/pants_test/contrib/python/checks/tasks/checkstyle/test_checkstyle.py +++ b/contrib/python/tests/python/pants_test/contrib/python/checks/tasks/checkstyle/test_checkstyle.py @@ -12,7 +12,7 @@ from pants.backend.python.targets.python_library import PythonLibrary from pants.base.build_environment import get_buildroot from pants.base.exceptions import TaskError -from pants.python.python_repos import PythonRepos +from pants.backend.python.subsystems.python_repos import PythonRepos from pants.util.contextutil import environment_as from pants.util.dirutil import safe_mkdtemp, safe_rmtree from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase diff --git a/contrib/python/tests/python/pants_test/contrib/python/checks/tasks/test_python_eval.py b/contrib/python/tests/python/pants_test/contrib/python/checks/tasks/test_python_eval.py index e77abab18014..97b62445aca6 100644 --- a/contrib/python/tests/python/pants_test/contrib/python/checks/tasks/test_python_eval.py +++ b/contrib/python/tests/python/pants_test/contrib/python/checks/tasks/test_python_eval.py @@ -4,7 +4,7 @@ from textwrap import dedent from pants.backend.python.subsystems.python_setup import PythonSetup -from pants.python.python_repos import PythonRepos +from pants.backend.python.subsystems.python_repos import PythonRepos from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase from pants.contrib.python.checks.tasks.python_eval import PythonEval diff --git a/examples/3rdparty/python/BUILD b/examples/3rdparty/python/BUILD index bc32f95ffda2..22b76c24b5e9 100644 --- a/examples/3rdparty/python/BUILD +++ b/examples/3rdparty/python/BUILD @@ -33,3 +33,8 @@ unpacked_whls( ], within_data_subdir='purelib/tensorflow', ) + +files( + name='examples_python_3rdparty', + sources=['**/*'], +) diff --git a/examples/src/python/example/tensorflow_custom_op/BUILD b/examples/src/python/example/tensorflow_custom_op/BUILD index 185ed20e2ace..fd26226af0f6 100644 --- a/examples/src/python/example/tensorflow_custom_op/BUILD +++ b/examples/src/python/example/tensorflow_custom_op/BUILD @@ -37,3 +37,21 @@ ctypes_compatible_cpp_library( ], ctypes_native_library=native_artifact(lib_name='tensorflow-zero-out-operator'), ) + + +python_binary( + name='show-tf-version', + source='show_tf_version.py', + dependencies=[ + 'examples/3rdparty/python:tensorflow', + ], + compatibility=['CPython>=3.6,<4'], +) + +files( + name='show-tf-version-files', + sources=['**/*'], + dependencies=[ + 'examples/3rdparty/python:examples_python_3rdparty', + ], +) diff --git a/examples/src/python/example/tensorflow_custom_op/show_tf_version.py b/examples/src/python/example/tensorflow_custom_op/show_tf_version.py new file mode 100644 index 000000000000..c6dc2686ff2d --- /dev/null +++ b/examples/src/python/example/tensorflow_custom_op/show_tf_version.py @@ -0,0 +1,6 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import tensorflow as tf + +print(f"tf version: {tf.__version__}") diff --git a/src/python/pants/backend/codegen/grpcio/python/grpcio_run.py b/src/python/pants/backend/codegen/grpcio/python/grpcio_run.py index 92959ca9244a..4300c1608330 100644 --- a/src/python/pants/backend/codegen/grpcio/python/grpcio_run.py +++ b/src/python/pants/backend/codegen/grpcio/python/grpcio_run.py @@ -7,7 +7,7 @@ from pants.backend.codegen.grpcio.python.grpcio_prep import GrpcioPrep from pants.backend.codegen.grpcio.python.python_grpcio_library import PythonGrpcioLibrary -from pants.backend.python.subsystems.pex_build_util import identify_missing_init_files +from pants.python.pex_build_util import identify_missing_init_files from pants.backend.python.targets.python_library import PythonLibrary from pants.base.build_environment import get_buildroot from pants.base.exceptions import TaskError diff --git a/src/python/pants/backend/project_info/tasks/export.py b/src/python/pants/backend/project_info/tasks/export.py index e09eacf6e2f0..c9e5c4141bf8 100644 --- a/src/python/pants/backend/project_info/tasks/export.py +++ b/src/python/pants/backend/project_info/tasks/export.py @@ -20,7 +20,6 @@ from pants.backend.jvm.tasks.ivy_task_mixin import IvyTaskMixin from pants.backend.project_info.tasks.export_version import DEFAULT_EXPORT_VERSION from pants.backend.python.interpreter_cache import PythonInterpreterCache -from pants.backend.python.subsystems.pex_build_util import has_python_requirements from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants.backend.python.targets.python_target import PythonTarget from pants.backend.python.targets.python_tests import PythonTests @@ -34,6 +33,7 @@ from pants.java.distribution.distribution import DistributionLocator from pants.java.executor import SubprocessExecutor from pants.java.jar.jar_dependency_utils import M2Coordinate +from pants.python.pex_build_util import has_python_requirements from pants.task.console_task import ConsoleTask from pants.util.memo import memoized_property diff --git a/src/python/pants/backend/python/register.py b/src/python/pants/backend/python/register.py index 51cad2985b97..71ad1686aa6d 100644 --- a/src/python/pants/backend/python/register.py +++ b/src/python/pants/backend/python/register.py @@ -45,10 +45,15 @@ from pants.build_graph.build_file_aliases import BuildFileAliases from pants.build_graph.resources import Resources from pants.goal.task_registrar import TaskRegistrar as task +from pants.python.pex_build_util import PexBuilderWrapper def global_subsystems(): - return python_native_code.PythonNativeCode, subprocess_environment.SubprocessEnvironment + return { + python_native_code.PythonNativeCode, + subprocess_environment.SubprocessEnvironment, + PexBuilderWrapper.Factory, + } def build_file_aliases(): diff --git a/src/python/pants/backend/python/rules/BUILD b/src/python/pants/backend/python/rules/BUILD index a708bc5c5160..37bc429bb09d 100644 --- a/src/python/pants/backend/python/rules/BUILD +++ b/src/python/pants/backend/python/rules/BUILD @@ -6,6 +6,7 @@ python_library( '3rdparty/python:dataclasses', '3rdparty/python:setuptools', 'src/python/pants/backend/python/subsystems', + 'src/python/pants/backend/python/subsystems/ipex', 'src/python/pants/build_graph', 'src/python/pants/engine/legacy:graph', 'src/python/pants/engine:build_files', diff --git a/src/python/pants/backend/python/rules/inject_init.py b/src/python/pants/backend/python/rules/inject_init.py index 1ea63be61f33..2c9dfdfeed0d 100644 --- a/src/python/pants/backend/python/rules/inject_init.py +++ b/src/python/pants/backend/python/rules/inject_init.py @@ -3,7 +3,7 @@ from dataclasses import dataclass -from pants.backend.python.subsystems.pex_build_util import identify_missing_init_files +from pants.python.pex_build_util import identify_missing_init_files from pants.engine.fs import EMPTY_DIRECTORY_DIGEST, Digest, Snapshot from pants.engine.isolated_process import ExecuteProcessRequest, ExecuteProcessResult from pants.engine.rules import rule diff --git a/src/python/pants/backend/python/subsystems/executable_pex_tool.py b/src/python/pants/backend/python/subsystems/executable_pex_tool.py index cca4fbc56ec0..c5cec8b22fde 100644 --- a/src/python/pants/backend/python/subsystems/executable_pex_tool.py +++ b/src/python/pants/backend/python/subsystems/executable_pex_tool.py @@ -7,7 +7,7 @@ from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo -from pants.backend.python.subsystems.pex_build_util import PexBuilderWrapper +from pants.python.pex_build_util import PexBuilderWrapper from pants.subsystem.subsystem import Subsystem from pants.util.dirutil import is_executable, safe_concurrent_creation diff --git a/src/python/pants/backend/python/subsystems/ipex/BUILD b/src/python/pants/backend/python/subsystems/ipex/BUILD new file mode 100644 index 000000000000..cebac0e31a1d --- /dev/null +++ b/src/python/pants/backend/python/subsystems/ipex/BUILD @@ -0,0 +1,7 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +# NB: This target is written into an .ipex file as the main script, and should not have any +# dependencies on another python code! .ipex files should always contain pex and setuptools +# requirements in order to run the main script! +python_library() diff --git a/src/python/pants/backend/python/subsystems/ipex/ipex_launcher.py b/src/python/pants/backend/python/subsystems/ipex/ipex_launcher.py new file mode 100644 index 000000000000..b6e316e35669 --- /dev/null +++ b/src/python/pants/backend/python/subsystems/ipex/ipex_launcher.py @@ -0,0 +1,131 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +"""Entrypoint script for a "dehydrated" .ipex file generated with --generate-ipex. + +This script will "hydrate" a normal .pex file in the same directory, then execute it. +""" + +import json +import os +import sys +import tempfile + +from pex import resolver +from pex.common import open_zip +from pex.fetcher import Fetcher, PyPIFetcher +from pex.interpreter import PythonInterpreter +from pex.pex_builder import PEXBuilder +from pex.pex_info import PexInfo +from pkg_resources import Requirement + + +APP_CODE_PREFIX = 'user_files/' + + +def _strip_app_code_prefix(path): + if not path.startswith(APP_CODE_PREFIX): + raise ValueError("Path {path} in IPEX-INFO did not begin with '{APP_CODE_PREFIX}'." + .format(path=path, APP_CODE_PREFIX=APP_CODE_PREFIX)) + return path[len(APP_CODE_PREFIX):] + + +def _log(message): + sys.stderr.write(message + '\n') + + +def _sanitize_requirements(requirements): + """ + Remove duplicate keys such as setuptools or pex which may be injected multiple times into the + resulting ipex when first executed. + """ + project_names = [] + new_requirements = {} + + for r in requirements: + r = Requirement(r) + if r.marker and not r.marker.evaluate(): + continue + if r.name not in new_requirements: + project_names.append(r.name) + new_requirements[r.name] = str(r) + sanitized_requirements = [new_requirements[n] for n in project_names] + + return sanitized_requirements + + +def modify_pex_info(pex_info, **kwargs): + new_info = json.loads(pex_info.dump()) + new_info.update(kwargs) + return PexInfo.from_json(json.dumps(new_info)) + + +def _hydrate_pex_file(self, hydrated_pex_file): + # We extract source files into a temporary directory before creating the pex. + td = tempfile.mkdtemp() + + with open_zip(self) as zf: + # Populate the pex with the pinned requirements and distribution names & hashes. + bootstrap_info = PexInfo.from_json(zf.read('BOOTSTRAP-PEX-INFO')) + bootstrap_builder = PEXBuilder(pex_info=bootstrap_info, interpreter=PythonInterpreter.get()) + + # Populate the pex with the needed code. + try: + ipex_info = json.loads(zf.read('IPEX-INFO').decode('utf-8')) + for path in ipex_info['code']: + unzipped_source = zf.extract(path, td) + bootstrap_builder.add_source(unzipped_source, env_filename=_strip_app_code_prefix(path)) + except Exception as e: + raise ValueError("Error: {e}. The IPEX-INFO for this .ipex file was:\n{info}" + .format(e=e, info=json.dumps(ipex_info, indent=4))) + + # Perform a fully pinned intransitive resolve to hydrate the install cache. + resolver_settings = ipex_info['resolver_settings'] + # TODO: Here we convert .indexes and .find_links into the old .fetchers until pants upgrades to + # pex 2.0. At that time, we can remove anything relating to fetchers from `resolver_settings`, and + # avoid removing the 'indexes' and 'find_links' keys, which are correct for pex 2.0. + fetchers = [PyPIFetcher(url) for url in resolver_settings.pop('indexes')] + fetchers.extend(Fetcher([url]) for url in resolver_settings.pop('find_links')) + resolver_settings['fetchers'] = fetchers + + sanitized_requirements = _sanitize_requirements(bootstrap_info.requirements) + bootstrap_info = modify_pex_info(bootstrap_info, requirements=sanitized_requirements) + bootstrap_builder.info = bootstrap_info + + resolved_distributions = resolver.resolve( + requirements=bootstrap_info.requirements, + cache=bootstrap_info.pex_root, + platform='current', + transitive=False, + interpreter=bootstrap_builder.interpreter, + **resolver_settings + ) + # TODO: this shouldn't be necessary, as we should be able to use the same 'distributions' from + # BOOTSTRAP-PEX-INFO. When the .ipex is executed, the normal pex bootstrap fails to see these + # requirements or recognize that they should be pulled from the cache for some reason. + for resolved_dist in resolved_distributions: + bootstrap_builder.add_distribution(resolved_dist.distribution) + + bootstrap_builder.build(hydrated_pex_file, bytecode_compile=False) + + +def main(self): + filename_base, ext = os.path.splitext(self) + + # If the ipex (this pex) is already named '.pex', ensure the output filename doesn't collide by + # inserting an intermediate '.ipex'! + if ext == '.pex': + hydrated_pex_file = '{filename_base}.ipex.pex'.format(filename_base=filename_base) + else: + hydrated_pex_file = '{filename_base}.pex'.format(filename_base=filename_base) + + if not os.path.exists(hydrated_pex_file): + _log('Hydrating {} to {}...'.format(self, hydrated_pex_file)) + _hydrate_pex_file(self, hydrated_pex_file) + + os.execv(sys.executable, [sys.executable, hydrated_pex_file] + sys.argv[1:]) + + +if __name__ == '__main__': + self = sys.argv[0] + main(self) diff --git a/src/python/pants/backend/python/subsystems/pex_build_util.py b/src/python/pants/backend/python/subsystems/pex_build_util.py deleted file mode 100644 index be387f906b01..000000000000 --- a/src/python/pants/backend/python/subsystems/pex_build_util.py +++ /dev/null @@ -1,373 +0,0 @@ -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -import logging -import os -from collections import defaultdict -from pathlib import Path -from typing import Callable, Sequence, Set - -from pex.fetcher import Fetcher -from pex.pex_builder import PEXBuilder -from pex.resolver import resolve -from pex.util import DistributionHelper -from twitter.common.collections import OrderedSet - -from pants.backend.python.python_requirement import PythonRequirement -from pants.backend.python.subsystems.python_setup import PythonSetup -from pants.backend.python.targets.python_binary import PythonBinary -from pants.backend.python.targets.python_distribution import PythonDistribution -from pants.backend.python.targets.python_library import PythonLibrary -from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary -from pants.backend.python.targets.python_tests import PythonTests -from pants.base.build_environment import get_buildroot -from pants.base.exceptions import TaskError -from pants.build_graph.files import Files -from pants.build_graph.target import Target -from pants.python.python_repos import PythonRepos -from pants.subsystem.subsystem import Subsystem -from pants.util.collections import assert_single_element -from pants.util.contextutil import temporary_file - - -def is_python_target(tgt: Target) -> bool: - # We'd like to take all PythonTarget subclasses, but currently PythonThriftLibrary and - # PythonAntlrLibrary extend PythonTarget, and until we fix that (which we can't do until - # we remove the old python pipeline entirely) we want to ignore those target types here. - return isinstance(tgt, (PythonLibrary, PythonTests, PythonBinary)) - - -def has_python_sources(tgt: Target) -> bool: - return is_python_target(tgt) and tgt.has_sources() - - -def is_local_python_dist(tgt: Target) -> bool: - return isinstance(tgt, PythonDistribution) - - -def has_resources(tgt: Target) -> bool: - return isinstance(tgt, Files) and tgt.has_sources() - - -def has_python_requirements(tgt: Target) -> bool: - return isinstance(tgt, PythonRequirementLibrary) - - -def always_uses_default_python_platform(tgt: Target) -> bool: - return isinstance(tgt, PythonTests) - - -def may_have_explicit_python_platform(tgt: Target) -> bool: - return isinstance(tgt, PythonBinary) - - -def targets_by_platform(targets, python_setup): - targets_requiring_default_platforms = [] - explicit_platform_settings = defaultdict(OrderedSet) - for target in targets: - if always_uses_default_python_platform(target): - targets_requiring_default_platforms.append(target) - elif may_have_explicit_python_platform(target): - for platform in target.platforms if target.platforms else python_setup.platforms: - explicit_platform_settings[platform].add(target) - # There are currently no tests for this because they're super platform specific and it's hard for - # us to express that on CI, but https://github.com/pantsbuild/pants/issues/7616 has an excellent - # repro case for why this is necessary. - for target in targets_requiring_default_platforms: - for platform in python_setup.platforms: - explicit_platform_settings[platform].add(target) - return dict(explicit_platform_settings) - - -def identify_missing_init_files(sources: Sequence[str]) -> Set[str]: - """Return the list of paths that would need to be added to ensure that every package has - an __init__.py. """ - packages: Set[str] = set() - for source in sources: - if source.endswith('.py'): - pkg_dir = os.path.dirname(source) - if pkg_dir and pkg_dir not in packages: - package = '' - for component in pkg_dir.split(os.sep): - package = os.path.join(package, component) - packages.add(package) - - return {os.path.join(package, '__init__.py') for package in packages} - set(sources) - - -def _create_source_dumper(builder: PEXBuilder, tgt: Target) -> Callable[[str], None]: - buildroot = get_buildroot() - - def get_chroot_path(relpath: str) -> str: - if type(tgt) == Files: - # Loose `Files`, as opposed to `Resources` or `PythonTarget`s, have no (implied) package - # structure and so we chroot them relative to the build root so that they can be accessed - # via the normal Python filesystem APIs just as they would be accessed outside the - # chrooted environment. NB: This requires we mark the pex as not zip safe so - # these `Files` can still be accessed in the context of a built pex distribution. - builder.info.zip_safe = False - return relpath - return str(Path(relpath).relative_to(tgt.target_base)) - - def dump_source(relpath: str) -> None: - source_path = str(Path(buildroot, relpath)) - dest_path = get_chroot_path(relpath) - if has_resources(tgt): - builder.add_resource(filename=source_path, env_filename=dest_path) - else: - builder.add_source(filename=source_path, env_filename=dest_path) - - return dump_source - - -class PexBuilderWrapper: - """Wraps PEXBuilder to provide an API that consumes targets and other BUILD file entities.""" - - class Factory(Subsystem): - options_scope = 'pex-builder-wrapper' - - @classmethod - def register_options(cls, register): - super(PexBuilderWrapper.Factory, cls).register_options(register) - register('--setuptools-version', advanced=True, default='40.6.3', - help='The setuptools version to include in the pex if namespace packages need to be ' - 'injected.') - - @classmethod - def subsystem_dependencies(cls): - return super(PexBuilderWrapper.Factory, cls).subsystem_dependencies() + ( - PythonRepos, - PythonSetup, - ) - - @classmethod - def create(cls, builder, log=None): - options = cls.global_instance().get_options() - setuptools_requirement = f'setuptools=={options.setuptools_version}' - - log = log or logging.getLogger(__name__) - - return PexBuilderWrapper(builder=builder, - python_repos_subsystem=PythonRepos.global_instance(), - python_setup_subsystem=PythonSetup.global_instance(), - setuptools_requirement=PythonRequirement(setuptools_requirement), - log=log) - - def __init__(self, - builder, - python_repos_subsystem, - python_setup_subsystem, - setuptools_requirement, - log): - assert isinstance(builder, PEXBuilder) - assert isinstance(python_repos_subsystem, PythonRepos) - assert isinstance(python_setup_subsystem, PythonSetup) - assert isinstance(setuptools_requirement, PythonRequirement) - assert log is not None - - self._builder = builder - self._python_repos_subsystem = python_repos_subsystem - self._python_setup_subsystem = python_setup_subsystem - self._setuptools_requirement = setuptools_requirement - self._log = log - - self._distributions = {} - self._frozen = False - - def add_requirement_libs_from(self, req_libs, platforms=None): - """Multi-platform dependency resolution for PEX files. - - :param builder: Dump the requirements into this builder. - :param interpreter: The :class:`PythonInterpreter` to resolve requirements for. - :param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve. - :param log: Use this logger. - :param platforms: A list of :class:`Platform`s to resolve requirements for. - Defaults to the platforms specified by PythonSetup. - """ - reqs = [req for req_lib in req_libs for req in req_lib.requirements] - self.add_resolved_requirements(reqs, platforms=platforms) - - class SingleDistExtractionError(Exception): pass - - def extract_single_dist_for_current_platform(self, reqs, dist_key): - """Resolve a specific distribution from a set of requirements matching the current platform. - - :param list reqs: A list of :class:`PythonRequirement` to resolve. - :param str dist_key: The value of `distribution.key` to match for a `distribution` from the - resolved requirements. - :return: The single :class:`pkg_resources.Distribution` matching `dist_key`. - :raises: :class:`self.SingleDistExtractionError` if no dists or multiple dists matched the given - `dist_key`. - """ - distributions = self._resolve_distributions_by_platform(reqs, platforms=['current']) - try: - matched_dist = assert_single_element(list( - dist - for _, dists in distributions.items() - for dist in dists - if dist.key == dist_key - )) - except (StopIteration, ValueError) as e: - raise self.SingleDistExtractionError( - f"Exactly one dist was expected to match name {dist_key} in requirements {reqs}: {e!r}" - ) - return matched_dist - - def _resolve_distributions_by_platform(self, reqs, platforms): - deduped_reqs = OrderedSet(reqs) - find_links = OrderedSet() - for req in deduped_reqs: - self._log.debug(f' Dumping requirement: {req}') - self._builder.add_requirement(str(req.requirement)) - if req.repository: - find_links.add(req.repository) - - # Resolve the requirements into distributions. - distributions = self._resolve_multi(self._builder.interpreter, deduped_reqs, platforms, - find_links) - return distributions - - def add_resolved_requirements(self, reqs, platforms=None): - """Multi-platform dependency resolution for PEX files. - - :param builder: Dump the requirements into this builder. - :param interpreter: The :class:`PythonInterpreter` to resolve requirements for. - :param reqs: A list of :class:`PythonRequirement` to resolve. - :param log: Use this logger. - :param platforms: A list of :class:`Platform`s to resolve requirements for. - Defaults to the platforms specified by PythonSetup. - """ - distributions = self._resolve_distributions_by_platform(reqs, platforms=platforms) - locations = set() - for platform, dists in distributions.items(): - for dist in dists: - if dist.location not in locations: - self._log.debug(f' Dumping distribution: .../{os.path.basename(dist.location)}') - self.add_distribution(dist) - locations.add(dist.location) - - def _resolve_multi(self, interpreter, requirements, platforms, find_links): - """Multi-platform dependency resolution for PEX files. - - Returns a list of distributions that must be included in order to satisfy a set of requirements. - That may involve distributions for multiple platforms. - - :param interpreter: The :class:`PythonInterpreter` to resolve for. - :param requirements: A list of :class:`PythonRequirement` objects to resolve. - :param platforms: A list of :class:`Platform`s to resolve for. - :param find_links: Additional paths to search for source packages during resolution. - :return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed - to satisfy the requirements on that platform. - """ - python_setup = self._python_setup_subsystem - python_repos = self._python_repos_subsystem - platforms = platforms or python_setup.platforms - find_links = find_links or [] - distributions = {} - fetchers = python_repos.get_fetchers() - fetchers.extend(Fetcher([path]) for path in find_links) - - for platform in platforms: - requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir, - str(interpreter.identity)) - resolved_dists = resolve( - requirements=[str(req.requirement) for req in requirements], - interpreter=interpreter, - fetchers=fetchers, - platform=platform, - context=python_repos.get_network_context(), - cache=requirements_cache_dir, - cache_ttl=python_setup.resolver_cache_ttl, - allow_prereleases=python_setup.resolver_allow_prereleases, - use_manylinux=python_setup.use_manylinux) - distributions[platform] = [resolved_dist.distribution for resolved_dist in resolved_dists] - - return distributions - - def add_sources_from(self, tgt: Target) -> None: - dump_source = _create_source_dumper(self._builder, tgt) - self._log.debug(f' Dumping sources: {tgt}') - for relpath in tgt.sources_relative_to_buildroot(): - try: - dump_source(relpath) - except OSError: - self._log.error(f'Failed to copy {relpath} for target {tgt.address.spec}') - raise - - if (getattr(tgt, '_resource_target_specs', None) or - getattr(tgt, '_synthetic_resources_target', None)): - # No one should be on old-style resources any more. And if they are, - # switching to the new python pipeline will be a great opportunity to fix that. - raise TaskError( - f'Old-style resources not supported for target {tgt.address.spec}. Depend on resources() ' - 'targets instead.' - ) - - def _prepare_inits(self) -> Set[str]: - chroot = self._builder.chroot() - sources = chroot.get('source') | chroot.get('resource') - missing_init_files = identify_missing_init_files(sources) - if missing_init_files: - with temporary_file(permissions=0o644) as ns_package: - ns_package.write(b'__import__("pkg_resources").declare_namespace(__name__) # type: ignore[attr-defined]') - ns_package.flush() - for missing_init_file in missing_init_files: - self._builder.add_source(filename=ns_package.name, env_filename=missing_init_file) - return missing_init_files - - def set_emit_warnings(self, emit_warnings): - self._builder.info.emit_warnings = emit_warnings - - def freeze(self) -> None: - if self._frozen: - return - if self._prepare_inits(): - dist = self._distributions.get('setuptools') - if not dist: - self.add_resolved_requirements([self._setuptools_requirement]) - self._builder.freeze(bytecode_compile=False) - self._frozen = True - - def set_entry_point(self, entry_point): - self._builder.set_entry_point(entry_point) - - def build(self, safe_path): - self.freeze() - self._builder.build(safe_path, bytecode_compile=False, deterministic_timestamp=True) - - def set_shebang(self, shebang): - self._builder.set_shebang(shebang) - - def add_interpreter_constraint(self, constraint): - self._builder.add_interpreter_constraint(constraint) - - def add_interpreter_constraints_from(self, constraint_tgts): - # TODO this would be a great place to validate the constraints and present a good error message - # if they are incompatible because all the sources of the constraints are available. - # See: https://github.com/pantsbuild/pex/blob/584b6e367939d24bc28aa9fa36eb911c8297dac8/pex/interpreter_constraints.py - constraint_tuples = { - self._python_setup_subsystem.compatibility_or_constraints(tgt.compatibility) - for tgt in constraint_tgts - } - for constraint_tuple in constraint_tuples: - for constraint in constraint_tuple: - self.add_interpreter_constraint(constraint) - - def add_direct_requirements(self, reqs): - for req in reqs: - self._builder.add_requirement(str(req)) - - def add_distribution(self, dist): - self._builder.add_distribution(dist) - self._register_distribution(dist) - - def add_dist_location(self, location): - self._builder.add_dist_location(location) - dist = DistributionHelper.distribution_from_path(location) - self._register_distribution(dist) - - def _register_distribution(self, dist): - self._distributions[dist.key] = dist - - def set_script(self, script): - self._builder.set_script(script) diff --git a/src/python/pants/backend/python/subsystems/pex_build_util_test.py b/src/python/pants/backend/python/subsystems/pex_build_util_test.py index 86bfa3a331dc..6f9dbd423aa3 100644 --- a/src/python/pants/backend/python/subsystems/pex_build_util_test.py +++ b/src/python/pants/backend/python/subsystems/pex_build_util_test.py @@ -8,7 +8,7 @@ from pex.pex_builder import PEXBuilder -from pants.backend.python.subsystems.pex_build_util import PexBuilderWrapper +from pants.python.pex_build_util import PexBuilderWrapper from pants.backend.python.targets.python_library import PythonLibrary from pants.source.source_root import SourceRootConfig from pants.testutil.subsystem.util import init_subsystem diff --git a/src/python/pants/backend/python/subsystems/python_native_code.py b/src/python/pants/backend/python/subsystems/python_native_code.py index bdddd8df53ef..d603409a6476 100644 --- a/src/python/pants/backend/python/subsystems/python_native_code.py +++ b/src/python/pants/backend/python/subsystems/python_native_code.py @@ -10,12 +10,12 @@ from pants.backend.native.subsystems.native_toolchain import NativeToolchain from pants.backend.native.targets.native_library import NativeLibrary from pants.backend.python.python_requirement import PythonRequirement -from pants.backend.python.subsystems import pex_build_util from pants.backend.python.subsystems.executable_pex_tool import ExecutablePexTool -from pants.backend.python.subsystems.python_setup import PythonSetup from pants.backend.python.targets.python_distribution import PythonDistribution from pants.base.exceptions import IncompatiblePlatformsError from pants.engine.rules import rule, subsystem_rule +from pants.python import pex_build_util +from pants.backend.python.subsystems.python_setup import PythonSetup from pants.subsystem.subsystem import Subsystem from pants.util.memo import memoized_property from pants.util.objects import SubclassesOf diff --git a/src/python/pants/backend/python/tasks/build_local_python_distributions.py b/src/python/pants/backend/python/tasks/build_local_python_distributions.py index fbfc0b3a4eb4..d1a55d6facf5 100644 --- a/src/python/pants/backend/python/tasks/build_local_python_distributions.py +++ b/src/python/pants/backend/python/tasks/build_local_python_distributions.py @@ -11,17 +11,14 @@ from pants.backend.native.targets.native_library import NativeLibrary from pants.backend.native.tasks.link_shared_libraries import SharedLibrary -from pants.backend.python.python_requirement import PythonRequirement -from pants.backend.python.subsystems.pex_build_util import is_local_python_dist -from pants.backend.python.subsystems.python_native_code import ( - BuildSetupRequiresPex, - PythonNativeCode, -) +from pants.backend.python.subsystems.python_native_code import BuildSetupRequiresPex, PythonNativeCode from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants.base.build_environment import get_buildroot from pants.base.exceptions import TargetDefinitionException, TaskError from pants.base.workunit import WorkUnitLabel from pants.build_graph.address import Address +from pants.python.pex_build_util import is_local_python_dist +from pants.backend.python.python_requirement import PythonRequirement from pants.task.task import Task from pants.util.collections import assert_single_element from pants.util.contextutil import pushd diff --git a/src/python/pants/backend/python/tasks/gather_sources.py b/src/python/pants/backend/python/tasks/gather_sources.py index 658405d4b706..1e7e4ff1972c 100644 --- a/src/python/pants/backend/python/tasks/gather_sources.py +++ b/src/python/pants/backend/python/tasks/gather_sources.py @@ -8,14 +8,14 @@ from pex.pex_builder import PEXBuilder from twitter.common.collections import OrderedSet -from pants.backend.python.subsystems.pex_build_util import ( - PexBuilderWrapper, - has_python_sources, - has_resources, - is_python_target, -) from pants.base.exceptions import TaskError from pants.invalidation.cache_manager import VersionedTargetSet +from pants.python.pex_build_util import ( + PexBuilderWrapper, + has_python_sources, + has_resources, + is_python_target, +) from pants.task.task import Task from pants.util.dirutil import safe_concurrent_creation diff --git a/src/python/pants/backend/python/tasks/local_python_distribution_artifact.py b/src/python/pants/backend/python/tasks/local_python_distribution_artifact.py index 4de1b1684ff8..a7a9629d75df 100644 --- a/src/python/pants/backend/python/tasks/local_python_distribution_artifact.py +++ b/src/python/pants/backend/python/tasks/local_python_distribution_artifact.py @@ -3,8 +3,8 @@ import os -from pants.backend.python.subsystems.pex_build_util import is_local_python_dist from pants.base.build_environment import get_buildroot +from pants.python.pex_build_util import is_local_python_dist from pants.task.task import Task from pants.util.dirutil import safe_mkdir from pants.util.fileutil import atomic_copy diff --git a/src/python/pants/backend/python/tasks/python_binary_create.py b/src/python/pants/backend/python/tasks/python_binary_create.py index 217259806c4f..ee1f8699c56a 100644 --- a/src/python/pants/backend/python/tasks/python_binary_create.py +++ b/src/python/pants/backend/python/tasks/python_binary_create.py @@ -2,24 +2,25 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os +from typing import cast from pex.interpreter import PythonInterpreter from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo -from pants.backend.python.subsystems.pex_build_util import ( - PexBuilderWrapper, - has_python_requirements, - has_python_sources, - has_resources, - is_python_target, -) from pants.backend.python.subsystems.python_native_code import PythonNativeCode from pants.backend.python.targets.python_binary import PythonBinary from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants.base.build_environment import get_buildroot from pants.base.exceptions import TaskError from pants.build_graph.target_scopes import Scopes +from pants.python.pex_build_util import ( + PexBuilderWrapper, + has_python_requirements, + has_python_sources, + has_resources, + is_python_target, +) from pants.task.task import Task from pants.util.contextutil import temporary_dir from pants.util.dirutil import safe_mkdir_for @@ -33,11 +34,34 @@ class PythonBinaryCreate(Task): @classmethod def register_options(cls, register): super().register_options(register) - register('--include-run-information', type=bool, default=False, - help="Include run information in the PEX's PEX-INFO for information like the timestamp the PEX was " - "created and the command line used to create it. This information may be helpful to you, but means " - "that the generated PEX will not be reproducible; that is, future runs of `./pants binary` will not " - "create the same byte-for-byte identical .pex files.") + register( + "--include-run-information", + type=bool, + default=False, + help="Include run information in the PEX's PEX-INFO for information like the timestamp the PEX was " + "created and the command line used to create it. This information may be helpful to you, but means " + "that the generated PEX will not be reproducible; that is, future runs of `./pants binary` will not " + "create the same byte-for-byte identical .pex files.", + ) + register( + "--generate-ipex", + type=bool, + default=False, + fingerprint=True, + help='Whether to generate a .ipex file, which will "hydrate" its dependencies when ' + "it is first executed, rather than at build time (the normal pex behavior). " + "This option can reduce the size of a shipped pex file by over 100x for common" + "deps such as tensorflow, but it does require access to the network when " + "first executed.", + ) + register( + "--output-file-extension", + type=str, + default=None, + fingerprint=True, + help="What extension to output the file with. This can be used to differentiate " + "ipex files from others.", + ) @classmethod def subsystem_dependencies(cls): @@ -52,11 +76,11 @@ def _python_native_code_settings(self): @classmethod def product_types(cls): - return ['pex_archives', 'deployable_archives'] + return ["pex_archives", "deployable_archives"] @classmethod def implementation_version(cls): - return super().implementation_version() + [('PythonBinaryCreate', 2)] + return super().implementation_version() + [("PythonBinaryCreate", 2)] @property def cache_target_dirs(self): @@ -66,7 +90,7 @@ def cache_target_dirs(self): def prepare(cls, options, round_manager): # See comment below for why we don't use the GatherSources.PYTHON_SOURCES product. round_manager.require_data(PythonInterpreter) - round_manager.optional_data('python') # For codegen. + round_manager.optional_data("python") # For codegen. round_manager.optional_product(PythonRequirementLibrary) # For local dists. @staticmethod @@ -77,6 +101,17 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._distdir = self.get_options().pants_distdir + @property + def _generate_ipex(self) -> bool: + return cast(bool, self.get_options().generate_ipex) + + def _get_output_pex_filename(self, target_name): + file_ext = self.get_options().output_file_extension + if file_ext is None: + file_ext = ".ipex" if self._generate_ipex else ".pex" + + return f"{target_name}{file_ext}" + def execute(self): binaries = self.context.targets(self.is_binary) @@ -85,31 +120,39 @@ def execute(self): for binary in binaries: name = binary.name if name in names: - raise TaskError(f'Cannot build two binaries with the same name in a single invocation. ' - '{binary} and {names[name]} both have the name {name}.') + raise TaskError( + f"Cannot build two binaries with the same name in a single invocation. " + "{binary} and {names[name]} both have the name {name}." + ) names[name] = binary with self.invalidated(binaries, invalidate_dependents=True) as invalidation_check: - python_deployable_archive = self.context.products.get('deployable_archives') - python_pex_product = self.context.products.get('pex_archives') + python_deployable_archive = self.context.products.get("deployable_archives") + python_pex_product = self.context.products.get("pex_archives") for vt in invalidation_check.all_vts: - pex_path = os.path.join(vt.results_dir, f'{vt.target.name}.pex') + pex_path = os.path.join( + vt.results_dir, self._get_output_pex_filename(vt.target.name) + ) if not vt.valid: - self.context.log.debug(f'cache for {vt.target} is invalid, rebuilding') + self.context.log.debug(f"cache for {vt.target} is invalid, rebuilding") self._create_binary(vt.target, vt.results_dir) else: - self.context.log.debug(f'using cache for {vt.target}') + self.context.log.debug(f"using cache for {vt.target}") basename = os.path.basename(pex_path) python_pex_product.add(vt.target, os.path.dirname(pex_path)).append(basename) python_deployable_archive.add(vt.target, os.path.dirname(pex_path)).append(basename) - self.context.log.debug('created {}'.format(os.path.relpath(pex_path, get_buildroot()))) + self.context.log.debug( + "created {}".format(os.path.relpath(pex_path, get_buildroot())) + ) # Create a copy for pex. pex_copy = os.path.join(self._distdir, os.path.basename(pex_path)) safe_mkdir_for(pex_copy) atomic_copy(pex_path, pex_copy) - self.context.log.info('created pex {}'.format(os.path.relpath(pex_copy, get_buildroot()))) + self.context.log.info( + "created pex {}".format(os.path.relpath(pex_copy, get_buildroot())) + ) def _create_binary(self, binary_tgt, results_dir): """Create a .pex file for the specified binary target.""" @@ -128,15 +171,22 @@ def _create_binary(self, binary_tgt, results_dir): pex_info.build_properties = build_properties pex_builder = PexBuilderWrapper.Factory.create( - builder=PEXBuilder(path=tmpdir, interpreter=interpreter, pex_info=pex_info, copy=True), - log=self.context.log) + builder=PEXBuilder( + path=tmpdir, interpreter=interpreter, pex_info=pex_info, copy=True + ), + log=self.context.log, + generate_ipex=self._generate_ipex, + ) if binary_tgt.shebang: - self.context.log.info('Found Python binary target {} with customized shebang, using it: {}' - .format(binary_tgt.name, binary_tgt.shebang)) + self.context.log.info( + "Found Python binary target {} with customized shebang, using it: {}".format( + binary_tgt.name, binary_tgt.shebang + ) + ) pex_builder.set_shebang(binary_tgt.shebang) else: - self.context.log.debug(f'No customized shebang found for {binary_tgt.name}') + self.context.log.debug(f"No customized shebang found for {binary_tgt.name}") # Find which targets provide sources and which specify requirements. source_tgts = [] @@ -162,10 +212,13 @@ def _create_binary(self, binary_tgt, results_dir): # We need to ensure that we are resolving for only the current platform if we are # including local python dist targets that have native extensions. - self._python_native_code_settings.check_build_for_current_platform_only(self.context.targets()) + self._python_native_code_settings.check_build_for_current_platform_only( + self.context.targets() + ) pex_builder.add_requirement_libs_from(req_tgts, platforms=binary_tgt.platforms) # Build the .pex file. - pex_path = os.path.join(results_dir, f'{binary_tgt.name}.pex') + pex_filename = self._get_output_pex_filename(binary_tgt.name) + pex_path = os.path.join(results_dir, pex_filename) pex_builder.build(pex_path) return pex_path diff --git a/src/python/pants/backend/python/tasks/python_tool_prep_base.py b/src/python/pants/backend/python/tasks/python_tool_prep_base.py index 1f05c280d857..40b3abbcd4f1 100644 --- a/src/python/pants/backend/python/tasks/python_tool_prep_base.py +++ b/src/python/pants/backend/python/tasks/python_tool_prep_base.py @@ -12,7 +12,7 @@ from pants.backend.python.interpreter_cache import PythonInterpreterCache from pants.backend.python.python_requirement import PythonRequirement -from pants.backend.python.subsystems.pex_build_util import PexBuilderWrapper +from pants.python.pex_build_util import PexBuilderWrapper from pants.backend.python.subsystems.python_tool_base import PythonToolBase from pants.base.build_environment import get_pants_cachedir from pants.base.exceptions import TaskError diff --git a/src/python/pants/backend/python/tasks/resolve_requirements.py b/src/python/pants/backend/python/tasks/resolve_requirements.py index 2e447a6f127a..fd78a73e6767 100644 --- a/src/python/pants/backend/python/tasks/resolve_requirements.py +++ b/src/python/pants/backend/python/tasks/resolve_requirements.py @@ -3,8 +3,8 @@ from pex.interpreter import PythonInterpreter -from pants.backend.python.subsystems.pex_build_util import has_python_requirements, is_python_target from pants.backend.python.tasks.resolve_requirements_task_base import ResolveRequirementsTaskBase +from pants.python.pex_build_util import has_python_requirements, is_python_target class ResolveRequirements(ResolveRequirementsTaskBase): diff --git a/src/python/pants/backend/python/tasks/resolve_requirements_task_base.py b/src/python/pants/backend/python/tasks/resolve_requirements_task_base.py index 376f2364d0ff..b7dc8c5bac42 100644 --- a/src/python/pants/backend/python/tasks/resolve_requirements_task_base.py +++ b/src/python/pants/backend/python/tasks/resolve_requirements_task_base.py @@ -9,7 +9,7 @@ from pex.pex_builder import PEXBuilder from pants.backend.python.python_requirement import PythonRequirement -from pants.backend.python.subsystems.pex_build_util import PexBuilderWrapper +from pants.python.pex_build_util import PexBuilderWrapper from pants.backend.python.subsystems.python_native_code import PythonNativeCode from pants.backend.python.subsystems.python_setup import PythonSetup from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary diff --git a/src/python/pants/backend/python/tasks/setup_py.py b/src/python/pants/backend/python/tasks/setup_py.py index 4630ee4b536f..fc54f848bca5 100644 --- a/src/python/pants/backend/python/tasks/setup_py.py +++ b/src/python/pants/backend/python/tasks/setup_py.py @@ -19,7 +19,6 @@ from twitter.common.dirutil.chroot import Chroot from pants.backend.python.rules.setup_py_util import distutils_repr -from pants.backend.python.subsystems.pex_build_util import is_local_python_dist from pants.backend.python.targets.python_binary import PythonBinary from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants.backend.python.targets.python_target import PythonTarget @@ -30,6 +29,7 @@ from pants.build_graph.address_lookup_error import AddressLookupError from pants.build_graph.build_graph import sort_targets from pants.build_graph.resources import Resources +from pants.python.pex_build_util import is_local_python_dist from pants.task.task import Task from pants.util.contextutil import temporary_file from pants.util.dirutil import safe_concurrent_creation, safe_rmtree, safe_walk diff --git a/src/python/pants/backend/python/tasks/unpack_wheels.py b/src/python/pants/backend/python/tasks/unpack_wheels.py index a85a7ec6d98c..451dfa93f902 100644 --- a/src/python/pants/backend/python/tasks/unpack_wheels.py +++ b/src/python/pants/backend/python/tasks/unpack_wheels.py @@ -7,12 +7,11 @@ from pex.pex_builder import PEXBuilder from pants.backend.python.interpreter_cache import PythonInterpreterCache -from pants.backend.python.subsystems.pex_build_util import PexBuilderWrapper +from pants.python.pex_build_util import PexBuilderWrapper from pants.backend.python.subsystems.python_setup import PythonSetup from pants.backend.python.targets.unpacked_whls import UnpackedWheels from pants.base.exceptions import TaskError from pants.base.fingerprint_strategy import DefaultFingerprintHashingMixin, FingerprintStrategy -from pants.fs.archive import ZIP from pants.task.unpack_remote_sources_base import UnpackRemoteSourcesBase from pants.util.contextutil import temporary_dir from pants.util.dirutil import mergetree, safe_concurrent_creation @@ -51,49 +50,55 @@ def subsystem_dependencies(cls): PythonSetup, ) - class _NativeCodeExtractionSetupFailure(Exception): pass - def _get_matching_wheel(self, pex_path, interpreter, requirements, module_name): - """Use PexBuilderWrapper to resolve a single wheel from the requirement specs using pex.""" - with self.context.new_workunit('extract-native-wheels'): + """Use PexBuilderWrapper to resolve a single wheel from the requirement specs using pex. + + N.B.: The resolved wheel is already "unpacked" by PEX. More accurately, it's installed in a + chroot. + """ + with self.context.new_workunit("extract-native-wheels"): with safe_concurrent_creation(pex_path) as chroot: pex_builder = PexBuilderWrapper.Factory.create( - builder=PEXBuilder(path=chroot, interpreter=interpreter), - log=self.context.log) - return pex_builder.extract_single_dist_for_current_platform(requirements, module_name) + builder=PEXBuilder(path=chroot, interpreter=interpreter), log=self.context.log + ) + + return pex_builder.extract_single_dist_for_current_platform( + requirements, dist_key=module_name + ) @memoized_method def _compatible_interpreter(self, unpacked_whls): - constraints = PythonSetup.global_instance().compatibility_or_constraints(unpacked_whls.compatibility) + constraints = PythonSetup.global_instance().compatibility_or_constraints( + unpacked_whls.compatibility + ) allowable_interpreters = PythonInterpreterCache.global_instance().setup(filters=constraints) return min(allowable_interpreters) - class WheelUnpackingError(TaskError): pass + class WheelUnpackingError(TaskError): + pass def unpack_target(self, unpacked_whls, unpack_dir): interpreter = self._compatible_interpreter(unpacked_whls) - with temporary_dir() as resolve_dir,\ - temporary_dir() as extract_dir: + with temporary_dir() as resolve_dir: try: - matched_dist = self._get_matching_wheel(resolve_dir, interpreter, - unpacked_whls.all_imported_requirements, - unpacked_whls.module_name) - ZIP.extract(matched_dist.location, extract_dir) + matched_dist = self._get_matching_wheel( + resolve_dir, + interpreter, + unpacked_whls.all_imported_requirements, + unpacked_whls.module_name, + ) + wheel_chroot = matched_dist.location if unpacked_whls.within_data_subdir: - data_dir_prefix = '{name}-{version}.data/{subdir}'.format( - name=matched_dist.project_name, - version=matched_dist.version, - subdir=unpacked_whls.within_data_subdir, - ) - dist_data_dir = os.path.join(extract_dir, data_dir_prefix) + # N.B.: Wheels with data dirs have the data installed under the top module. + dist_data_dir = os.path.join(wheel_chroot, unpacked_whls.module_name) else: - dist_data_dir = extract_dir + dist_data_dir = wheel_chroot + unpack_filter = self.get_unpack_filter(unpacked_whls) # Copy over the module's data files into `unpack_dir`. mergetree(dist_data_dir, unpack_dir, file_filter=unpack_filter) except Exception as e: raise self.WheelUnpackingError( - "Error extracting wheel for target {}: {}" - .format(unpacked_whls, str(e)), - e) + "Error extracting wheel for target {}: {}".format(unpacked_whls, str(e)), e + ) diff --git a/src/python/pants/init/global_subsystems.py b/src/python/pants/init/global_subsystems.py index 17b132ec86ac..916c1c334d12 100644 --- a/src/python/pants/init/global_subsystems.py +++ b/src/python/pants/init/global_subsystems.py @@ -5,7 +5,7 @@ from pants.goal.run_tracker import RunTracker from pants.init.repro import Reproducer from pants.process.subprocess import Subprocess -from pants.python.python_repos import PythonRepos +from pants.backend.python.subsystems.python_repos import PythonRepos from pants.reporting.reporting import Reporting from pants.scm.subsystems.changed import Changed from pants.source.source_root import SourceRootConfig diff --git a/src/python/pants/init/plugin_resolver.py b/src/python/pants/init/plugin_resolver.py index b49179d4d110..8e4fd6817ba2 100644 --- a/src/python/pants/init/plugin_resolver.py +++ b/src/python/pants/init/plugin_resolver.py @@ -13,7 +13,7 @@ from wheel.install import WheelFile from pants.option.global_options import GlobalOptionsRegistrar -from pants.python.python_repos import PythonRepos +from pants.backend.python.subsystems.python_repos import PythonRepos from pants.util.contextutil import temporary_dir from pants.util.dirutil import safe_delete, safe_mkdir, safe_open from pants.util.memo import memoized_property diff --git a/src/python/pants/option/custom_types.py b/src/python/pants/option/custom_types.py index 918911d3a963..99883d03943f 100644 --- a/src/python/pants/option/custom_types.py +++ b/src/python/pants/option/custom_types.py @@ -4,7 +4,7 @@ import os import re from enum import Enum -from typing import Dict, Iterable, List, Pattern, Sequence +from typing import Dict, Iterable, List, Optional, Pattern, Sequence, Type, Union from pants.base.deprecated import warn_or_error from pants.option.errors import ParseError @@ -25,6 +25,15 @@ def __init__(self) -> None: raise NotImplementedError('UnsetBool cannot be instantiated. It should only be used as a ' 'sentinel type.') + @classmethod + def coerce_bool(cls, value: Optional[Union[Type["UnsetBool"], bool]], default: bool) -> bool: + if value is None: + return default + if value is cls: + return default + assert isinstance(value, bool) + return value + def dict_option(s: str) -> "DictValueComponent": """An option of type 'dict'. diff --git a/src/python/pants/python/BUILD b/src/python/pants/python/BUILD index 819b41c9a2a9..29ee3849282f 100644 --- a/src/python/pants/python/BUILD +++ b/src/python/pants/python/BUILD @@ -5,6 +5,12 @@ python_library( dependencies=[ '3rdparty/python:pex', + '3rdparty/python:setuptools', + 'src/python/pants/base:build_environment', + 'src/python/pants/base:exceptions', + 'src/python/pants/backend/python/subsystems/ipex', + 'src/python/pants/build_graph', + 'src/python/pants/option', 'src/python/pants/subsystem', 'src/python/pants/util:memo', ] diff --git a/src/python/pants/python/pex_build_util.py b/src/python/pants/python/pex_build_util.py new file mode 100644 index 000000000000..3dec873e35f8 --- /dev/null +++ b/src/python/pants/python/pex_build_util.py @@ -0,0 +1,591 @@ +# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import json +import logging +import os +from collections import defaultdict +from pathlib import Path +from typing import Callable, Dict, List, Optional, Sequence, Set, Tuple + +from pex.fetcher import PyPIFetcher, Fetcher +from pex.interpreter import PythonInterpreter +from pex.pex_builder import PEXBuilder +from pex.pex_info import PexInfo +from pex.platforms import Platform +from pex.resolver import resolve +from pex.util import DistributionHelper +from pex.version import __version__ as pex_version +from pkg_resources import Distribution, get_provider +from twitter.common.collections import OrderedSet + +from pants.backend.python.subsystems.ipex import ipex_launcher +from pants.backend.python.targets.python_binary import PythonBinary +from pants.backend.python.targets.python_distribution import PythonDistribution +from pants.backend.python.targets.python_library import PythonLibrary +from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary +from pants.backend.python.targets.python_tests import PythonTests +from pants.base.build_environment import get_buildroot +from pants.base.exceptions import TaskError +from pants.build_graph.files import Files +from pants.build_graph.target import Target +from pants.option.custom_types import UnsetBool +from pants.backend.python.subsystems.python_repos import PythonRepos +from pants.backend.python.python_requirement import PythonRequirement +from pants.backend.python.subsystems.python_setup import PythonSetup +from pants.subsystem.subsystem import Subsystem +from pants.util.collections import assert_single_element +from pants.util.contextutil import temporary_file +from pants.util.strutil import module_dirname + + +def is_python_target(tgt: Target) -> bool: + # We'd like to take all PythonTarget subclasses, but currently PythonThriftLibrary and + # PythonAntlrLibrary extend PythonTarget, and until we fix that (which we can't do until + # we remove the old python pipeline entirely) we want to ignore those target types here. + return isinstance(tgt, (PythonLibrary, PythonTests, PythonBinary)) + + +def has_python_sources(tgt: Target) -> bool: + return is_python_target(tgt) and tgt.has_sources() + + +def is_local_python_dist(tgt: Target) -> bool: + return isinstance(tgt, PythonDistribution) + + +def has_resources(tgt: Target) -> bool: + return isinstance(tgt, Files) and tgt.has_sources() + + +def has_python_requirements(tgt: Target) -> bool: + return isinstance(tgt, PythonRequirementLibrary) + + +def always_uses_default_python_platform(tgt: Target) -> bool: + return isinstance(tgt, PythonTests) + + +def may_have_explicit_python_platform(tgt: Target) -> bool: + return isinstance(tgt, PythonBinary) + + +def targets_by_platform(targets, python_setup): + targets_requiring_default_platforms = [] + explicit_platform_settings = defaultdict(OrderedSet) + for target in targets: + if always_uses_default_python_platform(target): + targets_requiring_default_platforms.append(target) + elif may_have_explicit_python_platform(target): + for platform in target.platforms if target.platforms else python_setup.platforms: + explicit_platform_settings[platform].add(target) + # There are currently no tests for this because they're super platform specific and it's hard for + # us to express that on CI, but https://github.com/pantsbuild/pants/issues/7616 has an excellent + # repro case for why this is necessary. + for target in targets_requiring_default_platforms: + for platform in python_setup.platforms: + explicit_platform_settings[platform].add(target) + return dict(explicit_platform_settings) + + +def identify_missing_init_files(sources: Sequence[str]) -> Set[str]: + """Return the list of paths that would need to be added to ensure that every package has an + __init__.py.""" + packages: Set[str] = set() + for source in sources: + if source.endswith(".py"): + pkg_dir = os.path.dirname(source) + if pkg_dir and pkg_dir not in packages: + package = "" + for component in pkg_dir.split(os.sep): + package = os.path.join(package, component) + packages.add(package) + + return {os.path.join(package, "__init__.py") for package in packages} - set(sources) + + +class PexBuilderWrapper: + """Wraps PEXBuilder to provide an API that consumes targets and other BUILD file entities.""" + + class Factory(Subsystem): + options_scope = "pex-builder-wrapper" + + @classmethod + def register_options(cls, register): + super(PexBuilderWrapper.Factory, cls).register_options(register) + # TODO: make an analogy to cls.register_jvm_tool that can be overridden for python subsystems + # by a python_requirement_library() target, not just via pants.ini! + register( + "--setuptools-version", + advanced=True, + default="40.6.3", + fingerprint=True, + help="The setuptools version to include in the pex if namespace packages need " + "to be injected.", + ) + register( + "--pex-version", + advanced=True, + default=pex_version, + fingerprint=True, + help="The pex version to include in any generated ipex files. " + "NOTE: This should ideally be the same as the pex version which pants " + f"itself depends on, which right now is {pex_version}.", + ) + + @classmethod + def subsystem_dependencies(cls): + return super(PexBuilderWrapper.Factory, cls).subsystem_dependencies() + ( + PythonRepos, + PythonSetup, + ) + + @classmethod + def create(cls, builder, log=None, generate_ipex=False): + options = cls.global_instance().get_options() + setuptools_requirement = f"setuptools=={options.setuptools_version}" + pex_requirement = f"pex=={options.pex_version}" + + log = log or logging.getLogger(__name__) + + return PexBuilderWrapper( + builder=builder, + python_repos_subsystem=PythonRepos.global_instance(), + python_setup_subsystem=PythonSetup.global_instance(), + setuptools_requirement=PythonRequirement(setuptools_requirement), + pex_requirement=PythonRequirement(pex_requirement), + log=log, + generate_ipex=generate_ipex, + ) + + def __init__( + self, + builder: PEXBuilder, + python_repos_subsystem: PythonRepos, + python_setup_subsystem: PythonSetup, + setuptools_requirement: PythonRequirement, + pex_requirement: PythonRequirement, + log, + generate_ipex: bool = False, + ): + assert log is not None + + self._builder = builder + self._python_repos_subsystem = python_repos_subsystem + self._python_setup_subsystem = python_setup_subsystem + self._setuptools_requirement = setuptools_requirement + self._pex_requirement = pex_requirement + self._log = log + + self._distributions: Dict[str, Distribution] = {} + self._frozen = False + + self._generate_ipex = generate_ipex + # If we generate a .ipex, we need to ensure all the code we copy into the underlying PEXBuilder + # is also added to the new PEXBuilder created in `._shuffle_original_build_info_into_ipex()`. + self._all_added_sources_resources: List[Path] = [] + # If we generate a dehydrated "ipex" file, we need to make sure that it is aware of any special + # find_links repos attached to any single requirement, so it can later resolve those + # requirements when it is first bootstrapped, using the same resolve options. + self._all_find_links = OrderedSet() + + def add_requirement_libs_from(self, req_libs, platforms=None): + """Multi-platform dependency resolution for PEX files. + + :param builder: Dump the requirements into this builder. + :param interpreter: The :class:`PythonInterpreter` to resolve requirements for. + :param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve. + :param log: Use this logger. + :param platforms: A list of :class:`Platform`s to resolve requirements for. + Defaults to the platforms specified by PythonSetup. + """ + reqs = [req for req_lib in req_libs for req in req_lib.requirements] + self.add_resolved_requirements(reqs, platforms=platforms) + + class SingleDistExtractionError(Exception): + pass + + def extract_single_dist_for_current_platform(self, reqs, dist_key) -> Distribution: + """Resolve a specific distribution from a set of requirements matching the current platform. + + :param list reqs: A list of :class:`PythonRequirement` to resolve. + :param str dist_key: The value of `distribution.key` to match for a `distribution` from the + resolved requirements. + :return: The single :class:`pkg_resources.Distribution` matching `dist_key`. + :raises: :class:`self.SingleDistExtractionError` if no dists or multiple dists matched the + given `dist_key`. + """ + distributions, _transitive_requirements = self.resolve_distributions( + reqs, platforms=["current"] + ) + try: + matched_dist = assert_single_element( + list( + dist + for _, dists in distributions.items() + for dist in dists + if dist.key == dist_key + ) + ) + except (StopIteration, ValueError) as e: + raise self.SingleDistExtractionError( + f"Exactly one dist was expected to match name {dist_key} in requirements {reqs}: {e!r}" + ) + return matched_dist + + def resolve_distributions( + self, reqs: List[PythonRequirement], platforms: Optional[List[Platform]] = None, + ) -> Tuple[Dict[str, List[Distribution]], List[PythonRequirement]]: + """Multi-platform dependency resolution. + + :param reqs: A list of :class:`PythonRequirement` to resolve. + :param platforms: A list of platform strings to resolve requirements for. + Defaults to the platforms specified by PythonSetup. + :returns: A tuple `(map, transitive_reqs)`, where `map` is a dict mapping distribution name + to a list of resolved distributions, and `reqs` contains all transitive == + requirements + needed to resolve the initial given requirements `reqs` for the given platforms. + """ + deduped_reqs = OrderedSet(reqs) + find_links = OrderedSet() + for req in deduped_reqs: + self._log.debug(f" Dumping requirement: {req}") + self._builder.add_requirement(str(req.requirement)) + if req.repository: + find_links.add(req.repository) + + # Resolve the requirements into distributions. + distributions, transitive_requirements = self._resolve_multi( + self._builder.interpreter, list(deduped_reqs), platforms, list(find_links), + ) + return (distributions, transitive_requirements) + + def add_resolved_requirements( + self, + reqs: List[PythonRequirement], + platforms: Optional[List[Platform]] = None, + override_ipex_build_do_actually_add_distribution: bool = False, + ) -> None: + """Multi-platform dependency resolution for PEX files. + + :param builder: Dump the requirements into this builder. + :param interpreter: The :class:`PythonInterpreter` to resolve requirements for. + :param reqs: A list of :class:`PythonRequirement` to resolve. + :param log: Use this logger. + :param platforms: A list of :class:`Platform`s to resolve requirements for. + Defaults to the platforms specified by PythonSetup. + :param bool override_ipex_build_do_actually_add_distribution: When this PexBuilderWrapper is configured with + generate_ipex=True, this method won't add any distributions to + the output pex. The internal implementation of this class adds a + pex dependency to the output ipex file, and therefore needs to + override the default behavior of this method. + """ + distributions, transitive_requirements = self.resolve_distributions( + reqs, platforms=platforms + ) + locations: Set[str] = set() + for platform, dists in distributions.items(): + for dist in dists: + if dist.location not in locations: + if self._generate_ipex and not override_ipex_build_do_actually_add_distribution: + self._log.debug( + f" *AVOIDING* dumping distribution into ipex: .../{os.path.basename(dist.location)}" + ) + else: + self._log.debug( + f" Dumping distribution: .../{os.path.basename(dist.location)}" + ) + self.add_distribution(dist) + locations.add(dist.location) + # In addition to the top-level requirements, we add all the requirements matching the resolved + # distributions to the resulting pex. If `generate_ipex=True` is set, we need to have all the + # transitive requirements resolved in order to hydrate the .ipex with an intransitive resolve. + if self._generate_ipex and not override_ipex_build_do_actually_add_distribution: + self.add_direct_requirements(transitive_requirements) + + def _resolve_multi( + self, + interpreter: PythonInterpreter, + requirements: List[PythonRequirement], + platforms: Optional[List[Platform]], + find_links: Optional[List[str]], + ) -> Tuple[Dict[str, List[Distribution]], List[PythonRequirement]]: + """Multi-platform dependency resolution for PEX files. + + Returns a tuple containing a list of distributions that must be included in order to satisfy a + set of requirements, and the transitive == requirements for thosee distributions. This may + involve distributions for multiple platforms. + + :param interpreter: The :class:`PythonInterpreter` to resolve for. + :param requirements: A list of :class:`PythonRequirement` objects to resolve. + :param platforms: A list of :class:`Platform`s to resolve for. + :param find_links: Additional paths to search for source packages during resolution. + :return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed + to satisfy the requirements on that platform. + """ + python_setup = self._python_setup_subsystem + python_repos = self._python_repos_subsystem + platforms = platforms or python_setup.platforms + + find_links = list(find_links) if find_links else [] + find_links.extend(python_repos.repos) + + # Individual requirements from pants may have a `repository` link attached to them, which is + # extracted in `self.resolve_distributions()`. When generating a .ipex file with + # `generate_ipex=True`, we want to ensure these repos are known to the ipex launcher when it + # tries to resolve all the requirements from BOOTSTRAP-PEX-INFO. + self._all_find_links.update(OrderedSet(find_links)) + + distributions: Dict[str, List[Distribution]] = defaultdict(list) + transitive_requirements: List[PythonRequirement] = [] + + fetchers = [ + PyPIFetcher(index_url) for index_url in python_repos.indexes + ] + [ + Fetcher([find_links_url]) for find_links_url in find_links + ] + + for platform in platforms: + requirements_cache_dir = os.path.join( + python_setup.resolver_cache_dir, str(interpreter.identity) + ) + resolved_dists = resolve( + requirements=[str(req.requirement) for req in requirements], + interpreter=interpreter, + platform=platform, + fetchers=fetchers, + cache=requirements_cache_dir, + cache_ttl=python_setup.resolver_cache_ttl, + context=python_repos.get_network_context(), + allow_prereleases=UnsetBool.coerce_bool(python_setup.resolver_allow_prereleases, + default=True), + use_manylinux=UnsetBool.coerce_bool(python_setup.use_manylinux, + default=True), + ) + for resolved_dist in resolved_dists: + dist = resolved_dist.distribution + transitive_requirements.append(dist.as_requirement()) + distributions[platform].append(dist) + + return (distributions, transitive_requirements) + + def _create_source_dumper(self, tgt: Target) -> Callable[[str], None]: + buildroot = get_buildroot() + + def get_chroot_path(relpath: str) -> str: + if type(tgt) == Files: + # Loose `Files`, as opposed to `Resources` or `PythonTarget`s, have no (implied) package + # structure and so we chroot them relative to the build root so that they can be accessed + # via the normal Python filesystem APIs just as they would be accessed outside the + # chrooted environment. NB: This requires we mark the pex as not zip safe so + # these `Files` can still be accessed in the context of a built pex distribution. + self._builder.info.zip_safe = False + return relpath + return str(Path(relpath).relative_to(tgt.target_base)) + + def dump_source(relpath: str) -> None: + source_path = str(Path(buildroot, relpath)) + dest_path = get_chroot_path(relpath) + + self._all_added_sources_resources.append(Path(dest_path)) + if has_resources(tgt): + self._builder.add_resource(filename=source_path, env_filename=dest_path) + else: + self._builder.add_source(filename=source_path, env_filename=dest_path) + + return dump_source + + def add_sources_from(self, tgt: Target) -> None: + dump_source = self._create_source_dumper(tgt) + self._log.debug(f" Dumping sources: {tgt}") + for relpath in tgt.sources_relative_to_buildroot(): + try: + dump_source(relpath) + except OSError: + self._log.error(f"Failed to copy {relpath} for target {tgt.address.spec}") + raise + + if getattr(tgt, "_resource_target_specs", None) or getattr( + tgt, "_synthetic_resources_target", None + ): + # No one should be on old-style resources any more. And if they are, + # switching to the new python pipeline will be a great opportunity to fix that. + raise TaskError( + f"Old-style resources not supported for target {tgt.address.spec}. Depend on resources() " + "targets instead." + ) + + def _prepare_inits(self) -> Set[str]: + chroot = self._builder.chroot() + sources = chroot.get("source") | chroot.get("resource") + missing_init_files = identify_missing_init_files(sources) + if missing_init_files: + with temporary_file(permissions=0o644) as ns_package: + ns_package.write( + b'__import__("pkg_resources").declare_namespace(__name__) # type: ignore[attr-defined]' + ) + ns_package.flush() + for missing_init_file in missing_init_files: + self._all_added_sources_resources.append(Path(missing_init_file)) + self._builder.add_source( + filename=ns_package.name, env_filename=missing_init_file + ) + return missing_init_files + + def set_emit_warnings(self, emit_warnings): + self._builder.info.emit_warnings = emit_warnings + + def _shuffle_underlying_pex_builder(self) -> Tuple[PexInfo, Path]: + """Replace the original builder with a new one, and just pull files from the old chroot.""" + # Ensure that (the interpreter selected to resolve requirements when the ipex is first run) is + # (the exact same interpreter we used to resolve those requirements here). This is the only (?) + # way to ensure that the ipex bootstrap uses the *exact* same interpreter version. + self._builder.info = ipex_launcher.modify_pex_info( + self._builder.info, + interpreter_constraints=[str(self._builder.interpreter.identity.requirement)], + ) + + orig_info = self._builder.info.copy() + + orig_chroot = self._builder.chroot() + + # Mutate the PexBuilder object which is manipulated by this subsystem. + self._builder = PEXBuilder(interpreter=self._builder.interpreter) + + return (orig_info, Path(orig_chroot.path())) + + def _shuffle_original_build_info_into_ipex(self): + """Create a "dehydrated" ipex file without any of its requirements, and specify that in two. + + *-INFO files. + + See ipex_launcher.py for details of how these files are used. + """ + orig_pex_info, orig_chroot = self._shuffle_underlying_pex_builder() + + # Gather information needed to create IPEX-INFO. + all_code = [str(src) for src in self._all_added_sources_resources] + prefixed_code_paths = [os.path.join(ipex_launcher.APP_CODE_PREFIX, src) for src in all_code] + for src, prefixed in zip(all_code, prefixed_code_paths): + # NB: Need to add under 'source' label for `self._prepare_inits()` to pick it up! + self._builder.chroot().copy( + os.path.join(str(orig_chroot), src), prefixed, label="source" + ) + + python_repos = self._python_repos_subsystem + python_setup = self._python_setup_subsystem + + # NB: self._all_find_links is updated on every call to self._resolve_multi(), and therefore + # includes all of the links from python_repos.repos, as well as any links added within any + # individual requirements from that resolve. + + resolver_settings = dict( + indexes=list(python_repos.indexes), + find_links=list(self._all_find_links), + allow_prereleases=UnsetBool.coerce_bool( + python_setup.resolver_allow_prereleases, default=True + ), + use_manylinux=python_setup.use_manylinux, + ) + + # IPEX-INFO: A json mapping interpreted in ipex_launcher.py: + # { + # "code": [], + # "resolver_settings": {}, + # } + ipex_info = dict(code=prefixed_code_paths, resolver_settings=resolver_settings,) + with temporary_file(permissions=0o644) as ipex_info_file: + ipex_info_file.write(json.dumps(ipex_info).encode()) + ipex_info_file.flush() + self._builder.add_resource(filename=ipex_info_file.name, env_filename="IPEX-INFO") + + # BOOTSTRAP-PEX-INFO: The original PEX-INFO, which should be the PEX-INFO in the hydrated .pex + # file that is generated when the .ipex is first executed. + with temporary_file(permissions=0o644) as bootstrap_pex_info_file: + bootstrap_pex_info_file.write(orig_pex_info.dump().encode()) + bootstrap_pex_info_file.flush() + self._builder.add_resource( + filename=bootstrap_pex_info_file.name, env_filename="BOOTSTRAP-PEX-INFO" + ) + + # ipex.py: The special bootstrap script to hydrate the .ipex with the fully resolved + # requirements when it is first executed. + # Extract the file contents of our custom app launcher script from the pants package. + parent_module = module_dirname(module_dirname(ipex_launcher.__name__)) + ipex_launcher_provider = get_provider(parent_module) + ipex_launcher_script = ipex_launcher_provider.get_resource_string( + parent_module, "ipex/ipex_launcher.py" + ) + with temporary_file(permissions=0o644) as ipex_launcher_file: + ipex_launcher_file.write(ipex_launcher_script) + ipex_launcher_file.flush() + # Our .ipex file will use our custom app launcher! + self._builder.set_executable(ipex_launcher_file.name, env_filename="ipex.py") + + # The PEX-INFO we generate shouldn't have any requirements (except pex itself), or they will + # fail to bootstrap because they were unable to find those distributions. Instead, the .pex file + # produced when the .ipex is first executed will read and resolve all those requirements from + # the BOOTSTRAP-PEX-INFO. + self.add_resolved_requirements( + [self._pex_requirement, self._setuptools_requirement,], + override_ipex_build_do_actually_add_distribution=True, + ) + + def freeze(self) -> None: + if self._frozen: + return + + if self._prepare_inits(): + dist = self._distributions.get("setuptools") + if not dist: + self.add_resolved_requirements([self._setuptools_requirement]) + + if self._generate_ipex: + self._shuffle_original_build_info_into_ipex() + + self._builder.freeze(bytecode_compile=False) + self._frozen = True + + def set_entry_point(self, entry_point): + self._builder.set_entry_point(entry_point) + + def build(self, safe_path): + self.freeze() + self._builder.build(safe_path, bytecode_compile=False, deterministic_timestamp=True) + + def set_shebang(self, shebang): + self._builder.set_shebang(shebang) + + def add_interpreter_constraint(self, constraint): + self._builder.add_interpreter_constraint(constraint) + + def add_interpreter_constraints_from(self, constraint_tgts): + # TODO this would be a great place to validate the constraints and present a good error message + # if they are incompatible because all the sources of the constraints are available. + # See: https://github.com/pantsbuild/pex/blob/584b6e367939d24bc28aa9fa36eb911c8297dac8/pex/interpreter_constraints.py + constraint_tuples = { + self._python_setup_subsystem.compatibility_or_constraints(tgt.compatibility) + for tgt in constraint_tgts + } + for constraint_tuple in constraint_tuples: + for constraint in constraint_tuple: + self.add_interpreter_constraint(constraint) + + def add_direct_requirements(self, reqs): + for req in reqs: + self._builder.add_requirement(str(req)) + + def add_distribution(self, dist): + self._builder.add_distribution(dist) + self._register_distribution(dist) + + def add_dist_location(self, location): + self._builder.add_dist_location(location) + dist = DistributionHelper.distribution_from_path(location) + self._register_distribution(dist) + + def _register_distribution(self, dist): + self._distributions[dist.key] = dist + + def set_script(self, script): + self._builder.set_script(script) diff --git a/src/python/pants/util/strutil.py b/src/python/pants/util/strutil.py index ee6c7d841e11..346e200b0369 100644 --- a/src/python/pants/util/strutil.py +++ b/src/python/pants/util/strutil.py @@ -89,6 +89,11 @@ def create_path_env_var( return delimiter.join(path_dirs) +def module_dirname(module_path: str) -> str: + """Return the import path for the parent module of `module_path`.""" + return ".".join(module_path.split(".")[:-1]) + + def camelcase(string: str) -> str: """Convert snake casing (containing - or _ characters) to camel casing.""" return ''.join(word.capitalize() for word in re.split('[-_]', string)) diff --git a/tests/python/pants_test/backend/codegen/thrift/python/test_apache_thrift_py_gen.py b/tests/python/pants_test/backend/codegen/thrift/python/test_apache_thrift_py_gen.py index b10ea61c9167..d0aab18cc29d 100644 --- a/tests/python/pants_test/backend/codegen/thrift/python/test_apache_thrift_py_gen.py +++ b/tests/python/pants_test/backend/codegen/thrift/python/test_apache_thrift_py_gen.py @@ -14,7 +14,7 @@ from pants.backend.python.interpreter_cache import PythonInterpreterCache from pants.backend.python.targets.python_library import PythonLibrary from pants.base.build_environment import get_buildroot -from pants.python.python_repos import PythonRepos +from pants.backend.python.subsystems.python_repos import PythonRepos from pants.testutil.subsystem.util import global_subsystem_instance from pants.testutil.task_test_base import TaskTestBase diff --git a/tests/python/pants_test/backend/python/tasks/BUILD b/tests/python/pants_test/backend/python/tasks/BUILD index 51337074314e..778c0f7f9acd 100644 --- a/tests/python/pants_test/backend/python/tasks/BUILD +++ b/tests/python/pants_test/backend/python/tasks/BUILD @@ -153,6 +153,7 @@ python_tests( sources = ['test_python_binary_integration.py'], dependencies = [ '3rdparty/python:pex', + 'examples/src/python/example/tensorflow_custom_op:show-tf-version-files', 'src/python/pants/util:contextutil', 'src/python/pants/testutil:int-test', 'testprojects/src/python:python_distribution_directory', diff --git a/tests/python/pants_test/backend/python/tasks/test_python_binary_create.py b/tests/python/pants_test/backend/python/tasks/test_python_binary_create.py index b983797d86c7..d58edec7a162 100644 --- a/tests/python/pants_test/backend/python/tasks/test_python_binary_create.py +++ b/tests/python/pants_test/backend/python/tasks/test_python_binary_create.py @@ -5,6 +5,8 @@ import subprocess from textwrap import dedent +from colors import blue + from pants.backend.python.tasks.gather_sources import GatherSources from pants.backend.python.tasks.python_binary_create import PythonBinaryCreate from pants.backend.python.tasks.select_interpreter import SelectInterpreter @@ -25,31 +27,41 @@ def alias_groups(cls): def _assert_pex(self, binary, expected_output=None, expected_shebang=None): # The easiest way to create products required by the PythonBinaryCreate task is to # execute the relevant tasks. - si_task_type = self.synthesize_task_subtype(SelectInterpreter, 'si_scope') - gs_task_type = self.synthesize_task_subtype(GatherSources, 'gs_scope') + si_task_type = self.synthesize_task_subtype(SelectInterpreter, "si_scope") + gs_task_type = self.synthesize_task_subtype(GatherSources, "gs_scope") - task_context = self.context(for_task_types=[si_task_type, gs_task_type], target_roots=[binary]) - run_info_dir = os.path.join(self.pants_workdir, self.options_scope, 'test/info') + task_context = self.context( + for_task_types=[si_task_type, gs_task_type], target_roots=[binary] + ) + run_info_dir = os.path.join(self.pants_workdir, self.options_scope, "test/info") task_context.run_tracker.run_info = RunInfo(run_info_dir) - si_task_type(task_context, os.path.join(self.pants_workdir, 'si')).execute() - gs_task_type(task_context, os.path.join(self.pants_workdir, 'gs')).execute() + si_task_type(task_context, os.path.join(self.pants_workdir, "si")).execute() + gs_task_type(task_context, os.path.join(self.pants_workdir, "gs")).execute() test_task = self.create_task(task_context) test_task.execute() - self._check_products(task_context, binary, expected_output=expected_output, expected_shebang=expected_shebang) - - def _check_products(self, context, binary, expected_output=None, expected_shebang=None): - pex_name = f'{binary.address.target_name}.pex' - products = context.products.get('deployable_archives') + self._check_products( + test_task, + task_context, + binary, + expected_output=expected_output, + expected_shebang=expected_shebang, + ) + + def _check_products( + self, test_task, context, binary, expected_output=None, expected_shebang=None + ): + pex_name = test_task._get_output_pex_filename(binary.address.target_name) + products = context.products.get("deployable_archives") self.assertIsNotNone(products) product_data = products.get(binary) product_basedir = list(product_data.keys())[0] self.assertEqual(product_data[product_basedir], [pex_name]) # Check pex copy. - pex_copy = os.path.join(self.build_root, 'dist', pex_name) + pex_copy = os.path.join(self.build_root, "dist", pex_name) self.assertTrue(os.path.isfile(pex_copy)) # Check that the pex runs. @@ -59,27 +71,43 @@ def _check_products(self, context, binary, expected_output=None, expected_sheban # Check that the pex has the expected shebang. if expected_shebang: - with open(pex_copy, 'rb') as pex: + with open(pex_copy, "rb") as pex: line = pex.readline() self.assertEqual(expected_shebang, line) def test_deployable_archive_products_simple(self): - self.create_python_library('src/python/lib', 'lib', {'lib.py': dedent(""" + self.create_python_library( + "src/python/lib", + "lib", + { + "lib.py": dedent( + """ import os def main(): os.getcwd() - """)}) - - binary = self.create_python_binary('src/python/bin', 'bin', 'lib.lib:main', - dependencies=['src/python/lib']) + """ + ) + }, + ) + + binary = self.create_python_binary( + "src/python/bin", "bin", "lib.lib:main", dependencies=["src/python/lib"] + ) self._assert_pex(binary) def test_deployable_archive_products_files_deps(self): - self.create_library('src/things', 'files', 'things', sources=['loose_file']) - self.create_file('src/things/loose_file', 'data!') - self.create_python_library('src/python/lib', 'lib', {'lib.py': dedent(""" + self.create_library( + path="src/things", target_type="files", name="things", sources=["loose_file"] + ) + self.create_file("src/things/loose_file", "data!") + self.create_python_library( + "src/python/lib", + "lib", + { + "lib.py": dedent( + """ import io import os import sys @@ -90,20 +118,71 @@ def main(): loose_file = os.path.join(here, '../src/things/loose_file') with io.open(os.path.realpath(loose_file), 'r') as fp: sys.stdout.write(fp.read()) - """)}) + """ + ) + }, + ) - binary = self.create_python_binary('src/python/bin', 'bin', 'lib.lib:main', - dependencies=['src/python/lib', 'src/things']) - self._assert_pex(binary, expected_output='data!') + binary = self.create_python_binary( + "src/python/bin", "bin", "lib.lib:main", dependencies=["src/python/lib", "src/things"] + ) + self._assert_pex(binary, expected_output="data!") def test_shebang_modified(self): - self.create_python_library('src/python/lib', 'lib', {'lib.py': dedent(""" + self.create_python_library( + "src/python/lib", + "lib", + { + "lib.py": dedent( + """ def main(): print('Hello World!') - """)}) - - binary = self.create_python_binary('src/python/bin', 'bin', 'lib.lib:main', - shebang='/usr/bin/env python2', - dependencies=['src/python/lib']) - - self._assert_pex(binary, expected_output='Hello World!\n', expected_shebang=b'#!/usr/bin/env python2\n') + """ + ) + }, + ) + + binary = self.create_python_binary( + "src/python/bin", + "bin", + "lib.lib:main", + shebang="/usr/bin/env python2", + dependencies=["src/python/lib"], + ) + + self._assert_pex( + binary, expected_output="Hello World!\n", expected_shebang=b"#!/usr/bin/env python2\n" + ) + + def test_generate_ipex_ansicolors(self): + self.create_python_requirement_library( + "3rdparty/ipex", "ansicolors", requirements=["ansicolors"] + ) + self.create_python_library( + "src/ipex", + "lib", + { + "main.py": dedent( + """\ + from colors import blue + + print(blue('i just lazy-loaded the ansicolors dependency!')) + """ + ) + }, + ) + binary = self.create_python_binary( + "src/ipex", "bin", "main", dependencies=["3rdparty/ipex:ansicolors", ":lib",] + ) + + self.set_options(generate_ipex=True) + dist_dir = os.path.join(self.build_root, "dist") + + self._assert_pex( + binary, expected_output=blue("i just lazy-loaded the ansicolors dependency!") + "\n" + ) + + dehydrated_ipex_file = os.path.join(dist_dir, "bin.ipex") + assert os.path.isfile(dehydrated_ipex_file) + hydrated_pex_output_file = os.path.join(dist_dir, "bin.pex") + assert os.path.isfile(hydrated_pex_output_file) diff --git a/tests/python/pants_test/backend/python/tasks/test_python_binary_integration.py b/tests/python/pants_test/backend/python/tasks/test_python_binary_integration.py index 07af51390c45..b744164ef477 100644 --- a/tests/python/pants_test/backend/python/tasks/test_python_binary_integration.py +++ b/tests/python/pants_test/backend/python/tasks/test_python_binary_integration.py @@ -2,13 +2,16 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import functools +import glob import os +import subprocess from contextlib import contextmanager from textwrap import dedent from pex.pex_info import PexInfo from pants.testutil.pants_run_integration_test import PantsRunIntegrationTest +from pants.util.collections import assert_single_element from pants.util.contextutil import open_zip, temporary_dir @@ -117,58 +120,65 @@ def test_target_platform_expands_config(self): ) def platforms_test_impl( - self, - target_platforms, - config_platforms, - want_present_platforms, - want_missing_platforms=(), + self, target_platforms, config_platforms, want_present_platforms, want_missing_platforms=(), ): - def numpy_deps(deps): - return [d for d in deps if 'numpy' in d] + def p537_deps(deps): + return [d for d in deps if "p537" in d] + def assertInAny(substring, collection): - self.assertTrue(any(substring in d for d in collection), - f'Expected an entry matching "{substring}" in {collection}') + self.assertTrue( + any(substring in d for d in collection), + f'Expected an entry matching "{substring}" in {collection}', + ) + def assertNotInAny(substring, collection): - self.assertTrue(all(substring not in d for d in collection), - f'Expected an entry matching "{substring}" in {collection}') + self.assertTrue( + all(substring not in d for d in collection), + f'Expected an entry matching "{substring}" in {collection}', + ) - test_project = 'testprojects/src/python/cache_fields' - test_build = os.path.join(test_project, 'BUILD') - test_src = os.path.join(test_project, 'main.py') - test_pex = 'dist/cache_fields.pex' + test_project = "testprojects/src/python/cache_fields" + test_build = os.path.join(test_project, "BUILD") + test_src = os.path.join(test_project, "main.py") + test_pex = "dist/cache_fields.pex" with self.caching_config() as config, self.mock_buildroot() as buildroot, buildroot.pushd(): - config['python-setup'] = { - 'platforms': None - } - - buildroot.write_file(test_src, '') - - buildroot.write_file(test_build, - dedent(""" - python_binary( - source='main.py', - dependencies=[':numpy'], - {target_platforms} - ) - python_requirement_library( - name='numpy', - requirements=[ - python_requirement('numpy==1.14.5') - ] - ) - - """.format( - target_platforms="platforms = [{}],".format(", ".join(["'{}'".format(p) for p in target_platforms])) if target_platforms is not None else "", - )) + config["python-setup"] = {"platforms": []} + + buildroot.write_file(test_src, "") + + buildroot.write_file( + test_build, + dedent( + """ + python_binary( + source='main.py', + dependencies=[':numpy'], + {target_platforms} + ) + python_requirement_library( + name='numpy', + requirements=[ + python_requirement('p537==1.0.4') + ] + ) + + """.format( + target_platforms="platforms = [{}],".format( + ", ".join(["'{}'".format(p) for p in target_platforms]) + ) + if target_platforms is not None + else "", + ) + ), ) # When only the linux platform is requested, # only linux wheels should end up in the pex. if config_platforms is not None: - config['python-setup']['platforms'] = config_platforms + config["python-setup"]["platforms"] = config_platforms result = self.run_pants_with_workdir( - command=['binary', test_project], - workdir=os.path.join(buildroot.new_buildroot, '.pants.d'), + command=["binary", test_project], + workdir=os.path.join(buildroot.new_buildroot, ".pants.d"), config=config, build_root=buildroot.new_buildroot, tee_output=True, @@ -176,24 +186,54 @@ def assertNotInAny(substring, collection): self.assert_success(result) with open_zip(test_pex) as z: - deps = numpy_deps(z.namelist()) + deps = p537_deps(z.namelist()) for platform in want_present_platforms: assertInAny(platform, deps) for platform in want_missing_platforms: assertNotInAny(platform, deps) def test_platforms_with_native_deps(self): - result = self.run_pants([ - 'binary', - 'testprojects/src/python/python_distribution/ctypes:bin', - 'testprojects/src/python/python_distribution/ctypes:with_platforms', - ]) + result = self.run_pants( + [ + "binary", + "testprojects/src/python/python_distribution/ctypes:bin", + "testprojects/src/python/python_distribution/ctypes:with_platforms", + ] + ) self.assert_failure(result) - self.assertIn(dedent("""\ - Pants doesn't currently support cross-compiling native code. - The following targets set platforms arguments other than ['current'], which is unsupported for this reason. - Please either remove the platforms argument from these targets, or set them to exactly ['current']. - Bad targets: - testprojects/src/python/python_distribution/ctypes:with_platforms - """), result.stderr_data) - self.assertNotIn('testprojects/src/python/python_distribution/ctypes:bin', result.stderr_data) + self.assertIn( + dedent( + """\ + Pants doesn't currently support cross-compiling native code. + The following targets set platforms arguments other than ['current'], which is unsupported for this reason. + Please either remove the platforms argument from these targets, or set them to exactly ['current']. + Bad targets: + testprojects/src/python/python_distribution/ctypes:with_platforms + """ + ), + result.stderr_data, + ) + self.assertNotIn( + "testprojects/src/python/python_distribution/ctypes:bin", result.stderr_data + ) + + def test_generate_ipex_tensorflow(self): + with temporary_dir() as tmp_distdir: + with self.pants_results( + [ + f"--pants-distdir={tmp_distdir}", + # tensorflow==1.14.0 has a setuptools>=41.0.0 requirement, so the .ipex resolve fails + # without this override. + f"--pex-builder-wrapper-setuptools-version=41.0.0", + "--binary-py-generate-ipex", + "binary", + "examples/src/python/example/tensorflow_custom_op:show-tf-version", + ] + ) as pants_run: + self.assert_success(pants_run) + output_ipex = assert_single_element(glob.glob(os.path.join(tmp_distdir, "*"))) + ipex_basename = os.path.basename(output_ipex) + self.assertEqual(ipex_basename, "show-tf-version.ipex") + + pex_execution_output = subprocess.check_output([output_ipex]) + assert "tf version: 1.14.0" in pex_execution_output.decode() diff --git a/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py b/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py index 3a34b24c44d0..02f46bb8719e 100644 --- a/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py +++ b/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py @@ -13,7 +13,7 @@ ) from pants.backend.python.tasks.resolve_requirements import ResolveRequirements from pants.backend.python.tasks.select_interpreter import SelectInterpreter -from pants.python.python_repos import PythonRepos +from pants.backend.python.subsystems.python_repos import PythonRepos from pants.testutil.task_test_base import DeclarativeTaskTestMixin from pants.util.collections import assert_single_element from pants.util.enums import match