diff --git a/CHANGES.md b/CHANGES.md index fe7fb412d..d92d80443 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,20 @@ # Release Notes +## 2.20.3 + +This release fixes both PEX building and lock creation via +`pex3 lock {create,sync}` to be reproducible in more cases. Previously, +if a requirement only available in source form (an sdist, a local +project or a VCS requirement) had a build that was not reproducible due +to either file timestamps (where the `SOURCE_DATE_EPOCH` standard was +respected) or random iteration order (e.g.: the `setup.py` used sets in +certain in-opportune ways), Pex's outputs would mirror the problematic +requirement's non-reproducibility. Now Pex plumbs a fixed +`SOURCE_DATE_EPOCH` and `PYTHONHASHSEED` to all places sources are +built. + +* Plumb reproducible build env vars more thoroughly. (#2554) + ## 2.20.2 This release fixes an old bug handling certain sdist zips under diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 89c82f58c..44259352e 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -1254,7 +1254,9 @@ def main(args=None): try: with global_environment(options) as env: try: - resolver_configuration = resolver_options.configure(options) + resolver_configuration = resolver_options.configure( + options, use_system_time=options.use_system_time + ) except resolver_options.InvalidConfigurationError as e: die(str(e)) diff --git a/pex/build_system/pep_517.py b/pex/build_system/pep_517.py index c2b213ca1..5d7ce6e82 100644 --- a/pex/build_system/pep_517.py +++ b/pex/build_system/pep_517.py @@ -75,6 +75,7 @@ def _default_build_system( resolved=resolved_dists, build_backend=DEFAULT_BUILD_BACKEND, backend_path=(), + use_system_time=resolver.use_system_time(), **extra_env ) ) diff --git a/pex/build_system/pep_518.py b/pex/build_system/pep_518.py index b5f24ef29..2b3e8e510 100644 --- a/pex/build_system/pep_518.py +++ b/pex/build_system/pep_518.py @@ -7,6 +7,7 @@ import subprocess from pex.build_system import DEFAULT_BUILD_BACKEND +from pex.common import REPRODUCIBLE_BUILDS_ENV from pex.dist_metadata import Distribution from pex.interpreter import PythonInterpreter from pex.pex import PEX @@ -80,6 +81,7 @@ def create( build_backend, # type: str backend_path, # type: Tuple[str, ...] extra_requirements=None, # type: Optional[Iterable[str]] + use_system_time=False, # type: bool **extra_env # type: str ): # type: (...) -> Union[BuildSystem, Error] @@ -87,6 +89,8 @@ def create( pex_builder.info.venv = True pex_builder.info.venv_site_packages_copies = True pex_builder.info.venv_bin_path = BinPath.PREPEND + # Allow REPRODUCIBLE_BUILDS_ENV PYTHONHASHSEED env var to take effect. + pex_builder.info.venv_hermetic_scripts = False for req in requires: pex_builder.add_requirement(req) for dist in resolved: @@ -144,6 +148,8 @@ def create( env.update(extra_env) if backend_path: env.update(PEX_EXTRA_SYS_PATH=os.pathsep.join(backend_path)) + if not use_system_time: + env.update(REPRODUCIBLE_BUILDS_ENV) return cls( venv_pex=venv_pex, build_backend=build_backend, requires=tuple(requires), env=env ) @@ -190,4 +196,5 @@ def load_build_system( build_backend=build_system_table.build_backend, backend_path=build_system_table.backend_path, extra_requirements=extra_requirements, + use_system_time=resolver.use_system_time(), ) diff --git a/pex/cache/dirs.py b/pex/cache/dirs.py index 6a11bc0b9..818c84c32 100644 --- a/pex/cache/dirs.py +++ b/pex/cache/dirs.py @@ -136,7 +136,7 @@ def iter_transitive_dependents(self): PIP = Value( "pip", - version=0, + version=1, name="Pip Versions", description="Isolated Pip caches and Pip PEXes Pex uses to resolve distributions.", ) diff --git a/pex/cli/commands/lock.py b/pex/cli/commands/lock.py index 2884d2cee..621d74457 100644 --- a/pex/cli/commands/lock.py +++ b/pex/cli/commands/lock.py @@ -799,7 +799,10 @@ def _resolve_targets( # type: (...) -> Union[Targets, Error] target_config = target_configuration or target_options.configure( - self.options, pip_configuration=resolver_options.create_pip_configuration(self.options) + self.options, + pip_configuration=resolver_options.create_pip_configuration( + self.options, use_system_time=False + ), ) if style is not LockStyle.UNIVERSAL: return target_config.resolve_targets() @@ -867,7 +870,9 @@ def _gather_requirements( def _create(self): # type: () -> Result - pip_configuration = resolver_options.create_pip_configuration(self.options) + pip_configuration = resolver_options.create_pip_configuration( + self.options, use_system_time=False + ) target_configuration = target_options.configure( self.options, pip_configuration=pip_configuration ) @@ -963,7 +968,9 @@ def _export(self, requirement_configuration=RequirementConfiguration()): ) lockfile_path, lock_file = self._load_lockfile() - pip_configuration = resolver_options.create_pip_configuration(self.options) + pip_configuration = resolver_options.create_pip_configuration( + self.options, use_system_time=False + ) targets = target_options.configure( self.options, pip_configuration=pip_configuration ).resolve_targets() @@ -1091,7 +1098,9 @@ def _create_lock_update_request( ): # type: (...) -> Union[LockUpdateRequest, Error] - pip_configuration = resolver_options.create_pip_configuration(self.options) + pip_configuration = resolver_options.create_pip_configuration( + self.options, use_system_time=False + ) lock_updater = LockUpdater.create( lock_file=lock_file, repos_configuration=pip_configuration.repos_configuration, @@ -1506,7 +1515,8 @@ def _sync(self): # type: () -> Result resolver_configuration = cast( - LockRepositoryConfiguration, resolver_options.configure(self.options) + LockRepositoryConfiguration, + resolver_options.configure(self.options, use_system_time=False), ) production_assert(isinstance(resolver_configuration, LockRepositoryConfiguration)) pip_configuration = resolver_configuration.pip_configuration diff --git a/pex/common.py b/pex/common.py index f3baa821d..51c6a4ec3 100644 --- a/pex/common.py +++ b/pex/common.py @@ -54,6 +54,15 @@ _UNIX_EPOCH = datetime(year=1970, month=1, day=1, hour=0, minute=0, second=0, tzinfo=None) DETERMINISTIC_DATETIME_TIMESTAMP = (DETERMINISTIC_DATETIME - _UNIX_EPOCH).total_seconds() +# N.B.: The `SOURCE_DATE_EPOCH` env var is semi-standard magic for controlling +# build tools. Wheel, for example, has supported this since 2016. +# See: +# + https://reproducible-builds.org/docs/source-date-epoch/ +# + https://github.com/pypa/wheel/blob/1b879e53fed1f179897ed47e55a68bc51df188db/wheel/archive.py#L36-L39 +REPRODUCIBLE_BUILDS_ENV = dict( + PYTHONHASHSEED="0", SOURCE_DATE_EPOCH=str(int(DETERMINISTIC_DATETIME_TIMESTAMP)) +) + def is_pyc_dir(dir_path): # type: (Text) -> bool diff --git a/pex/pip/installation.py b/pex/pip/installation.py index 79fe16a2d..8ca1dcda9 100644 --- a/pex/pip/installation.py +++ b/pex/pip/installation.py @@ -11,7 +11,7 @@ from pex import pex_warnings, third_party from pex.atomic_directory import atomic_directory from pex.cache.dirs import CacheDir -from pex.common import pluralize, safe_mkdtemp +from pex.common import REPRODUCIBLE_BUILDS_ENV, pluralize, safe_mkdtemp from pex.dist_metadata import Requirement from pex.interpreter import PythonInterpreter from pex.orderedset import OrderedSet @@ -42,6 +42,7 @@ def _pip_installation( iter_distribution_locations, # type: Callable[[], Iterator[str]] fingerprint, # type: str interpreter=None, # type: Optional[PythonInterpreter] + use_system_time=False, # type: bool ): # type: (...) -> Pip pip_root = CacheDir.PIP.path(str(version)) @@ -54,6 +55,8 @@ def _pip_installation( isolated_pip_builder = PEXBuilder(path=chroot.work_dir) isolated_pip_builder.info.venv = True + # Allow REPRODUCIBLE_BUILDS_ENV PYTHONHASHSEED env var to take effect if needed. + isolated_pip_builder.info.venv_hermetic_scripts = False for dist_location in iter_distribution_locations(): isolated_pip_builder.add_dist_location(dist=dist_location) with named_temporary_file(prefix="", suffix=".py", mode="w") as fp: @@ -78,7 +81,11 @@ def _pip_installation( isolated_pip_builder.freeze() pip_cache = os.path.join(pip_root, "pip_cache") pip_pex = ensure_venv(PEX(pip_pex_path, interpreter=pip_interpreter)) - pip_venv = PipVenv(venv_dir=pip_pex.venv_dir, execute_args=tuple(pip_pex.execute_args())) + pip_venv = PipVenv( + venv_dir=pip_pex.venv_dir, + execute_env=REPRODUCIBLE_BUILDS_ENV if not use_system_time else {}, + execute_args=tuple(pip_pex.execute_args()), + ) return Pip(pip=pip_venv, version=version, pip_cache=pip_cache) @@ -98,6 +105,7 @@ def _vendored_installation( interpreter=None, # type: Optional[PythonInterpreter] resolver=None, # type: Optional[Resolver] extra_requirements=(), # type: Tuple[Requirement, ...] + use_system_time=False, # type: bool ): # type: (...) -> Pip @@ -111,6 +119,7 @@ def expose_vendored(): iter_distribution_locations=expose_vendored, interpreter=interpreter, fingerprint=_fingerprint(extra_requirements), + use_system_time=use_system_time, ) if not resolver: @@ -171,6 +180,7 @@ def iter_distribution_locations(): iter_distribution_locations=iter_distribution_locations, interpreter=interpreter, fingerprint=_fingerprint(extra_requirements), + use_system_time=use_system_time, ) @@ -204,6 +214,7 @@ def _resolved_installation( resolver=None, # type: Optional[Resolver] interpreter=None, # type: Optional[PythonInterpreter] extra_requirements=(), # type: Tuple[Requirement, ...] + use_system_time=False, # type: bool ): # type: (...) -> Pip targets = Targets.from_target(LocalInterpreter.create(interpreter)) @@ -222,6 +233,7 @@ def _resolved_installation( iter_distribution_locations=_bootstrap_pip(version, interpreter=interpreter), interpreter=interpreter, fingerprint=_fingerprint(extra_requirements), + use_system_time=use_system_time, ) requirements_by_project_name = OrderedDict( @@ -269,6 +281,7 @@ def resolve_distribution_locations(): iter_distribution_locations=resolve_distribution_locations, interpreter=interpreter, fingerprint=_fingerprint(extra_requirements), + use_system_time=use_system_time, ) @@ -277,6 +290,7 @@ class PipInstallation(object): interpreter = attr.ib() # type: PythonInterpreter version = attr.ib() # type: PipVersionValue extra_requirements = attr.ib() # type: Tuple[Requirement, ...] + use_system_time = attr.ib() # type: bool def check_python_applies(self): # type: () -> None @@ -396,6 +410,7 @@ def get_pip( interpreter=interpreter or PythonInterpreter.get(), version=calculated_version, extra_requirements=extra_requirements, + use_system_time=resolver.use_system_time() if resolver else False, ) pip = _PIP.get(installation) if pip is None: @@ -405,6 +420,7 @@ def get_pip( interpreter=interpreter, resolver=resolver, extra_requirements=installation.extra_requirements, + use_system_time=installation.use_system_time, ) else: pip = _resolved_installation( @@ -412,6 +428,7 @@ def get_pip( resolver=resolver, interpreter=interpreter, extra_requirements=installation.extra_requirements, + use_system_time=installation.use_system_time, ) _PIP[installation] = pip return pip diff --git a/pex/pip/tool.py b/pex/pip/tool.py index 38874e3da..25de49db9 100644 --- a/pex/pip/tool.py +++ b/pex/pip/tool.py @@ -272,6 +272,7 @@ def analyze(self, line): @attr.s(frozen=True) class PipVenv(object): venv_dir = attr.ib() # type: str + execute_env = attr.ib() # type: Mapping[str, str] _execute_args = attr.ib() # type: Tuple[str, ...] def execute_args(self, *args): @@ -431,6 +432,7 @@ def _spawn_pip_isolated( popen_kwargs["stdout"] = sys.stderr.fileno() popen_kwargs.update(stderr=subprocess.PIPE) + env.update(self._pip.execute_env) args = self._pip.execute_args(*command) rendered_env = " ".join( diff --git a/pex/resolve/configured_resolver.py b/pex/resolve/configured_resolver.py index 46dd112d7..6e76e9e66 100644 --- a/pex/resolve/configured_resolver.py +++ b/pex/resolve/configured_resolver.py @@ -45,8 +45,13 @@ def default(cls): pip_configuration = attr.ib() # type: PipConfiguration def is_default_repos(self): + # type: () -> bool return self.pip_configuration.repos_configuration == _DEFAULT_REPOS + def use_system_time(self): + # type: () -> bool + return self.pip_configuration.build_configuration.use_system_time + def resolve_lock( self, lock, # type: Lockfile diff --git a/pex/resolve/lockfile/json_codec.py b/pex/resolve/lockfile/json_codec.py index c693bbfc4..b8283ce79 100644 --- a/pex/resolve/lockfile/json_codec.py +++ b/pex/resolve/lockfile/json_codec.py @@ -231,6 +231,8 @@ def parse_version_specifier( for index, constraint in enumerate(get("constraints", list)) ] + use_system_time = get("use_system_time", bool, optional=True) + excluded = [ parse_requirement(req, path=".excluded[{index}]".format(index=index)) for index, req in enumerate(get("excluded", list, optional=True) or ()) @@ -352,6 +354,10 @@ def assemble_tag( prefer_older_binary=get("prefer_older_binary", bool), use_pep517=get("use_pep517", bool, optional=True), build_isolation=get("build_isolation", bool), + # N.B.: Although locks are now always generated under SOURCE_DATE_EPOCH=fixed and + # PYTHONHASHSEED=0 (aka: `use_system_time=False`), that did not use to be the case. In + # those old locks there was no "use_system_time" field. + use_system_time=use_system_time if use_system_time is not None else True, ), transitive=get("transitive", bool), excluded=excluded, @@ -399,6 +405,7 @@ def as_json_data( "prefer_older_binary": lockfile.prefer_older_binary, "use_pep517": lockfile.use_pep517, "build_isolation": lockfile.build_isolation, + "use_system_time": lockfile.use_system_time, "transitive": lockfile.transitive, "excluded": [str(exclude) for exclude in lockfile.excluded], "overridden": [str(override) for override in lockfile.overridden], diff --git a/pex/resolve/lockfile/model.py b/pex/resolve/lockfile/model.py index a90e2688e..bb309580e 100644 --- a/pex/resolve/lockfile/model.py +++ b/pex/resolve/lockfile/model.py @@ -106,6 +106,7 @@ def extract_requirement(req): prefer_older_binary=build_configuration.prefer_older_binary, use_pep517=build_configuration.use_pep517, build_isolation=build_configuration.build_isolation, + use_system_time=build_configuration.use_system_time, transitive=transitive, excluded=SortedTuple(excluded), overridden=SortedTuple(overridden), @@ -130,6 +131,7 @@ def extract_requirement(req): prefer_older_binary = attr.ib() # type: bool use_pep517 = attr.ib() # type: Optional[bool] build_isolation = attr.ib() # type: bool + use_system_time = attr.ib() # type: bool transitive = attr.ib() # type: bool excluded = attr.ib() # type: SortedTuple[Requirement] overridden = attr.ib() # type: SortedTuple[Requirement] @@ -147,6 +149,7 @@ def build_configuration(self): prefer_older_binary=self.prefer_older_binary, use_pep517=self.use_pep517, build_isolation=self.build_isolation, + use_system_time=self.use_system_time, ) def dependency_configuration(self): diff --git a/pex/resolve/resolver_configuration.py b/pex/resolve/resolver_configuration.py index 92df8b0c6..ab81358a4 100644 --- a/pex/resolve/resolver_configuration.py +++ b/pex/resolve/resolver_configuration.py @@ -98,6 +98,7 @@ def create( prefer_older_binary=False, # type: bool use_pep517=None, # type: Optional[bool] build_isolation=True, # type: bool + use_system_time=False, # type: bool ): # type: (...) -> BuildConfiguration return cls( @@ -108,6 +109,7 @@ def create( prefer_older_binary=prefer_older_binary, use_pep517=use_pep517, build_isolation=build_isolation, + use_system_time=use_system_time, ) allow_builds = attr.ib(default=True) # type: bool @@ -117,6 +119,7 @@ def create( prefer_older_binary = attr.ib(default=False) # type: bool use_pep517 = attr.ib(default=None) # type: Optional[bool] build_isolation = attr.ib(default=True) # type: bool + use_system_time = attr.ib(default=False) # type: bool def __attrs_post_init__(self): # type: () -> None diff --git a/pex/resolve/resolver_options.py b/pex/resolve/resolver_options.py index 902106e70..8d2144eba 100644 --- a/pex/resolve/resolver_options.py +++ b/pex/resolve/resolver_options.py @@ -539,15 +539,19 @@ class InvalidConfigurationError(Exception): ] -def configure(options): - # type: (Namespace) -> ResolverConfiguration +def configure( + options, # type: Namespace + use_system_time=False, # type: bool +): + # type: (...) -> ResolverConfiguration """Creates a resolver configuration from options registered by `register`. :param options: The resolver configuration options. + :param use_system_time: `False` to attempt use a reproducible timestamp for builds. :raise: :class:`InvalidConfigurationError` if the resolver configuration is invalid. """ - pip_configuration = create_pip_configuration(options) + pip_configuration = create_pip_configuration(options, use_system_time=use_system_time) pex_repository = getattr(options, "pex_repository", None) if pex_repository: @@ -593,11 +597,15 @@ def configure(options): return pip_configuration -def create_pip_configuration(options): - # type: (Namespace) -> PipConfiguration +def create_pip_configuration( + options, # type: Namespace + use_system_time=False, # type: bool +): + # type: (...) -> PipConfiguration """Creates a Pip configuration from options registered by `register`. :param options: The Pip resolver configuration options. + :param use_system_time: `False` to attempt use a reproducible timestamp for builds. """ if options.cache_ttl: @@ -631,6 +639,7 @@ def create_pip_configuration(options): prefer_older_binary=options.prefer_older_binary, use_pep517=options.use_pep517, build_isolation=options.build_isolation, + use_system_time=use_system_time, ) return PipConfiguration( diff --git a/pex/resolve/resolvers.py b/pex/resolve/resolvers.py index 6ac7d61aa..ef98e9651 100644 --- a/pex/resolve/resolvers.py +++ b/pex/resolve/resolvers.py @@ -222,6 +222,10 @@ def is_default_repos(self): # type: () -> bool raise NotImplementedError() + def use_system_time(self): + # type: () -> bool + raise NotImplementedError() + @abstractmethod def resolve_lock( self, diff --git a/pex/tools/commands/repository.py b/pex/tools/commands/repository.py index c1b703927..8f7b6e982 100644 --- a/pex/tools/commands/repository.py +++ b/pex/tools/commands/repository.py @@ -18,13 +18,7 @@ from pex.atomic_directory import atomic_directory from pex.cache.dirs import CacheDir from pex.commands.command import JsonMixin, OutputMixin -from pex.common import ( - DETERMINISTIC_DATETIME_TIMESTAMP, - pluralize, - safe_mkdir, - safe_mkdtemp, - safe_open, -) +from pex.common import REPRODUCIBLE_BUILDS_ENV, pluralize, safe_mkdir, safe_mkdtemp, safe_open from pex.compatibility import Queue from pex.dist_metadata import Distribution from pex.environment import PEXEnvironment @@ -280,12 +274,7 @@ def spawn_extract(distribution): # type: (Distribution) -> SpawnedJob[Text] env = os.environ.copy() if not self.options.use_system_time: - # N.B.: The `SOURCE_DATE_EPOCH` env var is semi-standard magic for controlling - # build tools. Wheel has supported this since 2016. - # See: - # + https://reproducible-builds.org/docs/source-date-epoch/ - # + https://github.com/pypa/wheel/blob/1b879e53fed1f179897ed47e55a68bc51df188db/wheel/archive.py#L36-L39 - env.update(SOURCE_DATE_EPOCH=str(int(DETERMINISTIC_DATETIME_TIMESTAMP))) + env.update(REPRODUCIBLE_BUILDS_ENV) job = spawn_python_job_with_setuptools_and_wheel( args=["-m", "wheel", "pack", "--dest-dir", dest_dir, distribution.location], interpreter=pex.interpreter, diff --git a/pex/version.py b/pex/version.py index 3d29bb977..0318a2934 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,4 +1,4 @@ # Copyright 2015 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = "2.20.2" +__version__ = "2.20.3" diff --git a/tests/integration/cli/commands/test_export.py b/tests/integration/cli/commands/test_export.py index afd1c012d..3d616f639 100644 --- a/tests/integration/cli/commands/test_export.py +++ b/tests/integration/cli/commands/test_export.py @@ -57,6 +57,7 @@ prefer_older_binary=False, use_pep517=None, build_isolation=True, + use_system_time=False, transitive=True, excluded=SortedTuple(), overridden=SortedTuple(), diff --git a/tests/integration/cli/commands/test_lock.py b/tests/integration/cli/commands/test_lock.py index add0de3a6..36eb82f53 100644 --- a/tests/integration/cli/commands/test_lock.py +++ b/tests/integration/cli/commands/test_lock.py @@ -1544,7 +1544,8 @@ def test_excludes_pep517_build_requirements_issue_1565(tmpdir): "resolver_version": "pip-legacy-resolver", "style": "universal", "transitive": true, - "use_pep517": null + "use_pep517": null, + "use_system_time": false } """ @@ -1855,7 +1856,8 @@ def test_excludes_pep517_build_requirements_issue_1565(tmpdir): "resolver_version": "pip-2020-resolver", "style": "universal", "transitive": true, - "use_pep517": null + "use_pep517": null, + "use_system_time": false } """ diff --git a/tests/integration/cli/commands/test_lock_reproducibility_hash_seed.py b/tests/integration/cli/commands/test_lock_reproducibility_hash_seed.py new file mode 100644 index 000000000..fd4253f67 --- /dev/null +++ b/tests/integration/cli/commands/test_lock_reproducibility_hash_seed.py @@ -0,0 +1,189 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import filecmp +import os.path +import subprocess +from textwrap import dedent + +import pytest +from tools.commands.test_venv import make_env + +from pex.common import safe_open +from pex.typing import TYPE_CHECKING +from testing.cli import run_pex3 + +if TYPE_CHECKING: + from typing import Any, Optional + + +@pytest.fixture +def reproducibility_hostile_project(tmpdir): + # type: (Any) -> str + project = os.path.join(str(tmpdir), "project") + + # N.B.: This simulates the setup.py seen here: + # https://github.com/Unstructured-IO/unstructured/tree/06c85235ee8f014eae417b44ca17872f13960280 + # This was brought to Pex's attention here: + # https://github.com/pantsbuild/pants/discussions/21145 + with safe_open(os.path.join(project, "setup.py"), "w") as fp: + fp.write( + dedent( + """\ + from setuptools import setup + + + csv_reqs = ["pandas"] + doc_reqs = ["python-docx>=1.1.2"] + docx_reqs = ["python-docx>=1.1.2"] + epub_reqs = ["pypandoc"] + image_reqs = [ + "onnx", + "pdf2image", + "pdfminer.six", + "pikepdf", + "pillow_heif", + "pypdf", + "google-cloud-vision", + "effdet", + "unstructured-inference==0.7.36", + "unstructured.pytesseract>=0.3.12", + ] + markdown_reqs = ["markdown"] + msg_reqs = ["python-oxmsg"] + odt_reqs = ["python-docx>=1.1.2", "pypandoc"] + org_reqs = ["pypandoc"] + pdf_reqs = [ + "onnx", + "pdf2image", + "pdfminer.six", + "pikepdf", + "pillow_heif", + "pypdf", + "google-cloud-vision", + "effdet", + "unstructured-inference==0.7.36", + "unstructured.pytesseract>=0.3.12", + ] + ppt_reqs = ["python-pptx<=0.6.23"] + pptx_reqs = ["python-pptx<=0.6.23"] + rtf_reqs = ["pypandoc"] + rst_reqs = ["pypandoc"] + tsv_reqs = ["pandas"] + xlsx_reqs = [ + "openpyxl", + "pandas", + "xlrd", + "networkx", + ] + + all_doc_reqs = list( + set( + csv_reqs + + docx_reqs + + epub_reqs + + image_reqs + + markdown_reqs + + msg_reqs + + odt_reqs + + org_reqs + + pdf_reqs + + pptx_reqs + + rtf_reqs + + rst_reqs + + tsv_reqs + + xlsx_reqs, + ), + ) + + + setup( + name="reproducibility_hostile", + version="0.1.0", + extras_require={ + "all-docs": all_doc_reqs + } + ) + """ + ) + ) + with safe_open(os.path.join(project, "pyproject.toml"), "w") as fp: + fp.write( + dedent( + """\ + [build-system] + requires = ["setuptools"] + backend = "setuptools.build_meta" + """ + ) + ) + return project + + +def create_lock( + tmpdir, # type: Any + requirement, # type: str + index=None, # type: Optional[int] +): + # type: (...) -> str + lock_file = os.path.join( + str(tmpdir), + "lock{index}.json".format(index=index) if index is not None else "lock.json", + ) + run_pex3( + "lock", + "create", + requirement, + "--indent", + "2", + "-o", + lock_file, + env=make_env(PYTHONHASHSEED="random"), + ).assert_success() + return lock_file + + +def test_reproducibility_hostile_project_lock( + tmpdir, # type: Any + reproducibility_hostile_project, # type: str +): + # type: (...) -> None + + lock = create_lock(tmpdir, reproducibility_hostile_project) + for attempt in range(2): + assert filecmp.cmp( + lock, create_lock(tmpdir, reproducibility_hostile_project, attempt), shallow=False + ) + + +def test_reproducibility_hostile_vcs_lock( + tmpdir, # type: Any + reproducibility_hostile_project, # type: str +): + # type: (...) -> None + + subprocess.check_call(args=["git", "init", reproducibility_hostile_project]) + subprocess.check_call( + args=["git", "config", "user.email", "forty@two.com"], cwd=reproducibility_hostile_project + ) + subprocess.check_call( + args=["git", "config", "user.name", "Douglas Adams"], cwd=reproducibility_hostile_project + ) + subprocess.check_call( + args=["git", "checkout", "-b", "Golgafrincham"], cwd=reproducibility_hostile_project + ) + subprocess.check_call(args=["git", "add", "."], cwd=reproducibility_hostile_project) + subprocess.check_call( + args=["git", "commit", "--no-gpg-sign", "-m", "Only commit."], + cwd=reproducibility_hostile_project, + ) + + vcs_requirement = "git+file://{project}#egg=reproducibility_hostile".format( + project=reproducibility_hostile_project + ) + + lock = create_lock(tmpdir, vcs_requirement) + for attempt in range(2): + assert filecmp.cmp(lock, create_lock(tmpdir, vcs_requirement, attempt), shallow=False) diff --git a/tests/test_pip.py b/tests/test_pip.py index 079def0b1..e0082ad6c 100644 --- a/tests/test_pip.py +++ b/tests/test_pip.py @@ -337,6 +337,7 @@ def test_pip_pex_interpreter_venv_hash_issue_1885( interpreter=current_interpreter, version=PipVersion.DEFAULT, extra_requirements=(), + use_system_time=False, ) _PIP.pop(installation, None) binary = current_interpreter.binary