Skip to content

Commit

Permalink
Plumb reproducible build env vars more thoroughly.
Browse files Browse the repository at this point in the history
This helps reduce cache sizes where sdists are built into wheels as well
as expanding the set of sdists Pex can lock reproducibly.
  • Loading branch information
jsirois committed Oct 9, 2024
1 parent 120873b commit f173109
Show file tree
Hide file tree
Showing 20 changed files with 302 additions and 28 deletions.
15 changes: 15 additions & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,20 @@
# Release Notes

## 2.20.3

This release fixes both PEX building and lock creation via
`pex3 lock {create,sync}` to be reproducible in more cases. Previously,
if a requirement only available in source form (an sdist, a local
project or a VCS requirement) had a build that was not reproducible due
to either file timestamps (where the `SOURCE_DATE_EPOCH` standard was
respected) or random iteration order (e.g.: the `setup.py` used sets in
certain in-opportune ways), Pex's outputs would mirror the problematic
requirement's non-reproducibility. Now Pex plumbs a fixed
`SOURCE_DATE_EPOCH` and `PYTHONHASHSEED` to all places sources are
built.

* Plumb reproducible build env vars more thoroughly. (#2554)

## 2.20.2

This release fixes an old bug handling certain sdist zips under
Expand Down
4 changes: 3 additions & 1 deletion pex/bin/pex.py
Original file line number Diff line number Diff line change
Expand Up @@ -1254,7 +1254,9 @@ def main(args=None):
try:
with global_environment(options) as env:
try:
resolver_configuration = resolver_options.configure(options)
resolver_configuration = resolver_options.configure(
options, use_system_time=options.use_system_time
)
except resolver_options.InvalidConfigurationError as e:
die(str(e))

Expand Down
1 change: 1 addition & 0 deletions pex/build_system/pep_517.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ def _default_build_system(
resolved=resolved_dists,
build_backend=DEFAULT_BUILD_BACKEND,
backend_path=(),
use_system_time=resolver.use_system_time(),
**extra_env
)
)
Expand Down
7 changes: 7 additions & 0 deletions pex/build_system/pep_518.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import subprocess

from pex.build_system import DEFAULT_BUILD_BACKEND
from pex.common import REPRODUCIBLE_BUILDS_ENV
from pex.dist_metadata import Distribution
from pex.interpreter import PythonInterpreter
from pex.pex import PEX
Expand Down Expand Up @@ -80,13 +81,16 @@ def create(
build_backend, # type: str
backend_path, # type: Tuple[str, ...]
extra_requirements=None, # type: Optional[Iterable[str]]
use_system_time=False, # type: bool
**extra_env # type: str
):
# type: (...) -> Union[BuildSystem, Error]
pex_builder = PEXBuilder()
pex_builder.info.venv = True
pex_builder.info.venv_site_packages_copies = True
pex_builder.info.venv_bin_path = BinPath.PREPEND
# Allow REPRODUCIBLE_BUILDS_ENV PYTHONHASHSEED env var to take effect.
pex_builder.info.venv_hermetic_scripts = False
for req in requires:
pex_builder.add_requirement(req)
for dist in resolved:
Expand Down Expand Up @@ -144,6 +148,8 @@ def create(
env.update(extra_env)
if backend_path:
env.update(PEX_EXTRA_SYS_PATH=os.pathsep.join(backend_path))
if not use_system_time:
env.update(REPRODUCIBLE_BUILDS_ENV)
return cls(
venv_pex=venv_pex, build_backend=build_backend, requires=tuple(requires), env=env
)
Expand Down Expand Up @@ -190,4 +196,5 @@ def load_build_system(
build_backend=build_system_table.build_backend,
backend_path=build_system_table.backend_path,
extra_requirements=extra_requirements,
use_system_time=resolver.use_system_time(),
)
2 changes: 1 addition & 1 deletion pex/cache/dirs.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ def iter_transitive_dependents(self):

PIP = Value(
"pip",
version=0,
version=1,
name="Pip Versions",
description="Isolated Pip caches and Pip PEXes Pex uses to resolve distributions.",
)
Expand Down
20 changes: 15 additions & 5 deletions pex/cli/commands/lock.py
Original file line number Diff line number Diff line change
Expand Up @@ -799,7 +799,10 @@ def _resolve_targets(
# type: (...) -> Union[Targets, Error]

target_config = target_configuration or target_options.configure(
self.options, pip_configuration=resolver_options.create_pip_configuration(self.options)
self.options,
pip_configuration=resolver_options.create_pip_configuration(
self.options, use_system_time=False
),
)
if style is not LockStyle.UNIVERSAL:
return target_config.resolve_targets()
Expand Down Expand Up @@ -867,7 +870,9 @@ def _gather_requirements(
def _create(self):
# type: () -> Result

pip_configuration = resolver_options.create_pip_configuration(self.options)
pip_configuration = resolver_options.create_pip_configuration(
self.options, use_system_time=False
)
target_configuration = target_options.configure(
self.options, pip_configuration=pip_configuration
)
Expand Down Expand Up @@ -963,7 +968,9 @@ def _export(self, requirement_configuration=RequirementConfiguration()):
)

lockfile_path, lock_file = self._load_lockfile()
pip_configuration = resolver_options.create_pip_configuration(self.options)
pip_configuration = resolver_options.create_pip_configuration(
self.options, use_system_time=False
)
targets = target_options.configure(
self.options, pip_configuration=pip_configuration
).resolve_targets()
Expand Down Expand Up @@ -1091,7 +1098,9 @@ def _create_lock_update_request(
):
# type: (...) -> Union[LockUpdateRequest, Error]

pip_configuration = resolver_options.create_pip_configuration(self.options)
pip_configuration = resolver_options.create_pip_configuration(
self.options, use_system_time=False
)
lock_updater = LockUpdater.create(
lock_file=lock_file,
repos_configuration=pip_configuration.repos_configuration,
Expand Down Expand Up @@ -1506,7 +1515,8 @@ def _sync(self):
# type: () -> Result

resolver_configuration = cast(
LockRepositoryConfiguration, resolver_options.configure(self.options)
LockRepositoryConfiguration,
resolver_options.configure(self.options, use_system_time=False),
)
production_assert(isinstance(resolver_configuration, LockRepositoryConfiguration))
pip_configuration = resolver_configuration.pip_configuration
Expand Down
9 changes: 9 additions & 0 deletions pex/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,15 @@
_UNIX_EPOCH = datetime(year=1970, month=1, day=1, hour=0, minute=0, second=0, tzinfo=None)
DETERMINISTIC_DATETIME_TIMESTAMP = (DETERMINISTIC_DATETIME - _UNIX_EPOCH).total_seconds()

# N.B.: The `SOURCE_DATE_EPOCH` env var is semi-standard magic for controlling
# build tools. Wheel, for example, has supported this since 2016.
# See:
# + https://reproducible-builds.org/docs/source-date-epoch/
# + https://github.com/pypa/wheel/blob/1b879e53fed1f179897ed47e55a68bc51df188db/wheel/archive.py#L36-L39
REPRODUCIBLE_BUILDS_ENV = dict(
PYTHONHASHSEED="0", SOURCE_DATE_EPOCH=str(int(DETERMINISTIC_DATETIME_TIMESTAMP))
)


def is_pyc_dir(dir_path):
# type: (Text) -> bool
Expand Down
21 changes: 19 additions & 2 deletions pex/pip/installation.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from pex import pex_warnings, third_party
from pex.atomic_directory import atomic_directory
from pex.cache.dirs import CacheDir
from pex.common import pluralize, safe_mkdtemp
from pex.common import REPRODUCIBLE_BUILDS_ENV, pluralize, safe_mkdtemp
from pex.dist_metadata import Requirement
from pex.interpreter import PythonInterpreter
from pex.orderedset import OrderedSet
Expand Down Expand Up @@ -42,6 +42,7 @@ def _pip_installation(
iter_distribution_locations, # type: Callable[[], Iterator[str]]
fingerprint, # type: str
interpreter=None, # type: Optional[PythonInterpreter]
use_system_time=False, # type: bool
):
# type: (...) -> Pip
pip_root = CacheDir.PIP.path(str(version))
Expand All @@ -54,6 +55,8 @@ def _pip_installation(

isolated_pip_builder = PEXBuilder(path=chroot.work_dir)
isolated_pip_builder.info.venv = True
# Allow REPRODUCIBLE_BUILDS_ENV PYTHONHASHSEED env var to take effect if needed.
isolated_pip_builder.info.venv_hermetic_scripts = False
for dist_location in iter_distribution_locations():
isolated_pip_builder.add_dist_location(dist=dist_location)
with named_temporary_file(prefix="", suffix=".py", mode="w") as fp:
Expand All @@ -78,7 +81,11 @@ def _pip_installation(
isolated_pip_builder.freeze()
pip_cache = os.path.join(pip_root, "pip_cache")
pip_pex = ensure_venv(PEX(pip_pex_path, interpreter=pip_interpreter))
pip_venv = PipVenv(venv_dir=pip_pex.venv_dir, execute_args=tuple(pip_pex.execute_args()))
pip_venv = PipVenv(
venv_dir=pip_pex.venv_dir,
execute_env=REPRODUCIBLE_BUILDS_ENV if not use_system_time else {},
execute_args=tuple(pip_pex.execute_args()),
)
return Pip(pip=pip_venv, version=version, pip_cache=pip_cache)


Expand All @@ -98,6 +105,7 @@ def _vendored_installation(
interpreter=None, # type: Optional[PythonInterpreter]
resolver=None, # type: Optional[Resolver]
extra_requirements=(), # type: Tuple[Requirement, ...]
use_system_time=False, # type: bool
):
# type: (...) -> Pip

Expand All @@ -111,6 +119,7 @@ def expose_vendored():
iter_distribution_locations=expose_vendored,
interpreter=interpreter,
fingerprint=_fingerprint(extra_requirements),
use_system_time=use_system_time,
)

if not resolver:
Expand Down Expand Up @@ -171,6 +180,7 @@ def iter_distribution_locations():
iter_distribution_locations=iter_distribution_locations,
interpreter=interpreter,
fingerprint=_fingerprint(extra_requirements),
use_system_time=use_system_time,
)


Expand Down Expand Up @@ -204,6 +214,7 @@ def _resolved_installation(
resolver=None, # type: Optional[Resolver]
interpreter=None, # type: Optional[PythonInterpreter]
extra_requirements=(), # type: Tuple[Requirement, ...]
use_system_time=False, # type: bool
):
# type: (...) -> Pip
targets = Targets.from_target(LocalInterpreter.create(interpreter))
Expand All @@ -222,6 +233,7 @@ def _resolved_installation(
iter_distribution_locations=_bootstrap_pip(version, interpreter=interpreter),
interpreter=interpreter,
fingerprint=_fingerprint(extra_requirements),
use_system_time=use_system_time,
)

requirements_by_project_name = OrderedDict(
Expand Down Expand Up @@ -269,6 +281,7 @@ def resolve_distribution_locations():
iter_distribution_locations=resolve_distribution_locations,
interpreter=interpreter,
fingerprint=_fingerprint(extra_requirements),
use_system_time=use_system_time,
)


Expand All @@ -277,6 +290,7 @@ class PipInstallation(object):
interpreter = attr.ib() # type: PythonInterpreter
version = attr.ib() # type: PipVersionValue
extra_requirements = attr.ib() # type: Tuple[Requirement, ...]
use_system_time = attr.ib() # type: bool

def check_python_applies(self):
# type: () -> None
Expand Down Expand Up @@ -396,6 +410,7 @@ def get_pip(
interpreter=interpreter or PythonInterpreter.get(),
version=calculated_version,
extra_requirements=extra_requirements,
use_system_time=resolver.use_system_time() if resolver else False,
)
pip = _PIP.get(installation)
if pip is None:
Expand All @@ -405,13 +420,15 @@ def get_pip(
interpreter=interpreter,
resolver=resolver,
extra_requirements=installation.extra_requirements,
use_system_time=installation.use_system_time,
)
else:
pip = _resolved_installation(
version=installation.version,
resolver=resolver,
interpreter=interpreter,
extra_requirements=installation.extra_requirements,
use_system_time=installation.use_system_time,
)
_PIP[installation] = pip
return pip
2 changes: 2 additions & 0 deletions pex/pip/tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,7 @@ def analyze(self, line):
@attr.s(frozen=True)
class PipVenv(object):
venv_dir = attr.ib() # type: str
execute_env = attr.ib() # type: Mapping[str, str]
_execute_args = attr.ib() # type: Tuple[str, ...]

def execute_args(self, *args):
Expand Down Expand Up @@ -431,6 +432,7 @@ def _spawn_pip_isolated(
popen_kwargs["stdout"] = sys.stderr.fileno()
popen_kwargs.update(stderr=subprocess.PIPE)

env.update(self._pip.execute_env)
args = self._pip.execute_args(*command)

rendered_env = " ".join(
Expand Down
5 changes: 5 additions & 0 deletions pex/resolve/configured_resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,13 @@ def default(cls):
pip_configuration = attr.ib() # type: PipConfiguration

def is_default_repos(self):
# type: () -> bool
return self.pip_configuration.repos_configuration == _DEFAULT_REPOS

def use_system_time(self):
# type: () -> bool
return self.pip_configuration.build_configuration.use_system_time

def resolve_lock(
self,
lock, # type: Lockfile
Expand Down
7 changes: 7 additions & 0 deletions pex/resolve/lockfile/json_codec.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,8 @@ def parse_version_specifier(
for index, constraint in enumerate(get("constraints", list))
]

use_system_time = get("use_system_time", bool, optional=True)

excluded = [
parse_requirement(req, path=".excluded[{index}]".format(index=index))
for index, req in enumerate(get("excluded", list, optional=True) or ())
Expand Down Expand Up @@ -352,6 +354,10 @@ def assemble_tag(
prefer_older_binary=get("prefer_older_binary", bool),
use_pep517=get("use_pep517", bool, optional=True),
build_isolation=get("build_isolation", bool),
# N.B.: Although locks are now always generated under SOURCE_DATE_EPOCH=fixed and
# PYTHONHASHSEED=0 (aka: `use_system_time=False`), that did not use to be the case. In
# those old locks there was no "use_system_time" field.
use_system_time=use_system_time if use_system_time is not None else True,
),
transitive=get("transitive", bool),
excluded=excluded,
Expand Down Expand Up @@ -399,6 +405,7 @@ def as_json_data(
"prefer_older_binary": lockfile.prefer_older_binary,
"use_pep517": lockfile.use_pep517,
"build_isolation": lockfile.build_isolation,
"use_system_time": lockfile.use_system_time,
"transitive": lockfile.transitive,
"excluded": [str(exclude) for exclude in lockfile.excluded],
"overridden": [str(override) for override in lockfile.overridden],
Expand Down
3 changes: 3 additions & 0 deletions pex/resolve/lockfile/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ def extract_requirement(req):
prefer_older_binary=build_configuration.prefer_older_binary,
use_pep517=build_configuration.use_pep517,
build_isolation=build_configuration.build_isolation,
use_system_time=build_configuration.use_system_time,
transitive=transitive,
excluded=SortedTuple(excluded),
overridden=SortedTuple(overridden),
Expand All @@ -130,6 +131,7 @@ def extract_requirement(req):
prefer_older_binary = attr.ib() # type: bool
use_pep517 = attr.ib() # type: Optional[bool]
build_isolation = attr.ib() # type: bool
use_system_time = attr.ib() # type: bool
transitive = attr.ib() # type: bool
excluded = attr.ib() # type: SortedTuple[Requirement]
overridden = attr.ib() # type: SortedTuple[Requirement]
Expand All @@ -147,6 +149,7 @@ def build_configuration(self):
prefer_older_binary=self.prefer_older_binary,
use_pep517=self.use_pep517,
build_isolation=self.build_isolation,
use_system_time=self.use_system_time,
)

def dependency_configuration(self):
Expand Down
3 changes: 3 additions & 0 deletions pex/resolve/resolver_configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ def create(
prefer_older_binary=False, # type: bool
use_pep517=None, # type: Optional[bool]
build_isolation=True, # type: bool
use_system_time=False, # type: bool
):
# type: (...) -> BuildConfiguration
return cls(
Expand All @@ -108,6 +109,7 @@ def create(
prefer_older_binary=prefer_older_binary,
use_pep517=use_pep517,
build_isolation=build_isolation,
use_system_time=use_system_time,
)

allow_builds = attr.ib(default=True) # type: bool
Expand All @@ -117,6 +119,7 @@ def create(
prefer_older_binary = attr.ib(default=False) # type: bool
use_pep517 = attr.ib(default=None) # type: Optional[bool]
build_isolation = attr.ib(default=True) # type: bool
use_system_time = attr.ib(default=False) # type: bool

def __attrs_post_init__(self):
# type: () -> None
Expand Down
Loading

0 comments on commit f173109

Please sign in to comment.