Expose more Pip options.

pex/bin/pex.py

@@ -605,6 +605,9 @@ def build_pex(
                         cache=cache,
                         build=resolver_configuration.allow_builds,
                         use_wheel=resolver_configuration.allow_wheels,
+                        prefer_older_binary=resolver_configuration.prefer_older_binary,
+                        use_pep517=resolver_configuration.use_pep517,
+                        build_isolation=resolver_configuration.build_isolation,
                         compile=options.compile,
                         manylinux=target_configuration.assume_manylinux,
                         max_parallel_jobs=resolver_configuration.max_jobs,

pex/cli/commands/lockfile/__init__.py

@@ -125,6 +125,9 @@ def create(
             cache=ENV.PEX_ROOT,
             build=pip_configuration.allow_builds,
             use_wheel=pip_configuration.allow_wheels,
+            prefer_older_binary=pip_configuration.prefer_older_binary,
+            use_pep517=pip_configuration.use_pep517,
+            build_isolation=pip_configuration.build_isolation,
             assume_manylinux=target_configuration.assume_manylinux,
             max_parallel_jobs=pip_configuration.max_jobs,
             lock_configuration=lock_configuration,
@@ -144,6 +147,9 @@ def create(
         allow_prereleases=pip_configuration.allow_prereleases,
         allow_wheels=pip_configuration.allow_wheels,
         allow_builds=pip_configuration.allow_builds,
+        prefer_older_binary=pip_configuration.prefer_older_binary,
+        use_pep517=pip_configuration.use_pep517,
+        build_isolation=pip_configuration.build_isolation,
         transitive=pip_configuration.transitive,
         locked_resolves=downloaded.locked_resolves,
     )

pex/cli/commands/lockfile/json_codec.py

@@ -65,6 +65,7 @@ def get(
         expected_type=compatibility.string,  # type: Union[Type, Tuple[Type, ...]]
         data=_load_json(lockfile_contents, source=source),  # type: Mapping
         path=".",  # type: str
+        optional=False,
     ):
         # type: (...) -> Any
 
@@ -79,30 +80,31 @@ def get(
                     value=data,
                 )
             )
-        try:
-            value = data[key]
-            if not isinstance(value, expected_type):
-                raise ParseError(
-                    "Expected '{path}[\"{key}\"]' in {source} to be of type {expected_type} "
-                    "but given {type} with value {value}.".format(
-                        path=path,
-                        key=key,
-                        source=source,
-                        expected_type=(
-                            " or ".join(t.__name__ for t in expected_type)
-                            if isinstance(expected_type, tuple)
-                            else expected_type.__name__
-                        ),
-                        type=type(value).__name__,
-                        value=value,
-                    )
-                )
-            return value
-        except KeyError:
+        if key not in data:
+            if optional:
+                return None
             raise ParseError(
                 "The object at '{path}' in {source} did not have the expected key "
                 "{key!r}.".format(path=path, source=source, key=key)
             )
+        value = data[key]
+        if not optional and not isinstance(value, expected_type):
+            raise ParseError(
+                "Expected '{path}[\"{key}\"]' in {source} to be of type {expected_type} "
+                "but given {type} with value {value}.".format(
+                    path=path,
+                    key=key,
+                    source=source,
+                    expected_type=(
+                        " or ".join(t.__name__ for t in expected_type)
+                        if isinstance(expected_type, tuple)
+                        else expected_type.__name__
+                    ),
+                    type=type(value).__name__,
+                    value=value,
+                )
+            )
+        return value
 
     def get_enum_value(
         enum_type,  # type: Type[Enum[_V]]
@@ -240,6 +242,9 @@ def assemble_tag(
         allow_prereleases=get("allow_prereleases", bool),
         allow_wheels=get("allow_wheels", bool),
         allow_builds=get("allow_builds", bool),
+        prefer_older_binary=get("prefer_older_binary", bool),
+        use_pep517=get("use_pep517", bool, optional=True),
+        build_isolation=get("build_isolation", bool),
         transitive=get("transitive", bool),
         locked_resolves=locked_resolves,
         source=source,
@@ -268,6 +273,9 @@ def as_json_data(lockfile):
         "allow_prereleases": lockfile.allow_prereleases,
         "allow_wheels": lockfile.allow_wheels,
         "allow_builds": lockfile.allow_builds,
+        "prefer_older_binary": lockfile.prefer_older_binary,
+        "use_pep517": lockfile.use_pep517,
+        "build_isolation": lockfile.build_isolation,
         "transitive": lockfile.transitive,
         "locked_resolves": [
             {

pex/cli/commands/lockfile/lockfile.py

@@ -133,6 +133,9 @@ def create(
         allow_prereleases,  # type: bool
         allow_wheels,  # type: bool
         allow_builds,  # type: bool
+        prefer_older_binary,  # type: bool
+        use_pep517,  # type: Optional[bool]
+        build_isolation,  # type: bool
         transitive,  # type: bool
         locked_resolves,  # type: Iterable[LockedResolve]
         source=None,  # type: Optional[str]
@@ -147,6 +150,9 @@ def create(
             allow_prereleases=allow_prereleases,
             allow_wheels=allow_wheels,
             allow_builds=allow_builds,
+            prefer_older_binary=prefer_older_binary,
+            use_pep517=use_pep517,
+            build_isolation=build_isolation,
             transitive=transitive,
             locked_resolves=SortedTuple(locked_resolves),
             source=source,
@@ -160,6 +166,9 @@ def create(
     allow_prereleases = attr.ib()  # type: bool
     allow_wheels = attr.ib()  # type: bool
     allow_builds = attr.ib()  # type: bool
+    prefer_older_binary = attr.ib()  # type: bool
+    use_pep517 = attr.ib()  # type: Optional[bool]
+    build_isolation = attr.ib()  # type: bool
     transitive = attr.ib()  # type: bool
     locked_resolves = attr.ib()  # type: SortedTuple[LockedResolve]
     source = attr.ib(default=None, eq=False)  # type: Optional[str]

pex/cli/commands/lockfile/updater.py

@@ -239,6 +239,9 @@ def create(
             allow_prereleases=lock_file.allow_prereleases,
             allow_wheels=lock_file.allow_wheels,
             allow_builds=lock_file.allow_builds,
+            prefer_older_binary=lock_file.prefer_older_binary,
+            use_pep517=lock_file.use_pep517,
+            build_isolation=lock_file.build_isolation,
             transitive=lock_file.transitive,
             repos_configuration=repos_configuration,
             network_configuration=network_configuration,

pex/interpreter.py

@@ -105,6 +105,12 @@ def get(cls, binary=None):
             sysconfig.get_config_var("MACOSX_DEPLOYMENT_TARGET")
         )
 
+        # Pex identifies interpreters using a bit of Pex code injected via an extraction of that
+        # code under the `PEX_ROOT` adjoined to `sys.path` via `PYTHONPATH`. We ignore such adjoined
+        # `sys.path` entries to discover the true base interpreter `sys.path`.
+        pythonpath = frozenset(os.environ.get("PYTHONPATH", "").split(os.pathsep))
+        sys_path = [item for item in sys.path if item and item not in pythonpath]
+
         return cls(
             binary=binary or sys.executable,
             prefix=sys.prefix,
@@ -115,6 +121,7 @@ def get(cls, binary=None):
                 # https://www.python.org/dev/peps/pep-0405/.
                 or getattr(sys, "base_prefix", sys.prefix)
             ),
+            sys_path=sys_path,
             python_tag=preferred_tag.interpreter,
             abi_tag=preferred_tag.abi,
             platform_tag=preferred_tag.platform,
@@ -128,7 +135,7 @@ def get(cls, binary=None):
     def decode(cls, encoded):
         TRACER.log("creating PythonIdentity from encoded: %s" % encoded, V=9)
         values = json.loads(encoded)
-        if len(values) != 10:
+        if len(values) != 11:
             raise cls.InvalidError("Invalid interpreter identity: %s" % encoded)
 
         supported_tags = values.pop("supported_tags")
@@ -161,6 +168,7 @@ def __init__(
         binary,  # type: str
         prefix,  # type: str
         base_prefix,  # type: str
+        sys_path,  # type: Iterable[str]
         python_tag,  # type: str
         abi_tag,  # type: str
         platform_tag,  # type: str
@@ -177,6 +185,7 @@ def __init__(
         self._binary = binary
         self._prefix = prefix
         self._base_prefix = base_prefix
+        self._sys_path = tuple(sys_path)
         self._python_tag = python_tag
         self._abi_tag = abi_tag
         self._platform_tag = platform_tag
@@ -190,6 +199,7 @@ def encode(self):
             binary=self._binary,
             prefix=self._prefix,
             base_prefix=self._base_prefix,
+            sys_path=self._sys_path,
             python_tag=self._python_tag,
             abi_tag=self._abi_tag,
             platform_tag=self._platform_tag,
@@ -216,6 +226,11 @@ def base_prefix(self):
         # type: () -> str
         return self._base_prefix
 
+    @property
+    def sys_path(self):
+        # type: () -> Tuple[str, ...]
+        return self._sys_path
+
     @property
     def python_tag(self):
         return self._python_tag
@@ -974,6 +989,17 @@ def prefix(self):
         """
         return self._identity.prefix
 
+    @property
+    def sys_path(self):
+        # type: () -> Tuple[str, ...]
+        """Return the interpreter's `sys.path`.
+
+        The implicit `$PWD` entry and any entries injected via PYTHONPATH or in the user site
+        directory are excluded such that the `sys.path` presented is the base interpreter `sys.path`
+        with no adornments.
+        """
+        return self._identity.sys_path
+
     class BaseInterpreterResolutionError(Exception):
         """Indicates the base interpreter for a virtual environment could not be resolved."""
 

pex/pip.py

@@ -790,6 +790,9 @@ def spawn_download_distributions(
         cache=None,  # type: Optional[str]
         build=True,  # type: bool
         use_wheel=True,  # type: bool
+        prefer_older_binary=False,  # type: bool
+        use_pep517=None,  # type: Optional[bool]
+        build_isolation=True,  # type: bool
         locker=None,  # type: Optional[Locker]
     ):
         # type: (...) -> Job
@@ -809,6 +812,8 @@ def spawn_download_distributions(
                 )
 
         download_cmd = ["download", "--dest", download_dir]
+        extra_env = {}  # type: Dict[str, str]
+
         if target.is_platform:
             # We're either resolving for a different host / platform or a different interpreter for
             # the current platform that we have no access to; so we need to let pip know and not
@@ -829,6 +834,16 @@ def spawn_download_distributions(
         if not use_wheel:
             download_cmd.extend(["--no-binary", ":all:"])
 
+        if prefer_older_binary:
+            download_cmd.append("--prefer-binary")
+
+        if use_pep517 is not None:
+            download_cmd.append("--use-pep517" if use_pep517 else "--no-use-pep517")
+
+        if not build_isolation:
+            download_cmd.append("--no-build-isolation")
+            extra_env.update(PEP517_BACKEND_PATH=os.pathsep.join(sys.path))
+
         if allow_prereleases:
             download_cmd.append("--pre")
 
@@ -846,7 +861,6 @@ def spawn_download_distributions(
         if requirements:
             download_cmd.extend(requirements)
 
-        extra_env = None
         log_analyzers = []  # type: List[_LogAnalyzer]
 
         if locker:
@@ -868,7 +882,7 @@ def spawn_download_distributions(
                 os.path.join(env_markers_dir, "env_markers.{}.json".format(platform)), "w"
             ) as fp:
                 json.dump(patched_environment, fp)
-            extra_env = {self._PATCHED_MARKERS_FILE_ENV_VAR_NAME: fp.name}
+            extra_env[self._PATCHED_MARKERS_FILE_ENV_VAR_NAME] = fp.name
             log_analyzers.append(_Issue10050Analyzer(platform=platform))
             TRACER.log(
                 "Patching environment markers for {} with {}".format(target, patched_environment),
@@ -917,12 +931,31 @@ def spawn_build_wheels(
         interpreter=None,  # type: Optional[PythonInterpreter]
         package_index_configuration=None,  # type: Optional[PackageIndexConfiguration]
         cache=None,  # type: Optional[str]
+        prefer_older_binary=False,  # type: bool
+        use_pep517=None,  # type: Optional[bool]
+        build_isolation=True,  # type: bool
         verify=True,  # type: bool
     ):
         # type: (...) -> Job
         wheel_cmd = ["wheel", "--no-deps", "--wheel-dir", wheel_dir]
+        extra_env = {}  # type: Dict[str, str]
+
+        # It's not clear if Pip's implementation of PEP-517 builds respects this option for
+        # resolving build dependencies, but in case it is we pass it.
+        if use_pep517 is not False and prefer_older_binary:
+            wheel_cmd.append("--prefer-binary")
+
+        if use_pep517 is not None:
+            wheel_cmd.append("--use-pep517" if use_pep517 else "--no-use-pep517")
+
+        if not build_isolation:
+            wheel_cmd.append("--no-build-isolation")
+            interpreter = interpreter or PythonInterpreter.get()
+            extra_env.update(PEP517_BACKEND_PATH=os.pathsep.join(interpreter.sys_path))
+
         if not verify:
             wheel_cmd.append("--no-verify")
+
         wheel_cmd.extend(distributions)
 
         return self._spawn_pip_isolated_job(
@@ -931,6 +964,7 @@ def spawn_build_wheels(
             package_index_configuration=package_index_configuration,
             cache=cache,
             interpreter=interpreter,
+            extra_env=extra_env,
         )
 
     @classmethod

pex/resolve/resolver_configuration.py

@@ -10,7 +10,7 @@
 
 if TYPE_CHECKING:
     import attr  # vendor:skip
-    from typing import Tuple
+    from typing import Optional, Tuple
 else:
     from pex.third_party import attr
 
@@ -40,6 +40,9 @@ class PipConfiguration(object):
     allow_prereleases = attr.ib(default=False)  # type: bool
     allow_wheels = attr.ib(default=True)  # type: bool
     allow_builds = attr.ib(default=True)  # type: bool
+    prefer_older_binary = attr.ib(default=False)  # type: bool
+    use_pep517 = attr.ib(default=None)  # type: Optional[bool]
+    build_isolation = attr.ib(default=True)  # type: bool
     transitive = attr.ib(default=True)  # type: bool
     max_jobs = attr.ib(default=DEFAULT_MAX_JOBS)  # type: int