From 31f0d25e8b6ad175457aa4b4194c02bf4a6b2a3e Mon Sep 17 00:00:00 2001 From: John Sirois Date: Sat, 28 Jul 2018 19:26:18 -0600 Subject: [PATCH 1/3] Upgrade to pex 1.4.5. Eliminate a few workarounds that are no longer needed and switch to using the released pex.pex in the release script. Work towards #5922 --- 3rdparty/python/requirements.txt | 2 +- build-support/bin/release.sh | 33 ++++++++++++------- .../pants/backend/python/interpreter_cache.py | 4 +-- src/python/pants/backend/python/pex_util.py | 14 -------- .../python/tasks/select_interpreter.py | 3 +- .../pants/backend/python/tasks/wrapped_pex.py | 22 +++---------- 6 files changed, 30 insertions(+), 48 deletions(-) diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index b51526dac15..5811ad431d6 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -14,7 +14,7 @@ mock==2.0.0 packaging==16.8 parameterized==0.6.1 pathspec==0.5.0 -pex==1.4.3 +pex==1.4.5 psutil==4.3.0 pycodestyle==2.4.0 pyflakes==2.0.0 diff --git a/build-support/bin/release.sh b/build-support/bin/release.sh index 641fca06f17..6de10928422 100755 --- a/build-support/bin/release.sh +++ b/build-support/bin/release.sh @@ -584,8 +584,6 @@ function build_pex() { build_3rdparty_packages "${PANTS_UNSTABLE_VERSION}" fi - activate_tmp_venv && trap deactivate RETURN && pip install "pex==1.4.3" || die "Failed to install pex." - local requirements=() for pkg_name in $PANTS_PEX_PACKAGES; do requirements=("${requirements[@]}" "${pkg_name}==${PANTS_UNSTABLE_VERSION}") @@ -596,16 +594,27 @@ function build_pex() { platform_flags=("${platform_flags[@]}" "--platform=${platform}") done - pex \ - -o "${dest}" \ - --entry-point="pants.bin.pants_loader:main" \ - --no-build \ - --no-pypi \ - --disable-cache \ - "${platform_flags[@]}" \ - -f "${DEPLOY_PANTS_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \ - -f "${DEPLOY_3RDPARTY_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \ - "${requirements[@]}" + ( + PEX_VERSION=$(grep "pex==" "${ROOT}/3rdparty/python/requirements.txt" | sed -e "s|pex==||") + PEX_PEX=pex27 + + cd $(mktemp -d -t build_pex.XXXXX) + trap "rm -rf $(pwd -P)" EXIT + + curl -sSL https://github.com/pantsbuild/pex/releases/download/v${PEX_VERSION}/${PEX_PEX} -O + chmod +x ./${PEX_PEX} + + ./${PEX_PEX} \ + -o "${dest}" \ + -c pants \ + --no-build \ + --no-pypi \ + --disable-cache \ + "${platform_flags[@]}" \ + -f "${DEPLOY_PANTS_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \ + -f "${DEPLOY_3RDPARTY_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \ + "${requirements[@]}" + ) if [[ "${PANTS_PEX_RELEASE}" == "stable" ]]; then mkdir -p "$(dirname "${stable_dest}")" diff --git a/src/python/pants/backend/python/interpreter_cache.py b/src/python/pants/backend/python/interpreter_cache.py index 01340de0882..9b0fe2c1455 100644 --- a/src/python/pants/backend/python/interpreter_cache.py +++ b/src/python/pants/backend/python/interpreter_cache.py @@ -13,7 +13,7 @@ from pex.resolver import resolve from pex.variables import Variables -from pants.backend.python.pex_util import create_bare_interpreter, expand_and_maybe_adjust_platform +from pants.backend.python.pex_util import expand_and_maybe_adjust_platform from pants.backend.python.targets.python_target import PythonTarget from pants.base.exceptions import TaskError from pants.process.lock import OwnerPrintingInterProcessFileLock @@ -105,7 +105,7 @@ def _interpreter_from_path(self, path, filters): executable = os.readlink(os.path.join(path, 'python')) except OSError: return None - interpreter = create_bare_interpreter(executable) + interpreter = PythonInterpreter.from_binary(executable, include_site_extras=False) if self._matches(interpreter, filters): return self._resolve(interpreter) return None diff --git a/src/python/pants/backend/python/pex_util.py b/src/python/pants/backend/python/pex_util.py index a93237de5ba..4d77e7af941 100644 --- a/src/python/pants/backend/python/pex_util.py +++ b/src/python/pants/backend/python/pex_util.py @@ -13,20 +13,6 @@ logger = logging.getLogger(__name__) -def create_bare_interpreter(binary_path): - """Creates an interpreter for python binary at the given path. - - The interpreter is bare in that it has no extras associated with it. - - :returns: A bare python interpreter with no extras. - :rtype: :class:`pex.interpreter.PythonInterpreter` - """ - # TODO(John Sirois): Replace with a more direct PythonInterpreter construction API call when - # https://github.com/pantsbuild/pex/issues/510 is fixed. - interpreter_with_extras = PythonInterpreter.from_binary(binary_path) - return PythonInterpreter(binary_path, interpreter_with_extras.identity, extras=None) - - def _interpreter_str(interp): ident = interp.identity return ('PythonInterpreter({binary!r}, {identity!r} with extended info: ' diff --git a/src/python/pants/backend/python/tasks/select_interpreter.py b/src/python/pants/backend/python/tasks/select_interpreter.py index 26965e0b715..32ee8b93155 100644 --- a/src/python/pants/backend/python/tasks/select_interpreter.py +++ b/src/python/pants/backend/python/tasks/select_interpreter.py @@ -10,7 +10,6 @@ from pex.interpreter import PythonInterpreter from pants.backend.python.interpreter_cache import PythonInterpreterCache -from pants.backend.python.pex_util import create_bare_interpreter from pants.backend.python.subsystems.python_repos import PythonRepos from pants.backend.python.subsystems.python_setup import PythonSetup from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary @@ -102,7 +101,7 @@ def _get_interpreter(interpreter_path_file): with open(interpreter_path_file, 'r') as infile: lines = infile.readlines() binary = lines[0].strip() - interpreter = create_bare_interpreter(binary) + interpreter = PythonInterpreter.from_binary(binary, include_site_extras=False) for line in lines[1:]: dist_name, dist_version, location = line.strip().split('\t') interpreter = interpreter.with_extra(dist_name, dist_version, location) diff --git a/src/python/pants/backend/python/tasks/wrapped_pex.py b/src/python/pants/backend/python/tasks/wrapped_pex.py index 1676f490154..61512b9eefa 100644 --- a/src/python/pants/backend/python/tasks/wrapped_pex.py +++ b/src/python/pants/backend/python/tasks/wrapped_pex.py @@ -19,7 +19,6 @@ class WrappedPEX(object): """ _PEX_PATH_ENV_VAR_NAME = 'PEX_PATH' - _PEX_PYTHON_PATH_ENV_VAR_NAME = 'PEX_PYTHON_PATH' def __init__(self, pex, extra_pex_paths=None): """ @@ -38,28 +37,17 @@ def path(self): def cmdline(self, args=()): cmdline = ' '.join(self._pex.cmdline(args)) - - def render_env_var(key, value): - return '{key}={value}'.format(key=key, value=value) - - env_vars = [(self._PEX_PYTHON_PATH_ENV_VAR_NAME, self._pex._interpreter.binary)] - pex_path = self._pex_path() if pex_path: - env_vars.append((self._PEX_PATH_ENV_VAR_NAME, pex_path)) - - return '{execution_control_env_vars} {cmdline}'.format( - execution_control_env_vars=' '.join(render_env_var(k, v) for k, v in env_vars), - cmdline=cmdline - ) + return '{env_var_name}={pex_path} {cmdline}'.format(env_var_name=self._PEX_PATH_ENV_VAR_NAME, + pex_path=pex_path, + cmdline=cmdline) + else: + return cmdline def run(self, *args, **kwargs): env = copy(kwargs.pop('env', {})) - # Hack around bug in PEX where custom interpreters are not forwarded to PEXEnvironments. - # TODO(John Sirois): Remove when https://github.com/pantsbuild/pex/issues/522 is fixed. - env[self._PEX_PYTHON_PATH_ENV_VAR_NAME] = self._pex._interpreter.binary - pex_path = self._pex_path() if pex_path: env[self._PEX_PATH_ENV_VAR_NAME] = pex_path From e802ff606fcbf0efcfbcc42f7334193edb14d69c Mon Sep 17 00:00:00 2001 From: John Sirois Date: Sun, 29 Jul 2018 20:24:19 -0600 Subject: [PATCH 2/3] Kill Conan use of WrappedPEX. --- .../pants/backend/native/subsystems/conan.py | 17 +++++++++-------- .../tasks/native_external_library_fetch.py | 6 +++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/python/pants/backend/native/subsystems/conan.py b/src/python/pants/backend/native/subsystems/conan.py index 84c381d1179..bd719ac6984 100644 --- a/src/python/pants/backend/native/subsystems/conan.py +++ b/src/python/pants/backend/native/subsystems/conan.py @@ -13,8 +13,9 @@ from pex.pex_info import PexInfo from pants.backend.python.python_requirement import PythonRequirement +from pants.backend.python.subsystems.python_repos import PythonRepos +from pants.backend.python.subsystems.python_setup import PythonSetup from pants.backend.python.tasks.pex_build_util import dump_requirements -from pants.backend.python.tasks.wrapped_pex import WrappedPEX from pants.base.build_environment import get_pants_cachedir from pants.subsystem.subsystem import Subsystem from pants.util.dirutil import safe_concurrent_creation @@ -49,6 +50,10 @@ class Conan(Subsystem): def implementation_version(cls): return super(Conan, cls).implementation_version() + [('Conan', 0)] + @classmethod + def subsystem_dependencies(cls): + return super(Conan, cls).subsystem_dependencies() + (PythonRepos, PythonSetup) + @classmethod def register_options(cls, register): super(Conan, cls).register_options(register) @@ -57,7 +62,6 @@ def register_options(cls, register): class ConanBinary(datatype(['pex'])): """A `conan` PEX binary.""" - pass def bootstrap_conan(self): pex_info = PexInfo.default() @@ -65,14 +69,11 @@ def bootstrap_conan(self): conan_bootstrap_dir = os.path.join(get_pants_cachedir(), 'conan_support') conan_pex_path = os.path.join(conan_bootstrap_dir, 'conan_binary') interpreter = PythonInterpreter.get() - if os.path.exists(conan_pex_path): - conan_binary = WrappedPEX(PEX(conan_pex_path, interpreter)) - return self.ConanBinary(pex=conan_binary) - else: + if not os.path.exists(conan_pex_path): with safe_concurrent_creation(conan_pex_path) as safe_path: builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info) reqs = [PythonRequirement(req) for req in self.get_options().conan_requirements] dump_requirements(builder, interpreter, reqs, logger) builder.freeze() - conan_binary = WrappedPEX(PEX(conan_pex_path, interpreter)) - return self.ConanBinary(pex=conan_binary) + conan_binary = PEX(conan_pex_path, interpreter) + return self.ConanBinary(pex=conan_binary) diff --git a/src/python/pants/backend/native/tasks/native_external_library_fetch.py b/src/python/pants/backend/native/tasks/native_external_library_fetch.py index 7ac82dd2287..f2a8abe555f 100644 --- a/src/python/pants/backend/native/tasks/native_external_library_fetch.py +++ b/src/python/pants/backend/native/tasks/native_external_library_fetch.py @@ -181,7 +181,7 @@ def ensure_conan_remote_configuration(self, conan_binary): try: # Slice the command line because subprocess errors when the first element in the # list of command strings is the setting of an environment variable. - stdout = subprocess.check_output(remove_conan_center_remote_cmdline.split()[1:]) + stdout = subprocess.check_output(remove_conan_center_remote_cmdline) self.context.log.debug(stdout) except subprocess.CalledProcessError as e: if not "'conan-center' not found in remotes" in e.output: @@ -197,7 +197,7 @@ def ensure_conan_remote_configuration(self, conan_binary): index_num, remote_url) try: - stdout = subprocess.check_output(add_pants_conan_remote_cmdline.split()[1:]) + stdout = subprocess.check_output(add_pants_conan_remote_cmdline) self.context.log.debug(stdout) except subprocess.CalledProcessError as e: if not "already exists in remotes" in e.output: @@ -252,7 +252,7 @@ def _fetch_packages(self, vt, vts_results_dir): # Invoke conan to pull package from remote. try: - stdout = subprocess.check_output(cmdline.split()[1:]) + stdout = subprocess.check_output(cmdline) except subprocess.CalledProcessError as e: raise self.NativeExternalLibraryFetchError( "Error invoking conan for fetch task: {}\n".format(e.output) From 399d0b842e27965b84f63a82d2852b25321d198d Mon Sep 17 00:00:00 2001 From: John Sirois Date: Sun, 29 Jul 2018 20:29:34 -0600 Subject: [PATCH 3/3] Remove obsolete comment. --- .../pants/backend/native/tasks/native_external_library_fetch.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/python/pants/backend/native/tasks/native_external_library_fetch.py b/src/python/pants/backend/native/tasks/native_external_library_fetch.py index f2a8abe555f..8a747839461 100644 --- a/src/python/pants/backend/native/tasks/native_external_library_fetch.py +++ b/src/python/pants/backend/native/tasks/native_external_library_fetch.py @@ -179,8 +179,6 @@ def ensure_conan_remote_configuration(self, conan_binary): # and replace it with Pants-controlled remotes. remove_conan_center_remote_cmdline = self._remove_conan_center_remote_cmdline(conan_binary) try: - # Slice the command line because subprocess errors when the first element in the - # list of command strings is the setting of an environment variable. stdout = subprocess.check_output(remove_conan_center_remote_cmdline) self.context.log.debug(stdout) except subprocess.CalledProcessError as e: