From 1aff7cdc3ccba92436bc99bc9b97ad18b0e1d271 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Sat, 8 May 2021 11:59:22 -0700 Subject: [PATCH] Support more verbose output for interpreter info. (#1347) Convert `-v` from a boolean to a counter and display more information for `-vv` (tags) and` -vvv` (environment markers and venv affiliation). This should be useful for debugging various issues in the field. --- pex/common.py | 6 +- pex/interpreter.py | 8 +- pex/pex_builder.py | 2 + pex/tools/commands/graph.py | 2 +- pex/tools/commands/interpreter.py | 39 ++-- .../commands/test_interpreter_command.py | 190 ++++++++++++++++++ 6 files changed, 228 insertions(+), 19 deletions(-) create mode 100644 tests/tools/commands/test_interpreter_command.py diff --git a/pex/common.py b/pex/common.py index e2ba79883..5554aacc1 100644 --- a/pex/common.py +++ b/pex/common.py @@ -529,6 +529,7 @@ def __init__(self, filename, orig_tag, new_tag): ) def __init__(self, chroot_base): + # type: (str) -> None """Create the chroot. :chroot_base Directory for the creation of the target chroot. @@ -536,9 +537,9 @@ def __init__(self, chroot_base): try: safe_mkdir(chroot_base) except OSError as e: - raise self.ChrootException("Unable to create chroot in %s: %s" % (chroot_base, e)) + raise self.Error("Unable to create chroot in %s: %s" % (chroot_base, e)) self.chroot = chroot_base - self.filesets = defaultdict(set) + self.filesets = defaultdict(set) # type: DefaultDict[str, Set[str]] def clone(self, into=None): """Clone this chroot. @@ -558,6 +559,7 @@ def clone(self, into=None): return new_chroot def path(self): + # type: () -> str """The path of the chroot.""" return self.chroot diff --git a/pex/interpreter.py b/pex/interpreter.py index 7a973fea6..1db016c38 100644 --- a/pex/interpreter.py +++ b/pex/interpreter.py @@ -250,6 +250,7 @@ def supported_tags(self): @property def env_markers(self): + # type: () -> Dict[str, str] return dict(self._env_markers) @property @@ -263,6 +264,7 @@ def interpreter(self): @property def requirement(self): + # type: () -> Requirement return self.distribution.as_requirement() @property @@ -1070,10 +1072,8 @@ def version_string(self): @property def platform(self): - """The most specific platform of this interpreter. - - :rtype: :class:`Platform` - """ + # type: () -> Platform + """The most specific platform of this interpreter.""" return next(self._identity.iter_supported_platforms()) @property diff --git a/pex/pex_builder.py b/pex/pex_builder.py index 1e6e14e99..b563b57be 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -220,6 +220,7 @@ def interpreter(self): return self._interpreter def chroot(self): + # type: () -> Chroot return self._chroot def clone(self, into=None): @@ -248,6 +249,7 @@ def clone(self, into=None): return clone def path(self): + # type: () -> str return self.chroot().path() @property diff --git a/pex/tools/commands/graph.py b/pex/tools/commands/graph.py index 6c684253b..d057fe577 100644 --- a/pex/tools/commands/graph.py +++ b/pex/tools/commands/graph.py @@ -39,7 +39,7 @@ def _create_dependency_graph(pex): ), ) marker_environment = pex.interpreter.identity.env_markers.copy() - marker_environment["extra"] = [] + marker_environment["extra"] = "" present_dists = frozenset(dist.project_name for dist in pex.resolve()) for dist in pex.resolve(): graph.add_node( diff --git a/pex/tools/commands/interpreter.py b/pex/tools/commands/interpreter.py index 8d579e6b6..3fd8d87f8 100644 --- a/pex/tools/commands/interpreter.py +++ b/pex/tools/commands/interpreter.py @@ -15,7 +15,7 @@ from pex.variables import ENV if TYPE_CHECKING: - from typing import Iterator + from typing import Any, Dict, Iterator logger = logging.getLogger(__name__) @@ -35,8 +35,15 @@ def add_arguments(self, parser): parser.add_argument( "-v", "--verbose", - action="store_true", - help="Print the interpreter requirement in addition to it's path.", + action="count", + default=0, + help=( + "Provide more information about the interpreter in json format. " + "Once: include the interpreter requirement and platform in addition to its path. " + "Twice: include the interpreter's supported tags. " + "Thrice: include the interpreter's environment markers and its venv affiliation, " + "if any." + ), ) self.add_json_options(parser, entity="verbose output") @@ -75,15 +82,23 @@ def run( try: for interpreter in self._find_interpreters(pex, all=options.all): if options.verbose: - self.dump_json( - options, - { - "path": interpreter.binary, - "requirement": str(interpreter.identity.requirement), - "platform": str(interpreter.platform), - }, - out, - ) + interpreter_info = { + "path": interpreter.binary, + "requirement": str(interpreter.identity.requirement), + "platform": str(interpreter.platform), + } # type: Dict[str, Any] + if options.verbose >= 2: + interpreter_info["supported_tags"] = [ + str(tag) for tag in interpreter.identity.supported_tags + ] + if options.verbose >= 3: + interpreter_info["env_markers"] = interpreter.identity.env_markers + interpreter_info["venv"] = interpreter.is_venv + if interpreter.is_venv: + interpreter_info[ + "base_interpreter" + ] = interpreter.resolve_base_interpreter().binary + self.dump_json(options, interpreter_info, out) else: out.write(interpreter.binary) out.write("\n") diff --git a/tests/tools/commands/test_interpreter_command.py b/tests/tools/commands/test_interpreter_command.py new file mode 100644 index 000000000..d2dd650b1 --- /dev/null +++ b/tests/tools/commands/test_interpreter_command.py @@ -0,0 +1,190 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import json +import os + +import pytest + +from pex.common import safe_mkdtemp +from pex.interpreter import PythonInterpreter +from pex.pex_builder import PEXBuilder +from pex.testing import PY35, PY36, ensure_python_interpreter +from pex.tools.commands.virtualenv import Virtualenv +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + import attr # vendor:skip + from typing import Iterable, Dict, Any +else: + from pex.third_party import attr + + +@pytest.fixture(scope="module") +def python35(): + # type: () -> PythonInterpreter + return PythonInterpreter.from_binary(ensure_python_interpreter(PY35)) + + +@pytest.fixture(scope="module") +def python36(): + # type: () -> PythonInterpreter + return PythonInterpreter.from_binary(ensure_python_interpreter(PY36)) + + +@attr.s(frozen=True) +class InterpreterTool(object): + tools_pex = attr.ib() # type: str + interpreter = attr.ib() # type: PythonInterpreter + other_interpreters = attr.ib(default=()) # type: Iterable[PythonInterpreter] + + @classmethod + def create( + cls, + interpreter, # type: PythonInterpreter + *other_interpreters # type: PythonInterpreter + ): + # type: (...) -> InterpreterTool + pex_builder = PEXBuilder(include_tools=True, interpreter=interpreter) + pex_builder.freeze() + return cls( + tools_pex=pex_builder.path(), + interpreter=interpreter, + other_interpreters=other_interpreters, + ) + + def run( + self, + *args, # type: str + **env # type: str + ): + # type: (...) -> str + cmd = [self.tools_pex, "interpreter"] + if args: + cmd.extend(args) + + environ = os.environ.copy() + interpreters = [self.interpreter] + interpreters.extend(self.other_interpreters) + environ.update( + PEX_PYTHON_PATH=":".join(interpreter.binary for interpreter in interpreters), + PEX_TOOLS="1", + ) + environ.update(env) + + _, stdout, _ = self.interpreter.execute(args=cmd, env=environ) + return stdout + + +@pytest.fixture(scope="module") +def interpreter_tool( + python35, # type: PythonInterpreter + python36, # type: PythonInterpreter +): + # type: (...) -> InterpreterTool + return InterpreterTool.create(python35, python36) + + +def expected_basic(interpreter): + # type: (PythonInterpreter) -> str + return interpreter.binary + + +def test_basic( + python35, # type: PythonInterpreter + interpreter_tool, # type: InterpreterTool +): + # type: (...) -> None + output = interpreter_tool.run() + assert expected_basic(python35) == output.strip() + + +def test_basic_all( + python35, # type: PythonInterpreter + python36, # type: PythonInterpreter + interpreter_tool, # type: InterpreterTool +): + # type: (...) -> None + output = interpreter_tool.run("-a") + assert [ + expected_basic(interpreter) for interpreter in (python35, python36) + ] == output.splitlines() + + +def expected_verbose(interpreter): + # type: (PythonInterpreter) -> Dict[str, Any] + return { + "path": interpreter.binary, + "platform": str(interpreter.platform), + "requirement": str(interpreter.identity.requirement), + } + + +def test_verbose( + python35, # type: PythonInterpreter + interpreter_tool, # type: InterpreterTool +): + # type: (...) -> None + output = interpreter_tool.run("-v") + assert expected_verbose(python35) == json.loads(output) + + +def test_verbose_all( + python35, # type: PythonInterpreter + python36, # type: PythonInterpreter + interpreter_tool, # type: InterpreterTool +): + # type: (...) -> None + output = interpreter_tool.run("-va") + assert [expected_verbose(interpreter) for interpreter in (python35, python36)] == [ + json.loads(line) for line in output.splitlines() + ] + + +def expected_verbose_verbose(interpreter): + # type: (PythonInterpreter) -> Dict[str, Any] + expected = expected_verbose(interpreter) + expected.update(supported_tags=[str(tag) for tag in interpreter.identity.supported_tags]) + return expected + + +def test_verbose_verbose( + python35, # type: PythonInterpreter + interpreter_tool, # type: InterpreterTool +): + # type: (...) -> None + output = interpreter_tool.run("-vv") + assert expected_verbose_verbose(python35) == json.loads(output) + + +def test_verbose_verbose_verbose( + python35, # type: PythonInterpreter + interpreter_tool, # type: InterpreterTool +): + # type: (...) -> None + output = interpreter_tool.run("-vvv") + expected = expected_verbose_verbose(python35) + expected.update(env_markers=python35.identity.env_markers, venv=False) + assert expected == json.loads(output) + + +def test_verbose_verbose_verbose_venv( + python36, # type: PythonInterpreter +): + # type: (...) -> None + venv = Virtualenv.create(venv_dir=safe_mkdtemp(), interpreter=python36, force=True) + assert venv.interpreter.is_venv + + # N.B.: Non-venv-mode PEXes always escape venvs to prevent `sys.path` contamination unless + # `PEX_INHERIT_PATH` is not "false". + output = InterpreterTool.create(venv.interpreter).run("-vvv", PEX_INHERIT_PATH="fallback") + + expected = expected_verbose_verbose(venv.interpreter) + expected.update( + env_markers=venv.interpreter.identity.env_markers, + venv=True, + base_interpreter=python36.binary, + ) + assert expected == json.loads(output)