From cc3844f92e8504b6873b798fb084eb4d9e91705a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 8 Jan 2022 14:25:04 +0100 Subject: [PATCH] Add --report option to pip install --- news/53.feature.rst | 3 + src/pip/_internal/commands/install.py | 19 ++ .../_internal/models/installation_report.py | 40 ++++ tests/functional/test_install_report.py | 175 ++++++++++++++++++ 4 files changed, 237 insertions(+) create mode 100644 news/53.feature.rst create mode 100644 src/pip/_internal/models/installation_report.py create mode 100644 tests/functional/test_install_report.py diff --git a/news/53.feature.rst b/news/53.feature.rst new file mode 100644 index 00000000000..707718efb38 --- /dev/null +++ b/news/53.feature.rst @@ -0,0 +1,3 @@ +Add ``--report`` to the install command to generate a json report of what was installed. +In combination with ``--dry-run`` and ``--ignore-installed`` it can be used to resolve +the requirements. diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 4e63df1bb4f..c3234081ac4 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -1,4 +1,5 @@ import errno +import json import operator import os import shutil @@ -21,6 +22,7 @@ from pip._internal.locations import get_scheme from pip._internal.metadata import get_environment from pip._internal.models.format_control import FormatControl +from pip._internal.models.installation_report import InstallationReport from pip._internal.operations.build.build_tracker import get_build_tracker from pip._internal.operations.check import ConflictDetails, check_install_conflicts from pip._internal.req import install_given_reqs @@ -250,6 +252,19 @@ def add_options(self) -> None: self.parser.insert_option_group(0, index_opts) self.parser.insert_option_group(0, self.cmd_opts) + self.cmd_opts.add_option( + "--report", + dest="json_report_file", + metavar="file", + default=None, + help=( + "Generate a JSON file describing what pip did to install " + "the provided requirements. " + "Can be used in combination with --dry-run and --ignore-installed " + "to 'resolve' the requirements." + ), + ) + @with_cleanup def run(self, options: Values, args: List[str]) -> int: if options.use_user_site and options.target_dir is not None: @@ -352,6 +367,10 @@ def run(self, options: Values, args: List[str]) -> int: requirement_set = resolver.resolve( reqs, check_supported_wheels=not options.target_dir ) + if options.json_report_file: + report = InstallationReport(requirement_set) + with open(options.json_report_file, "w") as f: + json.dump(report.to_dict(), f) if options.dry_run: items = [ diff --git a/src/pip/_internal/models/installation_report.py b/src/pip/_internal/models/installation_report.py new file mode 100644 index 00000000000..98271c550ed --- /dev/null +++ b/src/pip/_internal/models/installation_report.py @@ -0,0 +1,40 @@ +from typing import Any, Dict + +from pip._internal.req.req_install import InstallRequirement +from pip._internal.req.req_set import RequirementSet + + +class InstallationReport: + def __init__(self, req_set: RequirementSet): + self._req_set = req_set + + @classmethod + def _install_req_to_dict(cls, ireq: InstallRequirement) -> Dict[str, Any]: + assert ireq.download_info, f"No download_info for {ireq}" + res = { + # PEP 610 json for the download URL. download_info.archive_info.hash may + # be absent when the requirement was installed from the wheel cache + # and the cache entry was populated by an older pip version that did not + # record origin.json. + "download_info": ireq.download_info.to_dict(), + # is_direct is true if the requirement was a direct URL reference (which + # includes editable requirements), and false if the requirement was + # downloaded from a PEP 503 index or --find-links. + "is_direct": bool(ireq.original_link), + # requested is true if the requirement was specified by the user (aka + # top level requirement), and false if it was installed as a dependency of a + # requirement. https://peps.python.org/pep-0376/#requested + "requested": ireq.user_supplied, + # PEP 566 json encoding for metadata + # https://www.python.org/dev/peps/pep-0566/#json-compatible-metadata + "metadata": ireq.get_dist().json_metadata, + } + return res + + def to_dict(self) -> Dict[str, Any]: + return { + "install": { + name: self._install_req_to_dict(ireq) + for name, ireq in self._req_set.requirements.items() + } + } diff --git a/tests/functional/test_install_report.py b/tests/functional/test_install_report.py new file mode 100644 index 00000000000..7aa637c3680 --- /dev/null +++ b/tests/functional/test_install_report.py @@ -0,0 +1,175 @@ +import json +from pathlib import Path + +import pytest + +from ..lib import PipTestEnvironment, TestData + + +@pytest.mark.usefixtures("with_wheel") +def test_install_report_basic( + script: PipTestEnvironment, shared_data: TestData, tmp_path: Path +) -> None: + report_path = tmp_path / "report.json" + script.pip( + "install", + "simplewheel", + "--dry-run", + "--no-index", + "--find-links", + str(shared_data.root / "packages/"), + "--report", + str(report_path), + ) + report = json.loads(report_path.read_text()) + assert "install" in report + assert len(report["install"]) == 1 + assert "simplewheel" in report["install"] + simplewheel_report = report["install"]["simplewheel"] + assert simplewheel_report["metadata"]["name"] == "simplewheel" + assert simplewheel_report["requested"] is True + assert simplewheel_report["is_direct"] is False + url = simplewheel_report["download_info"]["url"] + assert url.startswith("file://") + assert url.endswith("/packages/simplewheel-2.0-1-py2.py3-none-any.whl") + assert ( + simplewheel_report["download_info"]["archive_info"]["hash"] + == "sha256=191d6520d0570b13580bf7642c97ddfbb46dd04da5dd2cf7bef9f32391dfe716" + ) + + +@pytest.mark.usefixtures("with_wheel") +def test_install_report_dep( + script: PipTestEnvironment, shared_data: TestData, tmp_path: Path +) -> None: + """Test dependencies are present in the install report with requested=False.""" + report_path = tmp_path / "report.json" + script.pip( + "install", + "require_simple", + "--dry-run", + "--no-index", + "--find-links", + str(shared_data.root / "packages/"), + "--report", + str(report_path), + ) + report = json.loads(report_path.read_text()) + assert len(report["install"]) == 2 + assert report["install"]["require-simple"]["requested"] is True + assert report["install"]["simple"]["requested"] is False + + +@pytest.mark.network +@pytest.mark.usefixtures("with_wheel") +def test_install_report_index(script: PipTestEnvironment, tmp_path: Path) -> None: + """Test report for wheels obtained from index.""" + report_path = tmp_path / "report.json" + script.pip( + "install", + "--dry-run", + "Paste[openid]==1.7.5.1", + "--report", + str(report_path), + ) + report = json.loads(report_path.read_text()) + assert len(report["install"]) == 2 + assert report["install"]["paste"]["requested"] is True + assert report["install"]["python-openid"]["requested"] is False + paste_report = report["install"]["paste"] + assert paste_report["download_info"]["url"].startswith( + "https://files.pythonhosted.org/" + ) + assert paste_report["download_info"]["url"].endswith("/Paste-1.7.5.1.tar.gz") + assert ( + paste_report["download_info"]["archive_info"]["hash"] + == "sha256=11645842ba8ec986ae8cfbe4c6cacff5c35f0f4527abf4f5581ae8b4ad49c0b6" + ) + + +@pytest.mark.network +@pytest.mark.usefixtures("with_wheel") +def test_install_report_vcs_and_wheel_cache( + script: PipTestEnvironment, tmp_path: Path +) -> None: + """Test report for wheels obtained from index.""" + cache_dir = tmp_path / "cache" + report_path = tmp_path / "report.json" + script.pip( + "install", + "git+https://github.com/pypa/pip-test-package" + "@5547fa909e83df8bd743d3978d6667497983a4b7", + "--cache-dir", + str(cache_dir), + "--report", + str(report_path), + ) + report = json.loads(report_path.read_text()) + assert len(report["install"]) == 1 + pip_test_package_report = report["install"]["pip-test-package"] + assert pip_test_package_report["is_direct"] is True + assert pip_test_package_report["requested"] is True + assert ( + pip_test_package_report["download_info"]["url"] + == "https://github.com/pypa/pip-test-package" + ) + assert pip_test_package_report["download_info"]["vcs_info"]["vcs"] == "git" + assert ( + pip_test_package_report["download_info"]["vcs_info"]["commit_id"] + == "5547fa909e83df8bd743d3978d6667497983a4b7" + ) + # Now do it again to make sure the cache is used and that the report still contains + # the original VCS url. + report_path.unlink() + result = script.pip( + "install", + "pip-test-package @ git+https://github.com/pypa/pip-test-package" + "@5547fa909e83df8bd743d3978d6667497983a4b7", + "--ignore-installed", + "--cache-dir", + str(cache_dir), + "--report", + str(report_path), + ) + assert "Using cached pip_test_package" in result.stdout + report = json.loads(report_path.read_text()) + assert len(report["install"]) == 1 + pip_test_package_report = report["install"]["pip-test-package"] + assert pip_test_package_report["is_direct"] is True + assert pip_test_package_report["requested"] is True + assert ( + pip_test_package_report["download_info"]["url"] + == "https://github.com/pypa/pip-test-package" + ) + assert pip_test_package_report["download_info"]["vcs_info"]["vcs"] == "git" + assert ( + pip_test_package_report["download_info"]["vcs_info"]["commit_id"] + == "5547fa909e83df8bd743d3978d6667497983a4b7" + ) + + +@pytest.mark.network +@pytest.mark.usefixtures("with_wheel") +def test_install_report_vcs_editable( + script: PipTestEnvironment, tmp_path: Path +) -> None: + """Test report for wheels obtained from index.""" + report_path = tmp_path / "report.json" + script.pip( + "install", + "--editable", + "git+https://github.com/pypa/pip-test-package" + "@5547fa909e83df8bd743d3978d6667497983a4b7" + "#egg=pip-test-package", + "--report", + str(report_path), + ) + report = json.loads(report_path.read_text()) + assert len(report["install"]) == 1 + pip_test_package_report = report["install"]["pip-test-package"] + assert pip_test_package_report["is_direct"] is True + assert pip_test_package_report["download_info"]["url"].startswith("file://") + assert pip_test_package_report["download_info"]["url"].endswith( + "/src/pip-test-package" + ) + assert pip_test_package_report["download_info"]["dir_info"]["editable"] is True