From e37178775b6fbee6568c81cabccbca867f1d0b48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 8 Jan 2022 14:25:04 +0100 Subject: [PATCH 01/14] Add --report option to pip install --- news/53.feature.rst | 3 + src/pip/_internal/commands/install.py | 20 ++ .../_internal/models/installation_report.py | 39 ++++ tests/functional/test_install_report.py | 175 ++++++++++++++++++ 4 files changed, 237 insertions(+) create mode 100644 news/53.feature.rst create mode 100644 src/pip/_internal/models/installation_report.py create mode 100644 tests/functional/test_install_report.py diff --git a/news/53.feature.rst b/news/53.feature.rst new file mode 100644 index 00000000000..707718efb38 --- /dev/null +++ b/news/53.feature.rst @@ -0,0 +1,3 @@ +Add ``--report`` to the install command to generate a json report of what was installed. +In combination with ``--dry-run`` and ``--ignore-installed`` it can be used to resolve +the requirements. diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 9bb99cdf2a5..67072e44c74 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -1,4 +1,5 @@ import errno +import json import operator import os import shutil @@ -21,6 +22,7 @@ from pip._internal.locations import get_scheme from pip._internal.metadata import get_environment from pip._internal.models.format_control import FormatControl +from pip._internal.models.installation_report import InstallationReport from pip._internal.operations.build.build_tracker import get_build_tracker from pip._internal.operations.check import ConflictDetails, check_install_conflicts from pip._internal.req import install_given_reqs @@ -250,6 +252,19 @@ def add_options(self) -> None: self.parser.insert_option_group(0, index_opts) self.parser.insert_option_group(0, self.cmd_opts) + self.cmd_opts.add_option( + "--report", + dest="json_report_file", + metavar="file", + default=None, + help=( + "Generate a JSON file describing what pip did to install " + "the provided requirements. " + "Can be used in combination with --dry-run and --ignore-installed " + "to 'resolve' the requirements." + ), + ) + @with_cleanup def run(self, options: Values, args: List[str]) -> int: if options.use_user_site and options.target_dir is not None: @@ -353,6 +368,11 @@ def run(self, options: Values, args: List[str]) -> int: reqs, check_supported_wheels=not options.target_dir ) + if options.json_report_file: + report = InstallationReport(requirement_set.requirements_to_install) + with open(options.json_report_file, "w", encoding="utf-8") as f: + json.dump(report.to_dict(), f) + if options.dry_run: would_install_items = sorted( (r.metadata["name"], r.metadata["version"]) diff --git a/src/pip/_internal/models/installation_report.py b/src/pip/_internal/models/installation_report.py new file mode 100644 index 00000000000..5ae87be567f --- /dev/null +++ b/src/pip/_internal/models/installation_report.py @@ -0,0 +1,39 @@ +from typing import Any, Dict, Sequence + +from pip._internal.req.req_install import InstallRequirement + + +class InstallationReport: + def __init__(self, install_requirements: Sequence[InstallRequirement]): + self._install_requirements = install_requirements + + @classmethod + def _install_req_to_dict(cls, ireq: InstallRequirement) -> Dict[str, Any]: + assert ireq.download_info, f"No download_info for {ireq}" + res = { + # PEP 610 json for the download URL. download_info.archive_info.hash may + # be absent when the requirement was installed from the wheel cache + # and the cache entry was populated by an older pip version that did not + # record origin.json. + "download_info": ireq.download_info.to_dict(), + # is_direct is true if the requirement was a direct URL reference (which + # includes editable requirements), and false if the requirement was + # downloaded from a PEP 503 index or --find-links. + "is_direct": bool(ireq.original_link), + # requested is true if the requirement was specified by the user (aka + # top level requirement), and false if it was installed as a dependency of a + # requirement. https://peps.python.org/pep-0376/#requested + "requested": ireq.user_supplied, + # PEP 566 json encoding for metadata + # https://www.python.org/dev/peps/pep-0566/#json-compatible-metadata + "metadata": ireq.get_dist().metadata_dict, + } + return res + + def to_dict(self) -> Dict[str, Any]: + return { + "install": { + ireq.get_dist().metadata["Name"]: self._install_req_to_dict(ireq) + for ireq in self._install_requirements + } + } diff --git a/tests/functional/test_install_report.py b/tests/functional/test_install_report.py new file mode 100644 index 00000000000..f88f20d14ad --- /dev/null +++ b/tests/functional/test_install_report.py @@ -0,0 +1,175 @@ +import json +from pathlib import Path + +import pytest + +from ..lib import PipTestEnvironment, TestData + + +@pytest.mark.usefixtures("with_wheel") +def test_install_report_basic( + script: PipTestEnvironment, shared_data: TestData, tmp_path: Path +) -> None: + report_path = tmp_path / "report.json" + script.pip( + "install", + "simplewheel", + "--dry-run", + "--no-index", + "--find-links", + str(shared_data.root / "packages/"), + "--report", + str(report_path), + ) + report = json.loads(report_path.read_text()) + assert "install" in report + assert len(report["install"]) == 1 + assert "simplewheel" in report["install"] + simplewheel_report = report["install"]["simplewheel"] + assert simplewheel_report["metadata"]["name"] == "simplewheel" + assert simplewheel_report["requested"] is True + assert simplewheel_report["is_direct"] is False + url = simplewheel_report["download_info"]["url"] + assert url.startswith("file://") + assert url.endswith("/packages/simplewheel-2.0-1-py2.py3-none-any.whl") + assert ( + simplewheel_report["download_info"]["archive_info"]["hash"] + == "sha256=191d6520d0570b13580bf7642c97ddfbb46dd04da5dd2cf7bef9f32391dfe716" + ) + + +@pytest.mark.usefixtures("with_wheel") +def test_install_report_dep( + script: PipTestEnvironment, shared_data: TestData, tmp_path: Path +) -> None: + """Test dependencies are present in the install report with requested=False.""" + report_path = tmp_path / "report.json" + script.pip( + "install", + "require_simple", + "--dry-run", + "--no-index", + "--find-links", + str(shared_data.root / "packages/"), + "--report", + str(report_path), + ) + report = json.loads(report_path.read_text()) + assert len(report["install"]) == 2 + assert report["install"]["require-simple"]["requested"] is True + assert report["install"]["simple"]["requested"] is False + + +@pytest.mark.network +@pytest.mark.usefixtures("with_wheel") +def test_install_report_index(script: PipTestEnvironment, tmp_path: Path) -> None: + """Test report for sdist obtained from index.""" + report_path = tmp_path / "report.json" + script.pip( + "install", + "--dry-run", + "Paste[openid]==1.7.5.1", + "--report", + str(report_path), + ) + report = json.loads(report_path.read_text()) + assert len(report["install"]) == 2 + assert report["install"]["Paste"]["requested"] is True + assert report["install"]["python-openid"]["requested"] is False + paste_report = report["install"]["Paste"] + assert paste_report["download_info"]["url"].startswith( + "https://files.pythonhosted.org/" + ) + assert paste_report["download_info"]["url"].endswith("/Paste-1.7.5.1.tar.gz") + assert ( + paste_report["download_info"]["archive_info"]["hash"] + == "sha256=11645842ba8ec986ae8cfbe4c6cacff5c35f0f4527abf4f5581ae8b4ad49c0b6" + ) + + +@pytest.mark.network +@pytest.mark.usefixtures("with_wheel") +def test_install_report_vcs_and_wheel_cache( + script: PipTestEnvironment, tmp_path: Path +) -> None: + """Test report for VCS reference, and interactions with the wheel cache.""" + cache_dir = tmp_path / "cache" + report_path = tmp_path / "report.json" + script.pip( + "install", + "git+https://github.com/pypa/pip-test-package" + "@5547fa909e83df8bd743d3978d6667497983a4b7", + "--cache-dir", + str(cache_dir), + "--report", + str(report_path), + ) + report = json.loads(report_path.read_text()) + assert len(report["install"]) == 1 + pip_test_package_report = report["install"]["pip-test-package"] + assert pip_test_package_report["is_direct"] is True + assert pip_test_package_report["requested"] is True + assert ( + pip_test_package_report["download_info"]["url"] + == "https://github.com/pypa/pip-test-package" + ) + assert pip_test_package_report["download_info"]["vcs_info"]["vcs"] == "git" + assert ( + pip_test_package_report["download_info"]["vcs_info"]["commit_id"] + == "5547fa909e83df8bd743d3978d6667497983a4b7" + ) + # Now do it again to make sure the cache is used and that the report still contains + # the original VCS url. + report_path.unlink() + result = script.pip( + "install", + "pip-test-package @ git+https://github.com/pypa/pip-test-package" + "@5547fa909e83df8bd743d3978d6667497983a4b7", + "--ignore-installed", + "--cache-dir", + str(cache_dir), + "--report", + str(report_path), + ) + assert "Using cached pip_test_package" in result.stdout + report = json.loads(report_path.read_text()) + assert len(report["install"]) == 1 + pip_test_package_report = report["install"]["pip-test-package"] + assert pip_test_package_report["is_direct"] is True + assert pip_test_package_report["requested"] is True + assert ( + pip_test_package_report["download_info"]["url"] + == "https://github.com/pypa/pip-test-package" + ) + assert pip_test_package_report["download_info"]["vcs_info"]["vcs"] == "git" + assert ( + pip_test_package_report["download_info"]["vcs_info"]["commit_id"] + == "5547fa909e83df8bd743d3978d6667497983a4b7" + ) + + +@pytest.mark.network +@pytest.mark.usefixtures("with_wheel") +def test_install_report_vcs_editable( + script: PipTestEnvironment, tmp_path: Path +) -> None: + """Test report remote editable.""" + report_path = tmp_path / "report.json" + script.pip( + "install", + "--editable", + "git+https://github.com/pypa/pip-test-package" + "@5547fa909e83df8bd743d3978d6667497983a4b7" + "#egg=pip-test-package", + "--report", + str(report_path), + ) + report = json.loads(report_path.read_text()) + assert len(report["install"]) == 1 + pip_test_package_report = report["install"]["pip-test-package"] + assert pip_test_package_report["is_direct"] is True + assert pip_test_package_report["download_info"]["url"].startswith("file://") + assert pip_test_package_report["download_info"]["url"].endswith( + "/src/pip-test-package" + ) + assert pip_test_package_report["download_info"]["dir_info"]["editable"] is True From fbb8f6509dc3d99bc1ddbb01b90f1e17cdbe1183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 3 Jun 2022 22:17:54 +0200 Subject: [PATCH 02/14] Add environment to the installation report --- src/pip/_internal/models/installation_report.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/models/installation_report.py b/src/pip/_internal/models/installation_report.py index 5ae87be567f..81f0df94191 100644 --- a/src/pip/_internal/models/installation_report.py +++ b/src/pip/_internal/models/installation_report.py @@ -1,5 +1,7 @@ from typing import Any, Dict, Sequence +from pip._vendor.packaging.markers import default_environment + from pip._internal.req.req_install import InstallRequirement @@ -35,5 +37,6 @@ def to_dict(self) -> Dict[str, Any]: "install": { ireq.get_dist().metadata["Name"]: self._install_req_to_dict(ireq) for ireq in self._install_requirements - } + }, + "environment": default_environment(), } From d6685d09cf4eaf410e9c7c1c1609ba7095590e7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 3 Jun 2022 22:36:57 +0200 Subject: [PATCH 03/14] Test that the install report has requires_dist This is important for legacy setuptools distributions that do not have Requires-Dist in PKG-INFO. --- tests/functional/test_install_report.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/test_install_report.py b/tests/functional/test_install_report.py index f88f20d14ad..7a1df9c101b 100644 --- a/tests/functional/test_install_report.py +++ b/tests/functional/test_install_report.py @@ -85,6 +85,7 @@ def test_install_report_index(script: PipTestEnvironment, tmp_path: Path) -> Non paste_report["download_info"]["archive_info"]["hash"] == "sha256=11645842ba8ec986ae8cfbe4c6cacff5c35f0f4527abf4f5581ae8b4ad49c0b6" ) + assert "requires_dist" in paste_report["metadata"] @pytest.mark.network From 2c84a1c16d82e7374f8a73ea6a4408d659eb2b58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 4 Jun 2022 12:56:38 +0200 Subject: [PATCH 04/14] Add requested_extras to installation report --- src/pip/_internal/models/installation_report.py | 3 +++ tests/functional/test_install_report.py | 1 + 2 files changed, 4 insertions(+) diff --git a/src/pip/_internal/models/installation_report.py b/src/pip/_internal/models/installation_report.py index 81f0df94191..1904c24b3f7 100644 --- a/src/pip/_internal/models/installation_report.py +++ b/src/pip/_internal/models/installation_report.py @@ -30,6 +30,9 @@ def _install_req_to_dict(cls, ireq: InstallRequirement) -> Dict[str, Any]: # https://www.python.org/dev/peps/pep-0566/#json-compatible-metadata "metadata": ireq.get_dist().metadata_dict, } + if ireq.user_supplied and ireq.extras: + # For top level requirements, the list of requested extras, if any. + res["requested_extras"] = list(sorted(ireq.extras)) return res def to_dict(self) -> Dict[str, Any]: diff --git a/tests/functional/test_install_report.py b/tests/functional/test_install_report.py index 7a1df9c101b..c0e333ca297 100644 --- a/tests/functional/test_install_report.py +++ b/tests/functional/test_install_report.py @@ -85,6 +85,7 @@ def test_install_report_index(script: PipTestEnvironment, tmp_path: Path) -> Non paste_report["download_info"]["archive_info"]["hash"] == "sha256=11645842ba8ec986ae8cfbe4c6cacff5c35f0f4527abf4f5581ae8b4ad49c0b6" ) + assert paste_report["requested_extras"] == ["openid"] assert "requires_dist" in paste_report["metadata"] From d32a62b3df2bdb861a5ef69900db46b631c3a153 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 4 Jun 2022 12:58:22 +0200 Subject: [PATCH 05/14] Use canonical names as keys in installation report --- src/pip/_internal/models/installation_report.py | 5 ++++- tests/functional/test_install_report.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/models/installation_report.py b/src/pip/_internal/models/installation_report.py index 1904c24b3f7..bff8819aa42 100644 --- a/src/pip/_internal/models/installation_report.py +++ b/src/pip/_internal/models/installation_report.py @@ -1,6 +1,7 @@ from typing import Any, Dict, Sequence from pip._vendor.packaging.markers import default_environment +from pip._vendor.packaging.utils import canonicalize_name from pip._internal.req.req_install import InstallRequirement @@ -38,7 +39,9 @@ def _install_req_to_dict(cls, ireq: InstallRequirement) -> Dict[str, Any]: def to_dict(self) -> Dict[str, Any]: return { "install": { - ireq.get_dist().metadata["Name"]: self._install_req_to_dict(ireq) + canonicalize_name(ireq.metadata["Name"]): self._install_req_to_dict( + ireq + ) for ireq in self._install_requirements }, "environment": default_environment(), diff --git a/tests/functional/test_install_report.py b/tests/functional/test_install_report.py index c0e333ca297..e2ed7d22c60 100644 --- a/tests/functional/test_install_report.py +++ b/tests/functional/test_install_report.py @@ -74,9 +74,9 @@ def test_install_report_index(script: PipTestEnvironment, tmp_path: Path) -> Non ) report = json.loads(report_path.read_text()) assert len(report["install"]) == 2 - assert report["install"]["Paste"]["requested"] is True + assert report["install"]["paste"]["requested"] is True assert report["install"]["python-openid"]["requested"] is False - paste_report = report["install"]["Paste"] + paste_report = report["install"]["paste"] assert paste_report["download_info"]["url"].startswith( "https://files.pythonhosted.org/" ) From 7fdff17543b2b9cb60aba1b64d954fadfbc66c10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 26 Jun 2022 17:38:57 +0200 Subject: [PATCH 06/14] install report: add suport for stdout output --- src/pip/_internal/commands/install.py | 11 ++++++++--- tests/functional/test_install_report.py | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 67072e44c74..b6d415ad40e 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -4,6 +4,7 @@ import os import shutil import site +import sys from optparse import SUPPRESS_HELP, Values from typing import Iterable, List, Optional @@ -261,7 +262,8 @@ def add_options(self) -> None: "Generate a JSON file describing what pip did to install " "the provided requirements. " "Can be used in combination with --dry-run and --ignore-installed " - "to 'resolve' the requirements." + "to 'resolve' the requirements. " + "When - is used as file name it writes to stdout." ), ) @@ -370,8 +372,11 @@ def run(self, options: Values, args: List[str]) -> int: if options.json_report_file: report = InstallationReport(requirement_set.requirements_to_install) - with open(options.json_report_file, "w", encoding="utf-8") as f: - json.dump(report.to_dict(), f) + if options.json_report_file == "-": + json.dump(report.to_dict(), sys.stdout, indent=2, ensure_ascii=True) + else: + with open(options.json_report_file, "w", encoding="utf-8") as f: + json.dump(report.to_dict(), f, indent=2, ensure_ascii=False) if options.dry_run: would_install_items = sorted( diff --git a/tests/functional/test_install_report.py b/tests/functional/test_install_report.py index e2ed7d22c60..d6204d383ff 100644 --- a/tests/functional/test_install_report.py +++ b/tests/functional/test_install_report.py @@ -175,3 +175,24 @@ def test_install_report_vcs_editable( "/src/pip-test-package" ) assert pip_test_package_report["download_info"]["dir_info"]["editable"] is True + + +@pytest.mark.usefixtures("with_wheel") +def test_install_report_to_stdout( + script: PipTestEnvironment, shared_data: TestData +) -> None: + result = script.pip( + "install", + "simplewheel", + "--quiet", + "--dry-run", + "--no-index", + "--find-links", + str(shared_data.root / "packages/"), + "--report", + "-", + ) + assert not result.stderr + report = json.loads(result.stdout) + assert "install" in report + assert len(report["install"]) == 1 From 6eab8e446fa6a418e637930ead9032fce863c1e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Mon, 27 Jun 2022 17:15:49 +0200 Subject: [PATCH 07/14] install report: add todo wrt environment markers --- src/pip/_internal/models/installation_report.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pip/_internal/models/installation_report.py b/src/pip/_internal/models/installation_report.py index bff8819aa42..d5621e69c1e 100644 --- a/src/pip/_internal/models/installation_report.py +++ b/src/pip/_internal/models/installation_report.py @@ -44,5 +44,11 @@ def to_dict(self) -> Dict[str, Any]: ) for ireq in self._install_requirements }, + # https://peps.python.org/pep-0508/#environment-markers + # TODO: currently, the resolver uses the default environment to evaluate + # environment markers, so that is what we report here. In the future, it + # should also take into account options such as --python-version or + # --platform, perhaps under the form of an environment_override field? + # https://github.com/pypa/pip/issues/11198 "environment": default_environment(), } From 0198eae492c423c1c86939295587b7830e7ecb24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 1 Jul 2022 18:18:34 +0200 Subject: [PATCH 08/14] install report: docs --- docs/html/cli/pip_install.rst | 12 ++ docs/html/reference/index.md | 1 + docs/html/reference/installation-report.md | 152 +++++++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 docs/html/reference/installation-report.md diff --git a/docs/html/cli/pip_install.rst b/docs/html/cli/pip_install.rst index 408ef25ff96..384d393fdd7 100644 --- a/docs/html/cli/pip_install.rst +++ b/docs/html/cli/pip_install.rst @@ -79,6 +79,18 @@ for an exception regarding pre-release versions). Where more than one source of the chosen version is available, it is assumed that any source is acceptable (as otherwise the versions would differ). +Obtaining information about what was installed +---------------------------------------------- + +The install command has a ``--report`` option that will generate a JSON report of what +pip has installed. In combination with the ``--dry-run`` and ``--ignore-installed`` it +can be used to *resolve* a set of requirements without actually installing them. + +The report can be written to a file, or to standard output (using ``--report -`` in +combination with ``--quiet``). + +The format of the JSON report is described in :doc:`../reference/installation-report`. + Installation Order ------------------ diff --git a/docs/html/reference/index.md b/docs/html/reference/index.md index 855dc79b37a..749e88096aa 100644 --- a/docs/html/reference/index.md +++ b/docs/html/reference/index.md @@ -9,4 +9,5 @@ interoperability standards that pip utilises/implements. build-system/index requirement-specifiers requirements-file-format +installation-report ``` diff --git a/docs/html/reference/installation-report.md b/docs/html/reference/installation-report.md new file mode 100644 index 00000000000..0e46658e395 --- /dev/null +++ b/docs/html/reference/installation-report.md @@ -0,0 +1,152 @@ +# Installation Report + +The `--report` option of the pip install command produces a detailed JSON report of what +it did install (or what it would have installed, if used with the `--dry-run` option). + +## Specification + +The report is a JSON object with the following properties: + +- `install`: an object where the properties are the canonicalized names of the + distribution packages (to be) installed and the values are of type + `InstallationReportItem` (see below). +- `environment`: an object describing the environment where the installation report was + generated. See [PEP 508 environment + markers](https://peps.python.org/pep-0508/#environment-markers) for more information. + Values have a string type. + +An `InstallationReportItem` is an object describing a (to be) installed distribution +package with the following properties: + +- `metadata`: the metadata of the distribution, converted to a JSON object according to + the [PEP 566 + transformation](https://www.python.org/dev/peps/pep-0566/#json-compatible-metadata). + +- `is_direct`: `true` if the requirement was provided as, or constrained to, a direct + URL reference. `false` if the requirements was provided as a name and version + specifier. + +- `download_info`: Information about the artifact (to be) downloaded for installation, + using the [direct + URL](https://packaging.python.org/en/latest/specifications/direct-url/) data + structure. When `is_direct` is `true`, this field is the same as the `direct_url.json` + metadata, otherwise it represents the URL of the artifact obtained from the index or + `--find-links`. +- `requested`: `true` if the requirement was explicitly provided by the user, either + directely via a command line argument or indirectly via a requirements file. `false` + if the requirement was installed as a dependency of another requirement. + +- `requested_extras`: extras requested by the user. This field is only present when the + `requested` field is true. + +## Example + +The following command: + +```console +pip install \ + --ignore-installed --dry-run --quiet \ + --report - \ + "pydantic>=1.9" git+https://github.com/pypa/packaging@main +``` + +will produce an output similar to this (metadata abriged for brevity): + +```json +{ + "install": { + "pydantic": { + "download_info": { + "url": "https://files.pythonhosted.org/packages/a4/0c/fbaa7319dcb5eecd3484686eb5a5c5702a6445adb566f01aee6de3369bc4/pydantic-1.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "archive_info": { + "hash": "sha256=18f3e912f9ad1bdec27fb06b8198a2ccc32f201e24174cec1b3424dda605a310" + } + }, + "is_direct": false, + "requested": true, + "metadata": { + "name": "pydantic", + "version": "1.9.1", + "requires_dist": [ + "typing-extensions (>=3.7.4.3)", + "dataclasses (>=0.6) ; python_version < \"3.7\"", + "python-dotenv (>=0.10.4) ; extra == 'dotenv'", + "email-validator (>=1.0.3) ; extra == 'email'" + ], + "requires_python": ">=3.6.1", + "provides_extra": [ + "dotenv", + "email" + ] + } + }, + "packaging": { + "download_info": { + "url": "https://github.com/pypa/packaging", + "vcs_info": { + "vcs": "git", + "requested_revision": "main", + "commit_id": "4f42225e91a0be634625c09e84dd29ea82b85e27" + } + }, + "is_direct": true, + "requested": true, + "metadata": { + "name": "packaging", + "version": "21.4.dev0", + "requires_dist": [ + "pyparsing (!=3.0.5,>=2.0.2)" + ], + "requires_python": ">=3.7" + } + }, + "pyparsing": { + "download_info": { + "url": "https://files.pythonhosted.org/packages/6c/10/a7d0fa5baea8fe7b50f448ab742f26f52b80bfca85ac2be9d35cdd9a3246/pyparsing-3.0.9-py3-none-any.whl", + "archive_info": { + "hash": "sha256=5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc" + } + }, + "is_direct": false, + "requested": false, + "metadata": { + "name": "pyparsing", + "version": "3.0.9", + "requires_dist": [ + "railroad-diagrams ; extra == \"diagrams\"", + "jinja2 ; extra == \"diagrams\"" + ], + "requires_python": ">=3.6.8" + } + }, + "typing-extensions": { + "download_info": { + "url": "https://files.pythonhosted.org/packages/75/e1/932e06004039dd670c9d5e1df0cd606bf46e29a28e65d5bb28e894ea29c9/typing_extensions-4.2.0-py3-none-any.whl", + "archive_info": { + "hash": "sha256=6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708" + } + }, + "is_direct": false, + "requested": false, + "metadata": { + "name": "typing_extensions", + "version": "4.2.0", + "requires_python": ">=3.7" + } + } + }, + "environment": { + "implementation_name": "cpython", + "implementation_version": "3.10.5", + "os_name": "posix", + "platform_machine": "x86_64", + "platform_release": "5.13-generic", + "platform_system": "Linux", + "platform_version": "...", + "python_full_version": "3.10.5", + "platform_python_implementation": "CPython", + "python_version": "3.10", + "sys_platform": "linux" + } +} +``` From 2841f4666930eb6c5eb6bca5de1824281ca04dea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 2 Jul 2022 12:28:31 +0200 Subject: [PATCH 09/14] install_report: add pip_version --- docs/html/reference/installation-report.md | 2 ++ src/pip/_internal/models/installation_report.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/html/reference/installation-report.md b/docs/html/reference/installation-report.md index 0e46658e395..bdee290e8bc 100644 --- a/docs/html/reference/installation-report.md +++ b/docs/html/reference/installation-report.md @@ -7,6 +7,7 @@ it did install (or what it would have installed, if used with the `--dry-run` op The report is a JSON object with the following properties: +- `pip_version`: a string with the version of pip used to produce the report. - `install`: an object where the properties are the canonicalized names of the distribution packages (to be) installed and the values are of type `InstallationReportItem` (see below). @@ -54,6 +55,7 @@ will produce an output similar to this (metadata abriged for brevity): ```json { + "pip_version": "22.2", "install": { "pydantic": { "download_info": { diff --git a/src/pip/_internal/models/installation_report.py b/src/pip/_internal/models/installation_report.py index d5621e69c1e..dde8ab20135 100644 --- a/src/pip/_internal/models/installation_report.py +++ b/src/pip/_internal/models/installation_report.py @@ -3,6 +3,7 @@ from pip._vendor.packaging.markers import default_environment from pip._vendor.packaging.utils import canonicalize_name +from pip import __version__ from pip._internal.req.req_install import InstallRequirement @@ -38,6 +39,7 @@ def _install_req_to_dict(cls, ireq: InstallRequirement) -> Dict[str, Any]: def to_dict(self) -> Dict[str, Any]: return { + "pip_version": __version__, "install": { canonicalize_name(ireq.metadata["Name"]): self._install_req_to_dict( ireq From 1fbfdc44233486299db4d4364cf8cc8ef98ceacb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 2 Jul 2022 17:53:37 +0200 Subject: [PATCH 10/14] install report: add version field Also, affirm the experimental status of the feature. --- docs/html/reference/installation-report.md | 11 +++++++++++ news/53.feature.rst | 6 +++--- src/pip/_internal/models/installation_report.py | 1 + 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/html/reference/installation-report.md b/docs/html/reference/installation-report.md index bdee290e8bc..dd5ce0304da 100644 --- a/docs/html/reference/installation-report.md +++ b/docs/html/reference/installation-report.md @@ -7,6 +7,16 @@ it did install (or what it would have installed, if used with the `--dry-run` op The report is a JSON object with the following properties: +- `version`: the string `0`, denoting that the installation report is an experimental + feature. This value will change to `1`, when the feature is deemed stable after + gathering user feedback (likely in pip 22.3 or 23.0). Backward incompatible changes + may be introduced in version `1` without notice. After that, it will change only if + and when backward incompatible changes are introduced, such as removing mandatory + fields or changing the semantics or data type of existing fields. The introduction of + backward incompatible changes will follow the usual pip processes such as the + deprecation cycle or feature flags. Tools must check this field to ensure they support + the corresponding version. + - `pip_version`: a string with the version of pip used to produce the report. - `install`: an object where the properties are the canonicalized names of the distribution packages (to be) installed and the values are of type @@ -55,6 +65,7 @@ will produce an output similar to this (metadata abriged for brevity): ```json { + "version": "0", "pip_version": "22.2", "install": { "pydantic": { diff --git a/news/53.feature.rst b/news/53.feature.rst index 707718efb38..a0525602446 100644 --- a/news/53.feature.rst +++ b/news/53.feature.rst @@ -1,3 +1,3 @@ -Add ``--report`` to the install command to generate a json report of what was installed. -In combination with ``--dry-run`` and ``--ignore-installed`` it can be used to resolve -the requirements. +Add an experimental ``--report`` option to the install command to generate a JSON report +of what was installed. In combination with ``--dry-run`` and ``--ignore-installed`` it +can be used to resolve the requirements. diff --git a/src/pip/_internal/models/installation_report.py b/src/pip/_internal/models/installation_report.py index dde8ab20135..4678d07eb3a 100644 --- a/src/pip/_internal/models/installation_report.py +++ b/src/pip/_internal/models/installation_report.py @@ -39,6 +39,7 @@ def _install_req_to_dict(cls, ireq: InstallRequirement) -> Dict[str, Any]: def to_dict(self) -> Dict[str, Any]: return { + "version": "0", "pip_version": __version__, "install": { canonicalize_name(ireq.metadata["Name"]): self._install_req_to_dict( From 652963548eefc95ff799b166f361143ee15addad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Tue, 5 Jul 2022 10:40:07 +0200 Subject: [PATCH 11/14] install report: added note about download_info.archive_info.hash --- docs/html/reference/installation-report.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/html/reference/installation-report.md b/docs/html/reference/installation-report.md index dd5ce0304da..005dc52d77f 100644 --- a/docs/html/reference/installation-report.md +++ b/docs/html/reference/installation-report.md @@ -43,6 +43,14 @@ package with the following properties: structure. When `is_direct` is `true`, this field is the same as the `direct_url.json` metadata, otherwise it represents the URL of the artifact obtained from the index or `--find-links`. + + ```{note} + For source archives, `download_info.archive_info.hash` may + be absent when the requirement was installed from the wheel cache + and the cache entry was populated by an older pip version that did not + record the origin URL of the downloaded artifact. + ``` + - `requested`: `true` if the requirement was explicitly provided by the user, either directely via a command line argument or indirectly via a requirements file. `false` if the requirement was installed as a dependency of another requirement. From e41b13424e81e62ba65bcfce0a834c44a09f6ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 10 Jul 2022 11:55:06 +0200 Subject: [PATCH 12/14] install report: use array instead of dict for install field --- docs/html/reference/installation-report.md | 21 ++++++++------- .../_internal/models/installation_report.py | 10 +++---- tests/functional/test_install_report.py | 26 ++++++++++++------- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/docs/html/reference/installation-report.md b/docs/html/reference/installation-report.md index 005dc52d77f..0be6be75394 100644 --- a/docs/html/reference/installation-report.md +++ b/docs/html/reference/installation-report.md @@ -18,14 +18,17 @@ The report is a JSON object with the following properties: the corresponding version. - `pip_version`: a string with the version of pip used to produce the report. -- `install`: an object where the properties are the canonicalized names of the - distribution packages (to be) installed and the values are of type - `InstallationReportItem` (see below). + +- `install`: an array of [InstallationReportItem](InstallationReportItem) representing + the distribution packages (to be) installed. + - `environment`: an object describing the environment where the installation report was generated. See [PEP 508 environment markers](https://peps.python.org/pep-0508/#environment-markers) for more information. Values have a string type. +(InstallationReportItem)= + An `InstallationReportItem` is an object describing a (to be) installed distribution package with the following properties: @@ -75,8 +78,8 @@ will produce an output similar to this (metadata abriged for brevity): { "version": "0", "pip_version": "22.2", - "install": { - "pydantic": { + "install": [ + { "download_info": { "url": "https://files.pythonhosted.org/packages/a4/0c/fbaa7319dcb5eecd3484686eb5a5c5702a6445adb566f01aee6de3369bc4/pydantic-1.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "archive_info": { @@ -101,7 +104,7 @@ will produce an output similar to this (metadata abriged for brevity): ] } }, - "packaging": { + { "download_info": { "url": "https://github.com/pypa/packaging", "vcs_info": { @@ -121,7 +124,7 @@ will produce an output similar to this (metadata abriged for brevity): "requires_python": ">=3.7" } }, - "pyparsing": { + { "download_info": { "url": "https://files.pythonhosted.org/packages/6c/10/a7d0fa5baea8fe7b50f448ab742f26f52b80bfca85ac2be9d35cdd9a3246/pyparsing-3.0.9-py3-none-any.whl", "archive_info": { @@ -140,7 +143,7 @@ will produce an output similar to this (metadata abriged for brevity): "requires_python": ">=3.6.8" } }, - "typing-extensions": { + { "download_info": { "url": "https://files.pythonhosted.org/packages/75/e1/932e06004039dd670c9d5e1df0cd606bf46e29a28e65d5bb28e894ea29c9/typing_extensions-4.2.0-py3-none-any.whl", "archive_info": { @@ -155,7 +158,7 @@ will produce an output similar to this (metadata abriged for brevity): "requires_python": ">=3.7" } } - }, + ], "environment": { "implementation_name": "cpython", "implementation_version": "3.10.5", diff --git a/src/pip/_internal/models/installation_report.py b/src/pip/_internal/models/installation_report.py index 4678d07eb3a..965f0952371 100644 --- a/src/pip/_internal/models/installation_report.py +++ b/src/pip/_internal/models/installation_report.py @@ -1,7 +1,6 @@ from typing import Any, Dict, Sequence from pip._vendor.packaging.markers import default_environment -from pip._vendor.packaging.utils import canonicalize_name from pip import __version__ from pip._internal.req.req_install import InstallRequirement @@ -41,12 +40,9 @@ def to_dict(self) -> Dict[str, Any]: return { "version": "0", "pip_version": __version__, - "install": { - canonicalize_name(ireq.metadata["Name"]): self._install_req_to_dict( - ireq - ) - for ireq in self._install_requirements - }, + "install": [ + self._install_req_to_dict(ireq) for ireq in self._install_requirements + ], # https://peps.python.org/pep-0508/#environment-markers # TODO: currently, the resolver uses the default environment to evaluate # environment markers, so that is what we report here. In the future, it diff --git a/tests/functional/test_install_report.py b/tests/functional/test_install_report.py index d6204d383ff..80516577791 100644 --- a/tests/functional/test_install_report.py +++ b/tests/functional/test_install_report.py @@ -1,11 +1,17 @@ import json from pathlib import Path +from typing import Any, Dict import pytest +from packaging.utils import canonicalize_name from ..lib import PipTestEnvironment, TestData +def _install_dict(report: Dict[str, Any]) -> Dict[str, Any]: + return {canonicalize_name(i["metadata"]["name"]): i for i in report["install"]} + + @pytest.mark.usefixtures("with_wheel") def test_install_report_basic( script: PipTestEnvironment, shared_data: TestData, tmp_path: Path @@ -24,8 +30,7 @@ def test_install_report_basic( report = json.loads(report_path.read_text()) assert "install" in report assert len(report["install"]) == 1 - assert "simplewheel" in report["install"] - simplewheel_report = report["install"]["simplewheel"] + simplewheel_report = _install_dict(report)["simplewheel"] assert simplewheel_report["metadata"]["name"] == "simplewheel" assert simplewheel_report["requested"] is True assert simplewheel_report["is_direct"] is False @@ -56,8 +61,8 @@ def test_install_report_dep( ) report = json.loads(report_path.read_text()) assert len(report["install"]) == 2 - assert report["install"]["require-simple"]["requested"] is True - assert report["install"]["simple"]["requested"] is False + assert _install_dict(report)["require-simple"]["requested"] is True + assert _install_dict(report)["simple"]["requested"] is False @pytest.mark.network @@ -74,9 +79,10 @@ def test_install_report_index(script: PipTestEnvironment, tmp_path: Path) -> Non ) report = json.loads(report_path.read_text()) assert len(report["install"]) == 2 - assert report["install"]["paste"]["requested"] is True - assert report["install"]["python-openid"]["requested"] is False - paste_report = report["install"]["paste"] + install_dict = _install_dict(report) + assert install_dict["paste"]["requested"] is True + assert install_dict["python-openid"]["requested"] is False + paste_report = install_dict["paste"] assert paste_report["download_info"]["url"].startswith( "https://files.pythonhosted.org/" ) @@ -108,7 +114,7 @@ def test_install_report_vcs_and_wheel_cache( ) report = json.loads(report_path.read_text()) assert len(report["install"]) == 1 - pip_test_package_report = report["install"]["pip-test-package"] + pip_test_package_report = report["install"][0] assert pip_test_package_report["is_direct"] is True assert pip_test_package_report["requested"] is True assert ( @@ -136,7 +142,7 @@ def test_install_report_vcs_and_wheel_cache( assert "Using cached pip_test_package" in result.stdout report = json.loads(report_path.read_text()) assert len(report["install"]) == 1 - pip_test_package_report = report["install"]["pip-test-package"] + pip_test_package_report = report["install"][0] assert pip_test_package_report["is_direct"] is True assert pip_test_package_report["requested"] is True assert ( @@ -168,7 +174,7 @@ def test_install_report_vcs_editable( ) report = json.loads(report_path.read_text()) assert len(report["install"]) == 1 - pip_test_package_report = report["install"]["pip-test-package"] + pip_test_package_report = report["install"][0] assert pip_test_package_report["is_direct"] is True assert pip_test_package_report["download_info"]["url"].startswith("file://") assert pip_test_package_report["download_info"]["url"].endswith( From 16029fbcc95c68d50c4d8f8ddbcd94db5195b687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Wed, 13 Jul 2022 10:49:05 +0200 Subject: [PATCH 13/14] install report: use rich to print json to stdout --- src/pip/_internal/commands/install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index b6d415ad40e..f399eca78fb 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -4,11 +4,11 @@ import os import shutil import site -import sys from optparse import SUPPRESS_HELP, Values from typing import Iterable, List, Optional from pip._vendor.packaging.utils import canonicalize_name +from pip._vendor.rich import print_json from pip._internal.cache import WheelCache from pip._internal.cli import cmdoptions @@ -373,7 +373,7 @@ def run(self, options: Values, args: List[str]) -> int: if options.json_report_file: report = InstallationReport(requirement_set.requirements_to_install) if options.json_report_file == "-": - json.dump(report.to_dict(), sys.stdout, indent=2, ensure_ascii=True) + print_json(data=report.to_dict()) else: with open(options.json_report_file, "w", encoding="utf-8") as f: json.dump(report.to_dict(), f, indent=2, ensure_ascii=False) From 074c6b5cadfef0fe058bc8cd67d36d39b0a251c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 14 Jul 2022 17:49:49 +0200 Subject: [PATCH 14/14] install report: added experimental status warning --- src/pip/_internal/commands/install.py | 6 ++++++ tests/functional/test_install_report.py | 12 +++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index f399eca78fb..29907645c81 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -371,6 +371,12 @@ def run(self, options: Values, args: List[str]) -> int: ) if options.json_report_file: + logger.warning( + "--report is currently an experimental option. " + "The output format may change in a future release " + "without prior warning." + ) + report = InstallationReport(requirement_set.requirements_to_install) if options.json_report_file == "-": print_json(data=report.to_dict()) diff --git a/tests/functional/test_install_report.py b/tests/functional/test_install_report.py index 80516577791..b61fd89c69f 100644 --- a/tests/functional/test_install_report.py +++ b/tests/functional/test_install_report.py @@ -26,6 +26,7 @@ def test_install_report_basic( str(shared_data.root / "packages/"), "--report", str(report_path), + allow_stderr_warning=True, ) report = json.loads(report_path.read_text()) assert "install" in report @@ -58,6 +59,7 @@ def test_install_report_dep( str(shared_data.root / "packages/"), "--report", str(report_path), + allow_stderr_warning=True, ) report = json.loads(report_path.read_text()) assert len(report["install"]) == 2 @@ -76,6 +78,7 @@ def test_install_report_index(script: PipTestEnvironment, tmp_path: Path) -> Non "Paste[openid]==1.7.5.1", "--report", str(report_path), + allow_stderr_warning=True, ) report = json.loads(report_path.read_text()) assert len(report["install"]) == 2 @@ -111,6 +114,7 @@ def test_install_report_vcs_and_wheel_cache( str(cache_dir), "--report", str(report_path), + allow_stderr_warning=True, ) report = json.loads(report_path.read_text()) assert len(report["install"]) == 1 @@ -138,6 +142,7 @@ def test_install_report_vcs_and_wheel_cache( str(cache_dir), "--report", str(report_path), + allow_stderr_warning=True, ) assert "Using cached pip_test_package" in result.stdout report = json.loads(report_path.read_text()) @@ -171,6 +176,7 @@ def test_install_report_vcs_editable( "#egg=pip-test-package", "--report", str(report_path), + allow_stderr_warning=True, ) report = json.loads(report_path.read_text()) assert len(report["install"]) == 1 @@ -197,8 +203,12 @@ def test_install_report_to_stdout( str(shared_data.root / "packages/"), "--report", "-", + allow_stderr_warning=True, + ) + assert result.stderr == ( + "WARNING: --report is currently an experimental option. " + "The output format may change in a future release without prior warning.\n" ) - assert not result.stderr report = json.loads(result.stdout) assert "install" in report assert len(report["install"]) == 1