From c947d00882ef6f12cbae1bfb71bc2677c4d2a82a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Tue, 25 Aug 2020 22:06:44 +0700 Subject: [PATCH 1/2] [fast-deps] Check download directory before making requests --- news/8804.feature | 3 +++ src/pip/_internal/operations/prepare.py | 29 +++++++++++++------------ 2 files changed, 18 insertions(+), 14 deletions(-) create mode 100644 news/8804.feature diff --git a/news/8804.feature b/news/8804.feature new file mode 100644 index 00000000000..a29333342e5 --- /dev/null +++ b/news/8804.feature @@ -0,0 +1,3 @@ +Check the download directory for existing wheels to possibly avoid +fetching metadata when the ``fast-deps`` feature is used with +``pip wheel`` and ``pip download``. diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 0a254052654..09540edce74 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -486,26 +486,27 @@ def prepare_linked_requirement(self, req, parallel_builds=False): link = req.link self._log_preparing_link(req) with indent_log(): - wheel_dist = self._fetch_metadata_using_lazy_wheel(link) - if wheel_dist is not None: - req.needs_more_preparation = True - return wheel_dist + download_dir = self._get_download_dir(req.link) + if download_dir is not None and link.is_wheel: + hashes = self._get_linked_req_hashes(req) + file_path = _check_download_dir(req.link, download_dir, hashes) + if file_path is not None: + self._downloaded[req.link.url] = file_path, None + else: + file_path = None + + if file_path is None: + wheel_dist = self._fetch_metadata_using_lazy_wheel(link) + if wheel_dist is not None: + req.needs_more_preparation = True + return wheel_dist return self._prepare_linked_requirement(req, parallel_builds) def prepare_linked_requirements_more(self, reqs, parallel_builds=False): # type: (Iterable[InstallRequirement], bool) -> None """Prepare a linked requirement more, if needed.""" reqs = [req for req in reqs if req.needs_more_preparation] - links = [] # type: List[Link] - for req in reqs: - download_dir = self._get_download_dir(req.link) - if download_dir is not None: - hashes = self._get_linked_req_hashes(req) - file_path = _check_download_dir(req.link, download_dir, hashes) - if download_dir is None or file_path is None: - links.append(req.link) - else: - self._downloaded[req.link.url] = file_path, None + links = [req.link for req in reqs] # Let's download to a temporary directory. tmpdir = TempDirectory(kind="unpack", globally_managed=True).path From 2ef8040495711c7e7e695f80a35e208f7404f879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Thu, 1 Oct 2020 20:56:01 +0700 Subject: [PATCH 2/2] Comment and rework conditionals in download dir check Co-authored-by: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com> --- src/pip/_internal/operations/prepare.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 09540edce74..fd5ff781786 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -486,20 +486,25 @@ def prepare_linked_requirement(self, req, parallel_builds=False): link = req.link self._log_preparing_link(req) with indent_log(): + # Check if the relevant file is already available + # in the download directory + file_path = None download_dir = self._get_download_dir(req.link) if download_dir is not None and link.is_wheel: hashes = self._get_linked_req_hashes(req) file_path = _check_download_dir(req.link, download_dir, hashes) - if file_path is not None: - self._downloaded[req.link.url] = file_path, None - else: - file_path = None - if file_path is None: + if file_path is not None: + # The file is already available, so mark it as downloaded + self._downloaded[req.link.url] = file_path, None + else: + # The file is not available, attempt to fetch only metadata wheel_dist = self._fetch_metadata_using_lazy_wheel(link) if wheel_dist is not None: req.needs_more_preparation = True return wheel_dist + + # None of the optimizations worked, fully prepare the requirement return self._prepare_linked_requirement(req, parallel_builds) def prepare_linked_requirements_more(self, reqs, parallel_builds=False):