Merge pull request #557 from suutari/dont-unpack-for-hashing

Hash packages without unpacking (Fixes #512 and #544)

CHANGELOG.md

@@ -14,6 +14,7 @@ when `--allow-unsafe` was not set. ([#517](https://github.com/jazzband/pip-tools
 - Fixed bug where pkg-resources would be removed by pip-sync in Ubuntu. ([#555](https://github.com/jazzband/pip-tools/pull/555)). Thanks @cemsbr
 - Fixed bug where the resolver would sometime not stabilize on requirements specifying extras. ([#566](https://github.com/jazzband/pip-tools/pull/566)). Thanks @vphilippon
 - Fixed an unicode encoding error when distribution package contains non-ASCII file names ([#567](https://github.com/jazzband/pip-tools/pull/567)). Thanks @suutari
+- Fixed package hashing doing unnecessary unpacking
 
 # 1.9.0 (2017-04-12)
 

piptools/repositories/pypi.py

@@ -4,9 +4,10 @@
 
 import hashlib
 import os
+from contextlib import contextmanager
 from shutil import rmtree
 
-from pip.download import unpack_url
+from pip.download import is_file_url, url_to_path
 from pip.index import PackageFinder
 from pip.req.req_set import RequirementSet
 from pip.wheel import Wheel
@@ -194,18 +195,38 @@ def get_hashes(self, ireq):
         }
 
     def _get_file_hash(self, location):
-        with TemporaryDirectory() as tmpdir:
-            unpack_url(
-                location, self.build_dir,
-                download_dir=tmpdir, only_download=True, session=self.session
-            )
-            files = os.listdir(tmpdir)
-            assert len(files) == 1
-            filename = os.path.abspath(os.path.join(tmpdir, files[0]))
-
-            h = hashlib.new(FAVORITE_HASH)
-            with open(filename, "rb") as fp:
-                for chunk in iter(lambda: fp.read(8096), b""):
-                    h.update(chunk)
-
+        h = hashlib.new(FAVORITE_HASH)
+        with open_local_or_remote_file(location, self.session) as fp:
+            for chunk in iter(lambda: fp.read(8096), b""):
+                h.update(chunk)
         return ":".join([FAVORITE_HASH, h.hexdigest()])
+
+
+@contextmanager
+def open_local_or_remote_file(link, session):
+    """
+    Open local or remote file for reading.
+
+    :type link: pip.index.Link
+    :type session: requests.Session
+    :raises ValueError: If link points to a local directory.
+    :return: a context manager to the opened file-like object
+    """
+    url = link.url_without_fragment
+
+    if is_file_url(link):
+        # Local URL
+        local_path = url_to_path(url)
+        if os.path.isdir(local_path):
+            raise ValueError("Cannot open directory for read: {}".format(url))
+        else:
+            with open(local_path, 'rb') as local_file:
+                yield local_file
+    else:
+        # Remote URL
+        headers = {"Accept-Encoding": "identity"}
+        response = session.get(url, headers=headers, stream=True)
+        try:
+            yield response.raw
+        finally:
+            response.close()

tests/test_repository_pypi.py

@@ -53,3 +53,12 @@ def test_generate_hashes_all_platforms(from_line):
     repository = PyPIRepository(pip_options, session)
     ireq = from_line('cffi==1.9.1')
     assert repository.get_hashes(ireq) == expected
+
+
+def test_generate_hashes_without_interfering_with_each_other(from_line):
+    pip_command = get_pip_command()
+    pip_options, _ = pip_command.parse_args([])
+    session = pip_command._build_session(pip_options)
+    repository = PyPIRepository(pip_options, session)
+    repository.get_hashes(from_line('cffi==1.9.1'))
+    repository.get_hashes(from_line('matplotlib==2.0.2'))