From 91876f58aa6232df2f621fe218714e260c908f34 Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 12 Sep 2023 00:12:18 +0200 Subject: [PATCH] add download cache msg --- .../downloaders/caching_file_downloader.py | 13 +++++++++- .../integration/cache/download_cache_test.py | 24 ++++++++++++++----- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/conans/client/downloaders/caching_file_downloader.py b/conans/client/downloaders/caching_file_downloader.py index 6e60d1a1fc1..4b65f6b4bd8 100644 --- a/conans/client/downloaders/caching_file_downloader.py +++ b/conans/client/downloaders/caching_file_downloader.py @@ -4,11 +4,12 @@ from urllib.parse import urlparse from urllib.request import url2pathname +from conan.api.output import ConanOutput from conans.client.downloaders.file_downloader import FileDownloader from conans.client.downloaders.download_cache import DownloadCache from conans.errors import NotFoundException, ConanException, AuthenticationException, \ ForbiddenException -from conans.util.files import mkdir, set_dirty_context_manager, remove_if_dirty +from conans.util.files import mkdir, set_dirty_context_manager, remove_if_dirty, human_size class SourcesCachingDownloader: @@ -158,6 +159,7 @@ def __init__(self, requester, config, scope=None): if self._download_cache and not os.path.isabs(self._download_cache): raise ConanException("core.download:download_cache must be an absolute path") self._file_downloader = FileDownloader(requester, scope=scope) + self._scope = scope def download(self, url, file_path, auth, verify_ssl, retry, retry_wait, metadata=False): if not self._download_cache or metadata: # Metadata not cached and can be overwritten @@ -175,6 +177,15 @@ def download(self, url, file_path, auth, verify_ssl, retry, retry_wait, metadata self._file_downloader.download(url, cached_path, retry=retry, retry_wait=retry_wait, verify_ssl=verify_ssl, auth=auth, overwrite=False) + else: # Found in cache! + total_length = os.path.getsize(cached_path) + is_large_file = total_length > 10000000 # 10 MB + if is_large_file: + base_name = os.path.basename(file_path) + hs = human_size(total_length) + ConanOutput(scope=self._scope).info(f"Copying {hs} {base_name} from download " + f"cache, instead of downloading it") + # Everything good, file in the cache, just copy it to final destination mkdir(os.path.dirname(file_path)) shutil.copy2(cached_path, file_path) diff --git a/conans/test/integration/cache/download_cache_test.py b/conans/test/integration/cache/download_cache_test.py index f603f50a11b..397803dccfc 100644 --- a/conans/test/integration/cache/download_cache_test.py +++ b/conans/test/integration/cache/download_cache_test.py @@ -14,7 +14,18 @@ def test_download_skip(self): """ basic proof that enabling download_cache avoids downloading things again """ client = TestClient(default_server_user=True) - client.save({"conanfile.py": GenConanfile().with_package_file("file.txt", "content")}) + # generate large random package file + conanfile = textwrap.dedent(""" + import os + from conan import ConanFile + from conan.tools.files import save + class Pkg(ConanFile): + def package(self): + fileSizeInBytes = 11000000 + with open(os.path.join(self.package_folder, "data.txt"), 'wb') as fout: + fout.write(os.urandom(fileSizeInBytes)) + """) + client.save({"conanfile.py": conanfile}) client.run("create . --name=mypkg --version=0.1 --user=user --channel=testing") client.run("upload * --confirm -r default") client.run("remove * -c") @@ -24,24 +35,25 @@ def test_download_skip(self): client.save({"global.conf": f"core.download:download_cache={tmp_folder}"}, path=client.cache.cache_folder) client.run("install --requires=mypkg/0.1@user/testing") - assert "Downloading" in client.out + assert "mypkg/0.1@user/testing: Downloading" in client.out client.run("remove * -c") client.run("install --requires=mypkg/0.1@user/testing") - # TODO assert "Downloading" not in client.out - + assert "mypkg/0.1@user/testing: Downloading" not in client.out + assert "conan_package.tgz from download cache, instead of downloading it" in client.out # removing the config downloads things client.save({"global.conf": ""}, path=client.cache.cache_folder) client.run("remove * -c") client.run("install --requires=mypkg/0.1@user/testing") - assert "Downloading" in client.out + assert "mypkg/0.1@user/testing: Downloading" in client.out client.save({"global.conf": f"core.download:download_cache={tmp_folder}"}, path=client.cache.cache_folder) client.run("remove * -c") client.run("install --requires=mypkg/0.1@user/testing") - # TODO assert "Downloading" not in client.out + assert "mypkg/0.1@user/testing: Downloading" not in client.out + assert "conan_package.tgz from download cache, instead of downloading it" in client.out def test_dirty_download(self): # https://github.com/conan-io/conan/issues/8578