Skip to content

Commit

Permalink
add download cache msg (#14716)
Browse files Browse the repository at this point in the history
  • Loading branch information
memsharded authored Sep 11, 2023
1 parent a23b95c commit e77134f
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 7 deletions.
13 changes: 12 additions & 1 deletion conans/client/downloaders/caching_file_downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@
from urllib.parse import urlparse
from urllib.request import url2pathname

from conan.api.output import ConanOutput
from conans.client.downloaders.file_downloader import FileDownloader
from conans.client.downloaders.download_cache import DownloadCache
from conans.errors import NotFoundException, ConanException, AuthenticationException, \
ForbiddenException
from conans.util.files import mkdir, set_dirty_context_manager, remove_if_dirty
from conans.util.files import mkdir, set_dirty_context_manager, remove_if_dirty, human_size


class SourcesCachingDownloader:
Expand Down Expand Up @@ -158,6 +159,7 @@ def __init__(self, requester, config, scope=None):
if self._download_cache and not os.path.isabs(self._download_cache):
raise ConanException("core.download:download_cache must be an absolute path")
self._file_downloader = FileDownloader(requester, scope=scope)
self._scope = scope

def download(self, url, file_path, auth, verify_ssl, retry, retry_wait, metadata=False):
if not self._download_cache or metadata: # Metadata not cached and can be overwritten
Expand All @@ -175,6 +177,15 @@ def download(self, url, file_path, auth, verify_ssl, retry, retry_wait, metadata
self._file_downloader.download(url, cached_path, retry=retry,
retry_wait=retry_wait, verify_ssl=verify_ssl,
auth=auth, overwrite=False)
else: # Found in cache!
total_length = os.path.getsize(cached_path)
is_large_file = total_length > 10000000 # 10 MB
if is_large_file:
base_name = os.path.basename(file_path)
hs = human_size(total_length)
ConanOutput(scope=self._scope).info(f"Copying {hs} {base_name} from download "
f"cache, instead of downloading it")

# Everything good, file in the cache, just copy it to final destination
mkdir(os.path.dirname(file_path))
shutil.copy2(cached_path, file_path)
24 changes: 18 additions & 6 deletions conans/test/integration/cache/download_cache_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,18 @@ def test_download_skip(self):
""" basic proof that enabling download_cache avoids downloading things again
"""
client = TestClient(default_server_user=True)
client.save({"conanfile.py": GenConanfile().with_package_file("file.txt", "content")})
# generate large random package file
conanfile = textwrap.dedent("""
import os
from conan import ConanFile
from conan.tools.files import save
class Pkg(ConanFile):
def package(self):
fileSizeInBytes = 11000000
with open(os.path.join(self.package_folder, "data.txt"), 'wb') as fout:
fout.write(os.urandom(fileSizeInBytes))
""")
client.save({"conanfile.py": conanfile})
client.run("create . --name=mypkg --version=0.1 --user=user --channel=testing")
client.run("upload * --confirm -r default")
client.run("remove * -c")
Expand All @@ -24,24 +35,25 @@ def test_download_skip(self):
client.save({"global.conf": f"core.download:download_cache={tmp_folder}"},
path=client.cache.cache_folder)
client.run("install --requires=mypkg/0.1@user/testing")
assert "Downloading" in client.out
assert "mypkg/0.1@user/testing: Downloading" in client.out

client.run("remove * -c")
client.run("install --requires=mypkg/0.1@user/testing")
# TODO assert "Downloading" not in client.out

assert "mypkg/0.1@user/testing: Downloading" not in client.out
assert "conan_package.tgz from download cache, instead of downloading it" in client.out
# removing the config downloads things
client.save({"global.conf": ""}, path=client.cache.cache_folder)
client.run("remove * -c")
client.run("install --requires=mypkg/0.1@user/testing")
assert "Downloading" in client.out
assert "mypkg/0.1@user/testing: Downloading" in client.out

client.save({"global.conf": f"core.download:download_cache={tmp_folder}"},
path=client.cache.cache_folder)

client.run("remove * -c")
client.run("install --requires=mypkg/0.1@user/testing")
# TODO assert "Downloading" not in client.out
assert "mypkg/0.1@user/testing: Downloading" not in client.out
assert "conan_package.tgz from download cache, instead of downloading it" in client.out

def test_dirty_download(self):
# https://github.com/conan-io/conan/issues/8578
Expand Down

0 comments on commit e77134f

Please sign in to comment.