Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add download cache msg #14716

Merged
merged 1 commit into from
Sep 11, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion conans/client/downloaders/caching_file_downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@
from urllib.parse import urlparse
from urllib.request import url2pathname

from conan.api.output import ConanOutput
from conans.client.downloaders.file_downloader import FileDownloader
from conans.client.downloaders.download_cache import DownloadCache
from conans.errors import NotFoundException, ConanException, AuthenticationException, \
ForbiddenException
from conans.util.files import mkdir, set_dirty_context_manager, remove_if_dirty
from conans.util.files import mkdir, set_dirty_context_manager, remove_if_dirty, human_size


class SourcesCachingDownloader:
Expand Down Expand Up @@ -158,6 +159,7 @@ def __init__(self, requester, config, scope=None):
if self._download_cache and not os.path.isabs(self._download_cache):
raise ConanException("core.download:download_cache must be an absolute path")
self._file_downloader = FileDownloader(requester, scope=scope)
self._scope = scope

def download(self, url, file_path, auth, verify_ssl, retry, retry_wait, metadata=False):
if not self._download_cache or metadata: # Metadata not cached and can be overwritten
Expand All @@ -175,6 +177,15 @@ def download(self, url, file_path, auth, verify_ssl, retry, retry_wait, metadata
self._file_downloader.download(url, cached_path, retry=retry,
retry_wait=retry_wait, verify_ssl=verify_ssl,
auth=auth, overwrite=False)
else: # Found in cache!
total_length = os.path.getsize(cached_path)
is_large_file = total_length > 10000000 # 10 MB
if is_large_file:
base_name = os.path.basename(file_path)
hs = human_size(total_length)
ConanOutput(scope=self._scope).info(f"Copying {hs} {base_name} from download "
f"cache, instead of downloading it")

# Everything good, file in the cache, just copy it to final destination
mkdir(os.path.dirname(file_path))
shutil.copy2(cached_path, file_path)
24 changes: 18 additions & 6 deletions conans/test/integration/cache/download_cache_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,18 @@ def test_download_skip(self):
""" basic proof that enabling download_cache avoids downloading things again
"""
client = TestClient(default_server_user=True)
client.save({"conanfile.py": GenConanfile().with_package_file("file.txt", "content")})
# generate large random package file
conanfile = textwrap.dedent("""
import os
from conan import ConanFile
from conan.tools.files import save
class Pkg(ConanFile):
def package(self):
fileSizeInBytes = 11000000
with open(os.path.join(self.package_folder, "data.txt"), 'wb') as fout:
fout.write(os.urandom(fileSizeInBytes))
""")
client.save({"conanfile.py": conanfile})
client.run("create . --name=mypkg --version=0.1 --user=user --channel=testing")
client.run("upload * --confirm -r default")
client.run("remove * -c")
Expand All @@ -24,24 +35,25 @@ def test_download_skip(self):
client.save({"global.conf": f"core.download:download_cache={tmp_folder}"},
path=client.cache.cache_folder)
client.run("install --requires=mypkg/0.1@user/testing")
assert "Downloading" in client.out
assert "mypkg/0.1@user/testing: Downloading" in client.out

client.run("remove * -c")
client.run("install --requires=mypkg/0.1@user/testing")
# TODO assert "Downloading" not in client.out

assert "mypkg/0.1@user/testing: Downloading" not in client.out
assert "conan_package.tgz from download cache, instead of downloading it" in client.out
# removing the config downloads things
client.save({"global.conf": ""}, path=client.cache.cache_folder)
client.run("remove * -c")
client.run("install --requires=mypkg/0.1@user/testing")
assert "Downloading" in client.out
assert "mypkg/0.1@user/testing: Downloading" in client.out

client.save({"global.conf": f"core.download:download_cache={tmp_folder}"},
path=client.cache.cache_folder)

client.run("remove * -c")
client.run("install --requires=mypkg/0.1@user/testing")
# TODO assert "Downloading" not in client.out
assert "mypkg/0.1@user/testing: Downloading" not in client.out
assert "conan_package.tgz from download cache, instead of downloading it" in client.out

def test_dirty_download(self):
# https://github.com/conan-io/conan/issues/8578
Expand Down