From 0efdda06f08cd5f08b44effb696c4d8f691b43f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n?= Date: Mon, 8 May 2023 17:35:19 +0200 Subject: [PATCH] Upload backup source even when the recipe has been exported --- conans/client/downloaders/download_cache.py | 12 ++++- conans/client/downloaders/file_downloader.py | 2 +- .../integration/cache/backup_sources_test.py | 48 +++++++++++++++++-- 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/conans/client/downloaders/download_cache.py b/conans/client/downloaders/download_cache.py index a917265a81d..0109af204c1 100644 --- a/conans/client/downloaders/download_cache.py +++ b/conans/client/downloaders/download_cache.py @@ -48,11 +48,18 @@ def get_backup_sources_files_to_upload(self, package_list): """ from a package_list of packages to upload, collect from the backup-sources cache the matching references to upload those backups too """ + def should_upload_sources(package): + return any(prev["upload"] for prev in package["revisions"].values()) + files_to_upload = [] path_backups = os.path.join(self._path, self._SOURCE_BACKUP) + if not os.path.exists(path_backups): return [] - all_refs = {str(k) for k, v in package_list.refs() if v.get("upload")} + + all_refs = {str(k) for k, ref in package_list.refs() + if ref.get("upload") or any(should_upload_sources(p) + for p in ref["packages"].values())} for f in os.listdir(path_backups): if f.endswith(".json"): f = os.path.join(path_backups, f) @@ -85,4 +92,7 @@ def update_backup_sources_json(cached_path, conanfile, urls): urls = [urls] existing_urls = summary["references"].setdefault(summary_key, []) existing_urls.extend(url for url in urls if url not in existing_urls) + conanfile.output.verbose(f"Updating ${summary_path} summary file") + summary_dump = json.dumps(summary) + conanfile.output.debug(f"New summary: ${summary_dump}") save(summary_path, json.dumps(summary)) diff --git a/conans/client/downloaders/file_downloader.py b/conans/client/downloaders/file_downloader.py index 7dfb0717a59..4997f54413f 100644 --- a/conans/client/downloaders/file_downloader.py +++ b/conans/client/downloaders/file_downloader.py @@ -104,7 +104,7 @@ def get_total_length(): total_length = get_total_length() if total_length > 100000: action = "Downloading" if range_start == 0 else "Continuing download of" - description = "{} {}".format(action, os.path.basename(file_path)) + description = f"${action} {os.path.basename(file_path)} from {url}" self._output.info(description) chunk_size = 1024 * 100 diff --git a/conans/test/integration/cache/backup_sources_test.py b/conans/test/integration/cache/backup_sources_test.py index 87613335aed..3c9923304c4 100644 --- a/conans/test/integration/cache/backup_sources_test.py +++ b/conans/test/integration/cache/backup_sources_test.py @@ -628,14 +628,54 @@ def source(self): assert f"Sources for http://localhost:{http_server.port}/internet/myfile.txt found in remote backup http://localhost:{http_server.port}/downloader2/" in client.out assert "sha256 signature failed for" in client.out - def test_upload_after_export(self): + def test_export_then_upload_workflow(self): client = TestClient(default_server_user=True) download_cache_folder = temp_folder() - client.save({"global.conf": "core.sources:upload_url=foo"}, + http_server = StoppableThreadBottle() + + http_server_base_folder_internet = temp_folder() + http_server_base_folder_backup1 = temp_folder() + + sha256 = "315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3" + save(os.path.join(http_server_base_folder_internet, "myfile.txt"), "Hello, world!") + + @http_server.server.get("/internet/") + def get_internet_file(file): + return static_file(file, http_server_base_folder_internet) + + @http_server.server.get("/downloader1/") + def get_file(file): + return static_file(file, http_server_base_folder_backup1) + + @http_server.server.put("/uploader/") + def put_file(file): + dest = os.path.join(http_server_base_folder_backup1, file) + with open(dest, 'wb') as f: + f.write(request.body.read()) + + http_server.run_server() + + conanfile = textwrap.dedent(f""" + from conan import ConanFile + from conan.tools.files import download + class Pkg2(ConanFile): + name = "pkg" + version = "1.0" + def source(self): + download(self, "http://localhost:{http_server.port}/internet/myfile.txt", "myfile.txt", + sha256="{sha256}") + """) + + client.save({"global.conf": f"core.sources:download_cache={download_cache_folder}\n" + f"core.sources:download_urls=['http://localhost:{http_server.port}/downloader1/', 'origin']\n" + f"core.sources:upload_url=http://localhost:{http_server.port}/uploader/"}, path=client.cache.cache_folder) - client.save({"conanfile.py": GenConanfile("pkg", "1.0")}) + client.save({"conanfile.py": conanfile}) client.run("export .") - # This used to crash because we were trying to list a missing dir if only exports were made client.run("upload * -c -r=default") + # This used to crash because we were trying to list a missing dir if only exports were made assert "[Errno 2] No such file or directory" not in client.out + client.run("create .") + client.run("upload * -c -r=default") + assert sha256 in os.listdir(http_server_base_folder_backup1)