From 6a75fe84a8935177d3b1a56954353d2f6888ef7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Mon, 10 Jun 2024 12:15:58 +0200 Subject: [PATCH 1/5] Don't fail when we can't overwrite the summary file --- conan/api/subapi/upload.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index f8ad2dc3e4b..1dfeffdcfc1 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -121,8 +121,18 @@ def upload_backup_sources(self, files): basename = os.path.basename(file) full_url = url + basename try: - # Always upload summary .json but only upload blob if it does not already exist - if file.endswith(".json") or not uploader.exists(full_url, auth=None): + if file.endswith(".json"): + # Try to update the summary, but don't fail if it is not possible + # (e.g. because the file needs overwriting, and we have no delete permissions) + output.info(f"Updating summary '{basename}' in backup sources server") + try: + uploader.upload(full_url, file, dedup=False, auth=None) + except (AuthenticationException, ForbiddenException) as e: + output.warning(f"Could not update summary '{basename}' " + f"in backup sources server. Missing permissions?: {e}") + elif not uploader.exists(full_url, auth=None): + # Upload the blob only if it does not exist, but don't rely on dedup, + # because the server might not support it output.info(f"Uploading file '{basename}' to backup sources server") uploader.upload(full_url, file, dedup=False, auth=None) else: From 14678cd92027217ddd468289e20a2a5b17e5b483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Mon, 10 Jun 2024 12:19:40 +0200 Subject: [PATCH 2/5] Simplify --- conan/api/subapi/upload.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index 1dfeffdcfc1..fd6efb00e19 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -120,27 +120,22 @@ def upload_backup_sources(self, files): for file in files: basename = os.path.basename(file) full_url = url + basename + is_summary = file.endswith(".json") try: - if file.endswith(".json"): - # Try to update the summary, but don't fail if it is not possible - # (e.g. because the file needs overwriting, and we have no delete permissions) - output.info(f"Updating summary '{basename}' in backup sources server") - try: - uploader.upload(full_url, file, dedup=False, auth=None) - except (AuthenticationException, ForbiddenException) as e: - output.warning(f"Could not update summary '{basename}' " - f"in backup sources server. Missing permissions?: {e}") - elif not uploader.exists(full_url, auth=None): - # Upload the blob only if it does not exist, but don't rely on dedup, - # because the server might not support it + if is_summary or not uploader.exists(full_url, auth=None): output.info(f"Uploading file '{basename}' to backup sources server") uploader.upload(full_url, file, dedup=False, auth=None) else: output.info(f"File '{basename}' already in backup sources server, " "skipping upload") except (AuthenticationException, ForbiddenException) as e: - raise ConanException(f"The source backup server '{url}' needs authentication" - f"/permissions, please provide 'source_credentials.json': {e}") + if is_summary: + output.warning("Could not update summary '{basename}' in backup sources server. " + "Skipping updating file but continuing with upload. " + f"Missing permissions?: {e}")s + else: + raise ConanException(f"The source backup server '{url}' needs authentication" + f"/permissions, please provide 'source_credentials.json': {e}") output.success("Upload backup sources complete\n") return files From 50fe6b54bd3b78eb808015503c2f6d5ee7e7505b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Mon, 10 Jun 2024 12:21:41 +0200 Subject: [PATCH 3/5] better output msgs --- conan/api/subapi/upload.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index fd6efb00e19..98077f836e4 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -121,16 +121,17 @@ def upload_backup_sources(self, files): basename = os.path.basename(file) full_url = url + basename is_summary = file.endswith(".json") + file_kind = "summary" if is_summary else "file" try: if is_summary or not uploader.exists(full_url, auth=None): - output.info(f"Uploading file '{basename}' to backup sources server") + output.info(f"Uploading {file_kind} '{basename}' to backup sources server") uploader.upload(full_url, file, dedup=False, auth=None) else: output.info(f"File '{basename}' already in backup sources server, " "skipping upload") except (AuthenticationException, ForbiddenException) as e: if is_summary: - output.warning("Could not update summary '{basename}' in backup sources server. " + output.warning(f"Could not update summary '{basename}' in backup sources server. " "Skipping updating file but continuing with upload. " f"Missing permissions?: {e}")s else: From 5a20922a34d893b54ca8a10bccf9ddb1390cf0a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Mon, 10 Jun 2024 12:22:24 +0200 Subject: [PATCH 4/5] Typo --- conan/api/subapi/upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index 98077f836e4..67cde47cc8f 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -133,7 +133,7 @@ def upload_backup_sources(self, files): if is_summary: output.warning(f"Could not update summary '{basename}' in backup sources server. " "Skipping updating file but continuing with upload. " - f"Missing permissions?: {e}")s + f"Missing permissions?: {e}") else: raise ConanException(f"The source backup server '{url}' needs authentication" f"/permissions, please provide 'source_credentials.json': {e}") From 0bfb822b67756b73ff515d84540df017d2787ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Thu, 13 Jun 2024 08:29:38 +0200 Subject: [PATCH 5/5] Auth test --- test/integration/cache/backup_sources_test.py | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/test/integration/cache/backup_sources_test.py b/test/integration/cache/backup_sources_test.py index 5cb40f1a11e..5c72e660976 100644 --- a/test/integration/cache/backup_sources_test.py +++ b/test/integration/cache/backup_sources_test.py @@ -538,8 +538,16 @@ def get_internet_file(file): def get_file(file): return static_file(file, http_server_base_folder_backup1) + @http_server.server.put("/downloader1/") + def put_file(file): + if file in os.listdir(http_server_base_folder_backup1): + return HTTPError(401, "You Are Not Allowed Here") + dest = os.path.join(http_server_base_folder_backup1, file) + with open(dest, 'wb') as f: + f.write(request.body.read()) + @http_server.server.get("/downloader2/") - def get_file(file): + def get_file_2(file): return static_file(file, http_server_base_folder_backup2) http_server.run_server() @@ -555,9 +563,10 @@ def source(self): sha256="{sha256}") """) client.save_home({"global.conf": f"core.sources:download_cache={download_cache_folder}\n" - f"core.sources:download_urls=['http://localhost:{http_server.port}/downloader1/', " - f"'origin', 'http://localhost:{http_server.port}/downloader2/']\n"}) - + f"core.sources:download_urls=['http://localhost:{http_server.port}/downloader1/', " + f"'origin', 'http://localhost:{http_server.port}/downloader2/']\n" + f"core.sources:upload_url=http://localhost:{http_server.port}/downloader1/\n" + "core.upload:retry=0\ncore.download:retry=0"}) client.save({"conanfile.py": conanfile}) client.run("create .") @@ -565,6 +574,17 @@ def source(self): # TODO: Check better message with Authentication error message assert "failed in 'origin'" in client.out + # Now try to upload once to the first backup server. It's configured so it has write permissions but no overwrite + client.run("upload * -c -r=default") + upload_server_contents = os.listdir(http_server_base_folder_backup1) + assert sha256 in upload_server_contents + assert sha256 + ".json" in upload_server_contents + + client.run("remove * -c -r=default") + + client.run("upload * -c -r=default") + assert f"Could not update summary '{sha256}.json' in backup sources server" in client.out + def test_ok_when_origin_bad_sha256(self): http_server_base_folder_internet = os.path.join(self.file_server.store, "internet") http_server_base_folder_backup2 = os.path.join(self.file_server.store, "backup2")