Add checks for duplicated refs when downloading from backup source

conans/client/downloaders/caching_file_downloader.py

@@ -61,6 +61,7 @@ def _caching_download(self, urls, file_path,
         with download_cache.lock(sha256):
             remove_if_dirty(cached_path)
 
+            url = None
             if os.path.exists(cached_path):
                 self._output.info(f"Source {urls} retrieved from local download cache")
             else:
@@ -72,16 +73,17 @@ def _caching_download(self, urls, file_path,
 
                         is_last = backup_url is backups_urls[-1]
                         if backup_url == "origin":  # recipe defined URLs
-                            if self._origin_download(urls, cached_path, retry, retry_wait,
-                                                     verify_ssl, auth, headers, md5, sha1, sha256,
-                                                     is_last):
+                            url = self._origin_download(urls, cached_path, retry, retry_wait,
+                                                        verify_ssl, auth, headers, md5, sha1, sha256,
+                                                        is_last)
+                            if url:
                                 break
                         else:
                             if self._backup_download(backup_url, backups_urls, sha256, cached_path,
                                                      urls, is_last):
                                 break
-
-            download_cache.update_backup_sources_json(cached_path, self._conanfile, urls)
+            if url:
+                download_cache.update_backup_sources_json(cached_path, self._conanfile, url)
             # Everything good, file in the cache, just copy it to final destination
             mkdir(os.path.dirname(file_path))
             shutil.copy2(cached_path, file_path)
@@ -91,19 +93,18 @@ def _origin_download(self, urls, cached_path, retry, retry_wait,
         """ download from the internet, the urls provided by the recipe (mirrors).
         """
         try:
-            self._download_from_urls(urls, cached_path, retry, retry_wait,
+            url = self._download_from_urls(urls, cached_path, retry, retry_wait,
                                      verify_ssl, auth, headers, md5, sha1, sha256)
+            if not is_last:
+                self._output.info(f"Sources for {urls} found in origin")
+            return url
         except ConanException as e:
             if is_last:
                 raise
             else:
                 # TODO: Improve printing of AuthenticationException
                 self._output.warning(f"Sources for {urls} failed in 'origin': {e}")
                 self._output.warning("Checking backups")
-        else:
-            if not is_last:
-                self._output.info(f"Sources for {urls} found in origin")
-            return True
 
     def _backup_download(self, backup_url, backups_urls, sha256, cached_path, urls, is_last):
         """ download from a Conan backup sources file server, like an Artifactory generic repo
@@ -143,7 +144,7 @@ def _download_from_urls(self, urls, file_path, retry, retry_wait, verify_ssl, au
                 else:
                     self._file_downloader.download(url, file_path, retry, retry_wait, verify_ssl,
                                                    auth, True, headers, md5, sha1, sha256)
-                return  # Success! Return to caller
+                return url  # Success! Return to caller
             except Exception as error:
                 if url != urls[-1]:  # If it is not the last one, do not raise, warn and move to next
                     msg = f"Could not download from the URL {url}: {error}."

conans/client/downloaders/download_cache.py

@@ -109,7 +109,7 @@ def should_upload_sources(package):
         return files_to_upload
 
     @staticmethod
-    def update_backup_sources_json(cached_path, conanfile, urls):
+    def update_backup_sources_json(cached_path, conanfile, url):
         """ create or update the sha256.json file with the references and new urls used
         """
         summary_path = cached_path + ".json"
@@ -125,10 +125,21 @@ def update_backup_sources_json(cached_path, conanfile, urls):
             # So best we can do is to set this as unknown
             summary_key = "unknown"
 
-        if not isinstance(urls, (list, tuple)):
-            urls = [urls]
+        from urllib.parse import urlparse
+
+        current_filename = os.path.basename(urlparse(url).path)
+
+        for ref, previous_urls in summary["references"].items():
+            if ref == summary_key or "unknown" in (ref, summary_key):
+                continue
+            prev_filenames = set(os.path.basename(urlparse(url).path) for url in previous_urls)
+            if current_filename in prev_filenames:
+                raise ConanException("This backup source has already been downloaded from a different URLs. "
+                                     "There's probably an issue in your declared sources. "
+                                     "If you have a valid use-case, please report it to the Conan team.")
+
         existing_urls = summary["references"].setdefault(summary_key, [])
-        existing_urls.extend(url for url in urls if url not in existing_urls)
+        existing_urls.append(url)
         conanfile.output.verbose(f"Updating ${summary_path} summary file")
         summary_dump = json.dumps(summary)
         conanfile.output.debug(f"New summary: ${summary_dump}")

test/integration/cache/backup_sources_test.py

@@ -51,17 +51,18 @@ class Pkg2(ConanFile):
                     name = "pkg"
                     version = "1.0"
                     def source(self):
-                        download(self, "http://localhost.mirror:5000/myfile.txt", "myfile.txt",
+                        fname = self.user if self.user else ""
+                        download(self, "http://localhost.mirror:5000/myfile" + fname + ".txt", "myfile.txt",
                                  sha256="{sha256}")
                 """)
             client.save({"conanfile.py": conanfile})
             client.run("source .")
 
             assert 2 == len(os.listdir(os.path.join(tmp_folder, "s")))
             content = json.loads(load(os.path.join(tmp_folder, "s", sha256 + ".json")))
-            assert "http://localhost.mirror:5000/myfile.txt" in content["references"]["unknown"]
-            assert "http://localhost:5000/myfile.txt" in content["references"]["unknown"]
-            assert len(content["references"]["unknown"]) == 2
+            # Not updated because we didn't actually fetch from the mirror
+            assert "http://localhost.mirror:5000/myfile.txt" not in content["references"]["unknown"]
+            assert len(content["references"]["unknown"]) == 1
 
             # Ensure the cache is working and we didn't break anything by modifying the summary
             client.run("source .")
@@ -75,7 +76,7 @@ def source(self):
             client.run("create . --user=barbarian --channel=stable")
             content = json.loads(load(os.path.join(tmp_folder, "s", sha256 + ".json")))
             assert content["references"]["pkg/1.0@barbarian/stable"] == \
-                   ["http://localhost.mirror:5000/myfile.txt"]
+                   ["http://localhost.mirror:5000/myfilebarbarian.txt"]
 
     @pytest.fixture(autouse=True)
     def _setup(self):
@@ -616,6 +617,35 @@ def source(self):
         assert f"Sources for {self.file_server.fake_url}/internet/myfile.txt found in remote backup {self.file_server.fake_url}/backup2/" in self.client.out
         assert "sha256 signature failed for" in self.client.out
 
+    def test_warn_when_duplicated_sha256(self):
+        http_server_base_folder_internet = os.path.join(self.file_server.store, "internet")
+
+        save(os.path.join(http_server_base_folder_internet, "myfile.txt"), "Hello, world!")
+
+        sha256 = "315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3"
+
+        conanfile = textwrap.dedent(f"""
+           from conan import ConanFile
+           from conan.tools.files import download
+           class Pkg2(ConanFile):
+               name = "pkg"
+               version = "%s"
+               def source(self):
+                   download(self, "{self.file_server.fake_url}/internet/myfile.txt", "myfile.txt",
+                            sha256="{sha256}")
+           """)
+
+        self.client.save_home(
+            {"global.conf": f"core.sources:download_cache={self.download_cache_folder}\n"
+                            f"core.sources:download_urls=['{self.file_server.fake_url}/backup', 'origin']\n"})
+
+        self.client.save({"conanfile.py": conanfile % "1.0"})
+        self.client.run("create .")
+
+        self.client.save({"conanfile.py": conanfile % "2.0"})
+        self.client.run("create .", assert_error=True)
+        assert "This backup source has already been downloaded from different URLs" in self.client.out
+
     def test_export_then_upload_workflow(self):
         mkdir(os.path.join(self.download_cache_folder, "s"))