Fix corrupted packages with missing conanmanifest.txt files

conans/client/cmd/uploader.py

@@ -360,18 +360,18 @@ def _package_files_to_upload(self, pref, policy, the_files, remote):
         remote_snapshot = self._remote_manager.get_package_snapshot(pref, remote)
 
         if remote_snapshot:
-            remote_manifest, _ = self._remote_manager.get_package_manifest(pref, remote)
-            local_manifest = FileTreeManifest.loads(load(the_files["conanmanifest.txt"]))
+            if self._remote_manager.package_snapshot_complete(remote_snapshot):
+                remote_manifest, _ = self._remote_manager.get_package_manifest(pref, remote)
+                local_manifest = FileTreeManifest.loads(load(the_files["conanmanifest.txt"]))
 
-            if remote_manifest == local_manifest:
-                return None, None
+                if remote_manifest == local_manifest:
+                    return None, None
 
-            if policy == UPLOAD_POLICY_NO_OVERWRITE:
-                raise ConanException("Local package is different from the remote package. "
-                                     "Forbidden overwrite.")
+                if policy == UPLOAD_POLICY_NO_OVERWRITE:
+                    raise ConanException("Local package is different from the remote package. "
+                                         "Forbidden overwrite.")
         files_to_upload = the_files
         deleted = set(remote_snapshot).difference(the_files)
-
         return files_to_upload, deleted
 
     def _upload_recipe_end_msg(self, ref, remote):

conans/client/remote_manager.py

@@ -11,7 +11,8 @@
 from conans.errors import ConanConnectionError, ConanException, NotFoundException, \
     NoRestV2Available, PackageNotFoundException
 from conans.paths import EXPORT_SOURCES_DIR_OLD, \
-    EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, rm_conandir
+    EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, rm_conandir, CONANINFO, \
+    CONAN_MANIFEST
 from conans.search.search import filter_packages
 from conans.util import progress_bar
 from conans.util.env_reader import get_env
@@ -129,6 +130,9 @@ def get_package(self, pref, dest_folder, remote, output, recorder):
         t1 = time.time()
         try:
             pref = self._resolve_latest_pref(pref, remote)
+            snapshot = self._call_remote(remote, "get_package_snapshot", pref)
+            if not self.package_snapshot_complete(snapshot):
+                raise NotFoundException
             zipped_files = self._call_remote(remote, "get_package", pref, dest_folder)
 
             with self._cache.package_layout(pref.ref).update_metadata() as metadata:
@@ -240,6 +244,15 @@ def _call_remote(self, remote, method, *argc, **argv):
             logger.error(traceback.format_exc())
             raise ConanException(exc, remote=remote)
 
+    @staticmethod
+    def package_snapshot_complete(snapshot):
+        integrity = True
+        for keyword in ["conaninfo", "conanmanifest", "conan_package"]:
+            if not any(keyword in key for key in snapshot):
+                integrity = False
+                break
+        return integrity
+
 
 def check_compressed_files(tgz_name, files):
     bare_name = os.path.splitext(tgz_name)[0]

conans/client/rest/rest_client_v1.py

@@ -142,9 +142,7 @@ def _upload_files(self, file_urls, files, output, retry, retry_wait):
         t1 = time.time()
         failed = []
         uploader = Uploader(self.requester, output, self.verify_ssl)
-        # Take advantage of filenames ordering, so that conan_package.tgz and conan_export.tgz
-        # can be < conanfile, conaninfo, and sent always the last, so smaller files go first
-        for filename, resource_url in sorted(file_urls.items(), reverse=True):
+        for filename, resource_url in sorted(file_urls.items()):
             output.rewrite_line("Uploading %s" % filename)
             auth, dedup = self._file_server_capabilities(resource_url)
             try:

conans/client/rest/rest_client_v2.py

@@ -174,9 +174,7 @@ def _upload_files(self, files, urls, retry, retry_wait):
         t1 = time.time()
         failed = []
         uploader = Uploader(self.requester, self._output, self.verify_ssl)
-        # Take advantage of filenames ordering, so that conan_package.tgz and conan_export.tgz
-        # can be < conanfile, conaninfo, and sent always the last, so smaller files go first
-        for filename in sorted(files, reverse=True):
+        for filename in sorted(files):
             self._output.rewrite_line("Uploading %s" % filename)
             resource_url = urls[filename]
             try:

conans/test/functional/command/upload_complete_test.py

@@ -175,7 +175,7 @@ def upload_error_test(self):
         client.run("upload Hello* --confirm --retry 1 --retry-wait=1", assert_error=True)
         self.assertNotIn("Waiting 1 seconds to retry...", client.user_io.out)
         self.assertIn("ERROR: Execute upload again to retry upload the failed files: "
-                      "conanmanifest.txt. [Remote: default]", client.user_io.out)
+                      "conan_export.tgz. [Remote: default]", client.user_io.out)
 
         # Try with broken connection even with 10 retries
         client = self._get_client(TerribleConnectionUploader)
@@ -329,13 +329,13 @@ def upload_all_test(self):
                  if line.startswith("Uploading")]
         self.assertEqual(lines, ["Uploading to remote 'default':",
                                  "Uploading Hello/1.2.1@frodo/stable to remote 'default'",
-                                 "Uploading conanmanifest.txt",
-                                 "Uploading conanfile.py",
                                  "Uploading conan_export.tgz",
-                                 "Uploading package 1/1: myfakeid to 'default'",
+                                 "Uploading conanfile.py",
                                  "Uploading conanmanifest.txt",
-                                 "Uploading conaninfo.txt",
+                                 "Uploading package 1/1: myfakeid to 'default'",
                                  "Uploading conan_package.tgz",
+                                 "Uploading conaninfo.txt",
+                                 "Uploading conanmanifest.txt",
                                  ])
         if self.client.cache.config.revisions_enabled:
             layout = self.client.cache.package_layout(self.ref)

conans/test/functional/corrupted_packages_test.py

@@ -0,0 +1,146 @@
+import os
+import textwrap
+import unittest
+
+from conans.model.ref import ConanFileReference, PackageReference
+from conans.util.env_reader import get_env
+from conans.test.utils.tools import TestClient, TestServer, NO_SETTINGS_PACKAGE_ID
+
+
+class CorruptedPackagesTest(unittest.TestCase):
+    """
+    Simulate a connection failure or file corruption in the server with missing files for a
+    package and make sure the search, install are possible. Check re-upload is always possible
+    even if the package in the server is not accessible
+    """
+
+    def setUp(self):
+        revisions_enabled = get_env("CONAN_REVISIONS_ENABLED")
+        self.server = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")])
+        self.client = TestClient(servers={"default": self.server})
+        conanfile = textwrap.dedent("""
+        from conans import ConanFile
+
+        class Pkg(ConanFile):
+            pass
+        """)
+        self.client.save({"conanfile.py": conanfile})
+        self.client.run("create . Pkg/0.1@user/testing")
+        self.client.run("upload * --all --confirm -r default")
+        # Check files are uploded in this order: conan_package.tgz, conaninfo.txt, conanmanifest.txt
+        order1 = str(self.client.out).find("Uploading conan_package.tgz")
+        order2 = str(self.client.out).find("Uploading conaninfo.txt", order1)
+        order3 = str(self.client.out).find("Uploading conanmanifest.txt", order2)
+        self.assertTrue(order1 < order2 < order3)
+        rrev = "210a4a16419aae28fea1f268a8e4f3d4" if revisions_enabled else "0"
+        pref_str = "Pkg/0.1@user/testing#%s" % rrev
+        prev = "11de80325e8db78617b05384825c6409" if revisions_enabled else "0"
+        self.pref = pref = PackageReference(ConanFileReference.loads(pref_str),
+                                NO_SETTINGS_PACKAGE_ID, prev)
+        self.manifest_path = self.server.server_store.get_package_file_path(pref,
+                                                                            "conanmanifest.txt")
+        self.info_path = self.server.server_store.get_package_file_path(pref, "conaninfo.txt")
+        self.tgz_path = self.server.server_store.get_package_file_path(pref, "conan_package.tgz")
+
+    def _assert_all_package_files_in_server(self):
+        self.assertTrue(os.path.exists(self.manifest_path))
+        self.assertTrue(os.path.exists(self.info_path))
+        self.assertTrue(os.path.exists(self.tgz_path))
+
+    def info_manifest_missing_test(self):
+        os.unlink(self.info_path)
+        os.unlink(self.manifest_path)
+        # Try search
+        self.client.run("search Pkg/0.1@user/testing -r default")
+        self.assertIn("There are no packages for reference 'Pkg/0.1@user/testing', "
+                      "but package recipe found", self.client.out)
+        # Try fresh install
+        self.client.run("remove * -f")
+        self.client.run("install Pkg/0.1@user/testing", assert_error=True)
+        self.assertIn("Pkg/0.1@user/testing:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 - Missing",
+                      self.client.out)
+        # Try upload of fresh package
+        self.client.run("create . Pkg/0.1@user/testing")
+        self.client.run("upload * --all --confirm -r default")
+        self._assert_all_package_files_in_server()
+
+    def manifest_missing_test(self):
+        os.unlink(self.manifest_path)
+        # Try search
+        self.client.run("search Pkg/0.1@user/testing -r default")
+        self.assertIn("Package_ID: 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", self.client.out)
+        # Try fresh install
+        self.client.run("remove * -f")
+        self.client.run("install Pkg/0.1@user/testing", assert_error=True)
+        self.assertIn("ERROR: Binary package not found", self.client.out)
+        self.assertIn(NO_SETTINGS_PACKAGE_ID, self.client.out)
+        # Try upload of fresh package
+        self.client.run("create . Pkg/0.1@user/testing")
+        self.client.run("upload * --all --confirm")
+        self._assert_all_package_files_in_server()
+
+    def tgz_info_missing_test(self):
+        os.unlink(self.tgz_path)
+        os.unlink(self.info_path)
+        # Try search
+        self.client.run("search Pkg/0.1@user/testing -r default")
+        self.assertIn("There are no packages for reference 'Pkg/0.1@user/testing', "
+                      "but package recipe found", self.client.out)
+        # Try fresh install
+        self.client.run("remove * -f")
+        self.client.run("install Pkg/0.1@user/testing", assert_error=True)
+        self.assertIn("Pkg/0.1@user/testing:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 - Missing",
+                      self.client.out)
+        # Try upload of fresh package
+        self.client.run("create . Pkg/0.1@user/testing")
+        self.client.run("upload * --all --confirm")
+        self.assertIn("Uploading conan_package.tgz", self.client.out)
+        self.assertIn("Uploading conaninfo.txt", self.client.out)
+        self._assert_all_package_files_in_server()
+
+    def tgz_missing_test(self):
+        os.unlink(self.tgz_path)
+        # Try search
+        self.client.run("search Pkg/0.1@user/testing -r default")
+        # Try fresh install
+        self.client.run("remove * -f")
+        self.client.run("install Pkg/0.1@user/testing", assert_error=True)
+        self.assertIn("ERROR: Binary package not found", self.client.out)
+        # Try upload of fresh package
+        self.client.run("create . Pkg/0.1@user/testing")
+        self.client.run("upload * --all --confirm")
+        self.assertIn("Uploading conan_package.tgz", self.client.out)
+        self._assert_all_package_files_in_server()
+
+    def tgz_manifest_missing_test(self):
+        os.unlink(self.tgz_path)
+        os.unlink(self.manifest_path)
+        # Try search
+        self.client.run("search Pkg/0.1@user/testing -r default")
+        self.assertIn("Package_ID: 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", self.client.out)
+        # Try fresh install
+        self.client.run("remove * -f")
+        self.client.run("install Pkg/0.1@user/testing", assert_error=True)
+        self.assertIn("ERROR: Binary package not found", self.client.out)
+        # Try upload of fresh package
+        self.client.run("create . Pkg/0.1@user/testing")
+        self.client.run("upload * --all --confirm")
+        self._assert_all_package_files_in_server()
+
+    def tgz_manifest_info_missing_test(self):
+        os.unlink(self.tgz_path)
+        os.unlink(self.manifest_path)
+        os.unlink(self.info_path)
+        # Try search
+        self.client.run("search Pkg/0.1@user/testing -r default")
+        self.assertIn("There are no packages for reference 'Pkg/0.1@user/testing', "
+                      "but package recipe found", self.client.out)
+        # Try fresh install
+        self.client.run("remove * -f")
+        self.client.run("install Pkg/0.1@user/testing", assert_error=True)
+        self.assertIn("Pkg/0.1@user/testing:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 - Missing",
+                      self.client.out)
+        # Try upload of fresh package
+        self.client.run("create . Pkg/0.1@user/testing")
+        self.client.run("upload * --all --confirm")
+        self._assert_all_package_files_in_server()

conans/test/integration/install_update_test.py

@@ -234,7 +234,7 @@ def reuse_test(self):
         self.client.run("install Hello0/1.0@lasote/stable --build")
         self.client.run("upload Hello0/1.0@lasote/stable --all")
 
-        client2 = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]})
+        client2 = TestClient(servers=self.servers, users={"myremote": [("lasote", "mypass")]})
         client2.run("install Hello0/1.0@lasote/stable")
 
         self.assertEquals(str(client2.out).count("Downloading conaninfo.txt"), 1)