fixed duplicated download of manifest and manifest download output (c…

…onan-io#3552)

* fixed duplicated download of manifest and manifest download output

* fixing rest api test

* merged

conans/client/cmd/uploader.py

@@ -95,10 +95,12 @@ def _upload(self, conan_file, conan_ref, packages_ids, retry, retry_wait,
                                      reference=conan_ref, remote=upload_remote)
 
         if policy != UPLOAD_POLICY_FORCE:
-            self._check_recipe_date(conan_ref, upload_remote)
+            remote_manifest = self._check_recipe_date(conan_ref, upload_remote)
+        else:
+            remote_manifest = None
 
         self._user_io.out.info("Uploading %s to remote '%s'" % (str(conan_ref), upload_remote.name))
-        self._upload_recipe(conan_ref, retry, retry_wait, policy, upload_remote)
+        self._upload_recipe(conan_ref, retry, retry_wait, policy, upload_remote, remote_manifest)
 
         recorder.add_recipe(str(conan_ref), upload_remote.name, upload_remote.url)
 
@@ -122,15 +124,15 @@ def _upload(self, conan_file, conan_ref, packages_ids, retry, retry_wait,
         self._plugin_manager.execute("post_upload", conanfile_path=conanfile_path,
                                      reference=conan_ref, remote=upload_remote)
 
-    def _upload_recipe(self, conan_reference, retry, retry_wait, policy, remote):
+    def _upload_recipe(self, conan_reference, retry, retry_wait, policy, remote, remote_manifest):
         conan_file_path = self._client_cache.conanfile(conan_reference)
         current_remote = self._registry.get_recipe_remote(conan_reference)
         if remote != current_remote:
             conanfile = self._loader.load_class(conan_file_path)
             complete_recipe_sources(self._remote_manager, self._client_cache, self._registry,
                                     conanfile, conan_reference)
         result = self._remote_manager.upload_recipe(conan_reference, remote, retry, retry_wait,
-                                                    policy=policy)
+                                                    policy=policy, remote_manifest=remote_manifest)
         return result
 
     def _upload_package(self, package_ref, index=1, total=1, retry=None, retry_wait=None,
@@ -159,3 +161,5 @@ def _check_recipe_date(self, conan_ref, remote):
             raise ConanException("Remote recipe is newer than local recipe: "
                                  "\n Remote date: %s\n Local date: %s" %
                                  (remote_recipe_manifest.time, local_manifest.time))
+
+        return remote_recipe_manifest

conans/client/remote_manager.py

@@ -41,7 +41,8 @@ def __init__(self, client_cache, auth_manager, output, plugin_manager):
         self._auth_manager = auth_manager
         self._plugin_manager = plugin_manager
 
-    def upload_recipe(self, conan_reference, remote, retry, retry_wait, policy=None):
+
+    def upload_recipe(self, conan_reference, remote, retry, retry_wait, policy, remote_manifest):
         conanfile_path = self._client_cache.conanfile(conan_reference)
         self._plugin_manager.execute("pre_upload_recipe", conanfile_path=conanfile_path,
                                      reference=conan_reference, remote=remote)
@@ -66,7 +67,7 @@ def upload_recipe(self, conan_reference, remote, retry, retry_wait, policy=None)
             return None
 
         ret, new_ref = self._call_remote(remote, "upload_recipe", conan_reference, the_files, retry,
-                                         retry_wait, policy)
+                                         retry_wait, policy, remote_manifest)
         duration = time.time() - t1
         log_recipe_upload(new_ref, duration, the_files, remote.name)
         if ret:

conans/client/rest/auth_manager.py

@@ -125,9 +125,9 @@ def set_custom_headers(self, username):
     # ######### CONAN API METHODS ##########
 
     @input_credentials_if_unauthorized
-    def upload_recipe(self, conan_reference, the_files, retry, retry_wait, policy):
+    def upload_recipe(self, conan_reference, the_files, retry, retry_wait, policy, remote_manifest):
         return self._rest_client.upload_recipe(conan_reference, the_files, retry, retry_wait,
-                                               policy)
+                                               policy, remote_manifest)
 
     @input_credentials_if_unauthorized
     def upload_package(self, package_reference, the_files, retry, retry_wait, policy):

conans/client/rest/rest_client.py

@@ -61,9 +61,9 @@ def get_package(self, package_reference, dest_folder):
     def get_path(self, conan_reference, package_id, path):
         return self._get_api().get_path(conan_reference, package_id, path)
 
-    def upload_recipe(self, conan_reference, the_files, retry, retry_wait, no_overwrite):
+    def upload_recipe(self, conan_reference, the_files, retry, retry_wait, policy, remote_manifest):
         return self._get_api().upload_recipe(conan_reference, the_files, retry, retry_wait,
-                                             no_overwrite)
+                                             policy, remote_manifest)
 
     def upload_package(self, package_reference, the_files, retry, retry_wait, no_overwrite):
         return self._get_api().upload_package(package_reference, the_files, retry, retry_wait,

conans/client/rest/rest_client_common.py

@@ -141,7 +141,8 @@ def get_json(self, url, data=None):
             raise ConanException("Unexpected server response %s" % result)
         return result
 
-    def upload_recipe(self, conan_reference, the_files, retry, retry_wait, policy):
+    def upload_recipe(self, conan_reference, the_files, retry, retry_wait, policy,
+                      remote_manifest):
         """
         the_files: dict with relative_path: content
         """
@@ -151,7 +152,7 @@ def upload_recipe(self, conan_reference, the_files, retry, retry_wait, policy):
         remote_snapshot, conan_reference = self._get_recipe_snapshot(conan_reference)
 
         if remote_snapshot and policy != UPLOAD_POLICY_FORCE:
-            remote_manifest = self.get_conan_manifest(conan_reference)
+            remote_manifest = remote_manifest or self.get_conan_manifest(conan_reference)
             local_manifest = FileTreeManifest.loads(load(the_files["conanmanifest.txt"]))
 
             if remote_manifest == local_manifest:

conans/client/rest/rest_client_v1.py

@@ -12,10 +12,8 @@
 from conans.model.ref import PackageReference
 from conans.paths import CONAN_MANIFEST, CONANINFO, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, \
     PACKAGE_TGZ_NAME
-from conans.util.files import decode_text, load
+from conans.util.files import decode_text
 from conans.util.log import logger
-from conans.client.cmd.uploader import UPLOAD_POLICY_NO_OVERWRITE,\
-    UPLOAD_POLICY_NO_OVERWRITE_RECIPE, UPLOAD_POLICY_FORCE
 
 
 def complete_url(base_url, url):
@@ -33,20 +31,23 @@ class RestV1Methods(RestCommonMethods):
     def remote_api_url(self):
         return "%s/v1" % self.remote_url.rstrip("/")
 
-    def _download_files(self, file_urls):
+    def _download_files(self, file_urls, quiet=False):
         """
         :param: file_urls is a dict with {filename: url}
 
         Its a generator, so it yields elements for memory performance
         """
-        downloader = Downloader(self.requester, self._output, self.verify_ssl)
+        output = self._output if not quiet else None
+        downloader = Downloader(self.requester, output, self.verify_ssl)
         # Take advantage of filenames ordering, so that conan_package.tgz and conan_export.tgz
         # can be < conanfile, conaninfo, and sent always the last, so smaller files go first
         for filename, resource_url in sorted(file_urls.items(), reverse=True):
-            self._output.writeln("Downloading %s" % filename)
+            if output:
+                output.writeln("Downloading %s" % filename)
             auth, _ = self._file_server_capabilities(resource_url)
             contents = downloader.download(resource_url, auth=auth)
-            self._output.writeln("")
+            if output:
+                output.writeln("")
             yield os.path.normpath(filename), contents
 
     def _file_server_capabilities(self, resource_url):
@@ -69,7 +70,7 @@ def get_conan_manifest(self, conan_reference):
         urls = self._get_file_to_url_dict(url)
 
         # Get the digest
-        contents = self._download_files(urls)
+        contents = self._download_files(urls, quiet=True)
         # Unroll generator and decode shas (plain text)
         contents = {key: decode_text(value) for key, value in dict(contents).items()}
         return FileTreeManifest.loads(contents[CONAN_MANIFEST])
@@ -82,7 +83,7 @@ def get_package_manifest(self, package_reference):
         urls = self._get_file_to_url_dict(url)
 
         # Get the digest
-        contents = self._download_files(urls)
+        contents = self._download_files(urls, quiet=True)
         # Unroll generator and decode shas (plain text)
         contents = {key: decode_text(value) for key, value in dict(contents).items()}
         return FileTreeManifest.loads(contents[CONAN_MANIFEST])

conans/client/rest/uploader_downloader.py

@@ -169,9 +169,10 @@ def _download_data(self, response, file_path):
             if not file_path:
                 ret += response.content
             else:
-                total_length = len(response.content)
-                progress = human_readable_progress(total_length, total_length)
-                print_progress(self.output, 50, progress)
+                if self.output:
+                    total_length = len(response.content)
+                    progress = human_readable_progress(total_length, total_length)
+                    print_progress(self.output, 50, progress)
                 save_append(file_path, response.content)
         else:
             total_length = int(total_length)
@@ -191,13 +192,12 @@ def download_chunks(file_handler=None, ret_buffer=None):
                         ret_buffer.extend(data)
                     if file_handler is not None:
                         file_handler.write(to_file_bytes(data))
-
-                    units = progress_units(download_size, total_length)
-                    progress = human_readable_progress(download_size, total_length)
-                    if last_progress != units:  # Avoid screen refresh if nothing has change
-                        if self.output:
+                    if self.output:
+                        units = progress_units(download_size, total_length)
+                        progress = human_readable_progress(download_size, total_length)
+                        if last_progress != units:  # Avoid screen refresh if nothing has change
                             print_progress(self.output, units, progress)
-                        last_progress = units
+                            last_progress = units
                 return download_size
 
             if file_path:
@@ -245,7 +245,8 @@ def call_with_retry(out, retry, retry_wait, method, *args, **kwargs):
             if counter == (retry - 1):
                 raise
             else:
-                msg = exception_message_safe(exc)
-                out.error(msg)
-                out.info("Waiting %d seconds to retry..." % retry_wait)
+                if out:
+                    msg = exception_message_safe(exc)
+                    out.error(msg)
+                    out.info("Waiting %d seconds to retry..." % retry_wait)
                 time.sleep(retry_wait)

conans/test/remote/rest_api_test.py

@@ -281,4 +281,4 @@ class MyConan(ConanFile):
         abs_paths[CONAN_MANIFEST] = os.path.join(tmp_dir, CONAN_MANIFEST)
         conan_digest.save(tmp_dir)
 
-        self.api.upload_recipe(conan_reference, abs_paths, retry, retry_wait, None)
+        self.api.upload_recipe(conan_reference, abs_paths, retry, retry_wait, None, None)