diff --git a/conans/client/cmd/uploader.py b/conans/client/cmd/uploader.py index 0ebf746f372..ce5f0363b2e 100644 --- a/conans/client/cmd/uploader.py +++ b/conans/client/cmd/uploader.py @@ -95,10 +95,12 @@ def _upload(self, conan_file, conan_ref, packages_ids, retry, retry_wait, reference=conan_ref, remote=upload_remote) if policy != UPLOAD_POLICY_FORCE: - self._check_recipe_date(conan_ref, upload_remote) + remote_manifest = self._check_recipe_date(conan_ref, upload_remote) + else: + remote_manifest = None self._user_io.out.info("Uploading %s to remote '%s'" % (str(conan_ref), upload_remote.name)) - self._upload_recipe(conan_ref, retry, retry_wait, policy, upload_remote) + self._upload_recipe(conan_ref, retry, retry_wait, policy, upload_remote, remote_manifest) recorder.add_recipe(str(conan_ref), upload_remote.name, upload_remote.url) @@ -122,7 +124,7 @@ def _upload(self, conan_file, conan_ref, packages_ids, retry, retry_wait, self._plugin_manager.execute("post_upload", conanfile_path=conanfile_path, reference=conan_ref, remote=upload_remote) - def _upload_recipe(self, conan_reference, retry, retry_wait, policy, remote): + def _upload_recipe(self, conan_reference, retry, retry_wait, policy, remote, remote_manifest): conan_file_path = self._client_cache.conanfile(conan_reference) current_remote = self._registry.get_recipe_remote(conan_reference) if remote != current_remote: @@ -130,7 +132,7 @@ def _upload_recipe(self, conan_reference, retry, retry_wait, policy, remote): complete_recipe_sources(self._remote_manager, self._client_cache, self._registry, conanfile, conan_reference) result = self._remote_manager.upload_recipe(conan_reference, remote, retry, retry_wait, - policy=policy) + policy=policy, remote_manifest=remote_manifest) return result def _upload_package(self, package_ref, index=1, total=1, retry=None, retry_wait=None, @@ -159,3 +161,5 @@ def _check_recipe_date(self, conan_ref, remote): raise ConanException("Remote recipe is newer than local recipe: " "\n Remote date: %s\n Local date: %s" % (remote_recipe_manifest.time, local_manifest.time)) + + return remote_recipe_manifest diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index bee67955893..0add6305082 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -41,7 +41,8 @@ def __init__(self, client_cache, auth_manager, output, plugin_manager): self._auth_manager = auth_manager self._plugin_manager = plugin_manager - def upload_recipe(self, conan_reference, remote, retry, retry_wait, policy=None): + + def upload_recipe(self, conan_reference, remote, retry, retry_wait, policy, remote_manifest): conanfile_path = self._client_cache.conanfile(conan_reference) self._plugin_manager.execute("pre_upload_recipe", conanfile_path=conanfile_path, reference=conan_reference, remote=remote) @@ -66,7 +67,7 @@ def upload_recipe(self, conan_reference, remote, retry, retry_wait, policy=None) return None ret, new_ref = self._call_remote(remote, "upload_recipe", conan_reference, the_files, retry, - retry_wait, policy) + retry_wait, policy, remote_manifest) duration = time.time() - t1 log_recipe_upload(new_ref, duration, the_files, remote.name) if ret: diff --git a/conans/client/rest/auth_manager.py b/conans/client/rest/auth_manager.py index 8093a74bebd..f01d0a251ba 100644 --- a/conans/client/rest/auth_manager.py +++ b/conans/client/rest/auth_manager.py @@ -125,9 +125,9 @@ def set_custom_headers(self, username): # ######### CONAN API METHODS ########## @input_credentials_if_unauthorized - def upload_recipe(self, conan_reference, the_files, retry, retry_wait, policy): + def upload_recipe(self, conan_reference, the_files, retry, retry_wait, policy, remote_manifest): return self._rest_client.upload_recipe(conan_reference, the_files, retry, retry_wait, - policy) + policy, remote_manifest) @input_credentials_if_unauthorized def upload_package(self, package_reference, the_files, retry, retry_wait, policy): diff --git a/conans/client/rest/rest_client.py b/conans/client/rest/rest_client.py index ed7f7fe745e..5ce6be0df88 100644 --- a/conans/client/rest/rest_client.py +++ b/conans/client/rest/rest_client.py @@ -61,9 +61,9 @@ def get_package(self, package_reference, dest_folder): def get_path(self, conan_reference, package_id, path): return self._get_api().get_path(conan_reference, package_id, path) - def upload_recipe(self, conan_reference, the_files, retry, retry_wait, no_overwrite): + def upload_recipe(self, conan_reference, the_files, retry, retry_wait, policy, remote_manifest): return self._get_api().upload_recipe(conan_reference, the_files, retry, retry_wait, - no_overwrite) + policy, remote_manifest) def upload_package(self, package_reference, the_files, retry, retry_wait, no_overwrite): return self._get_api().upload_package(package_reference, the_files, retry, retry_wait, diff --git a/conans/client/rest/rest_client_common.py b/conans/client/rest/rest_client_common.py index 570bbf779b0..002ea336b8d 100644 --- a/conans/client/rest/rest_client_common.py +++ b/conans/client/rest/rest_client_common.py @@ -141,7 +141,8 @@ def get_json(self, url, data=None): raise ConanException("Unexpected server response %s" % result) return result - def upload_recipe(self, conan_reference, the_files, retry, retry_wait, policy): + def upload_recipe(self, conan_reference, the_files, retry, retry_wait, policy, + remote_manifest): """ the_files: dict with relative_path: content """ @@ -151,7 +152,7 @@ def upload_recipe(self, conan_reference, the_files, retry, retry_wait, policy): remote_snapshot, conan_reference = self._get_recipe_snapshot(conan_reference) if remote_snapshot and policy != UPLOAD_POLICY_FORCE: - remote_manifest = self.get_conan_manifest(conan_reference) + remote_manifest = remote_manifest or self.get_conan_manifest(conan_reference) local_manifest = FileTreeManifest.loads(load(the_files["conanmanifest.txt"])) if remote_manifest == local_manifest: diff --git a/conans/client/rest/rest_client_v1.py b/conans/client/rest/rest_client_v1.py index e67852b42f7..1d9f6ef8300 100644 --- a/conans/client/rest/rest_client_v1.py +++ b/conans/client/rest/rest_client_v1.py @@ -12,10 +12,8 @@ from conans.model.ref import PackageReference from conans.paths import CONAN_MANIFEST, CONANINFO, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, \ PACKAGE_TGZ_NAME -from conans.util.files import decode_text, load +from conans.util.files import decode_text from conans.util.log import logger -from conans.client.cmd.uploader import UPLOAD_POLICY_NO_OVERWRITE,\ - UPLOAD_POLICY_NO_OVERWRITE_RECIPE, UPLOAD_POLICY_FORCE def complete_url(base_url, url): @@ -33,20 +31,23 @@ class RestV1Methods(RestCommonMethods): def remote_api_url(self): return "%s/v1" % self.remote_url.rstrip("/") - def _download_files(self, file_urls): + def _download_files(self, file_urls, quiet=False): """ :param: file_urls is a dict with {filename: url} Its a generator, so it yields elements for memory performance """ - downloader = Downloader(self.requester, self._output, self.verify_ssl) + output = self._output if not quiet else None + downloader = Downloader(self.requester, output, self.verify_ssl) # Take advantage of filenames ordering, so that conan_package.tgz and conan_export.tgz # can be < conanfile, conaninfo, and sent always the last, so smaller files go first for filename, resource_url in sorted(file_urls.items(), reverse=True): - self._output.writeln("Downloading %s" % filename) + if output: + output.writeln("Downloading %s" % filename) auth, _ = self._file_server_capabilities(resource_url) contents = downloader.download(resource_url, auth=auth) - self._output.writeln("") + if output: + output.writeln("") yield os.path.normpath(filename), contents def _file_server_capabilities(self, resource_url): @@ -69,7 +70,7 @@ def get_conan_manifest(self, conan_reference): urls = self._get_file_to_url_dict(url) # Get the digest - contents = self._download_files(urls) + contents = self._download_files(urls, quiet=True) # Unroll generator and decode shas (plain text) contents = {key: decode_text(value) for key, value in dict(contents).items()} return FileTreeManifest.loads(contents[CONAN_MANIFEST]) @@ -82,7 +83,7 @@ def get_package_manifest(self, package_reference): urls = self._get_file_to_url_dict(url) # Get the digest - contents = self._download_files(urls) + contents = self._download_files(urls, quiet=True) # Unroll generator and decode shas (plain text) contents = {key: decode_text(value) for key, value in dict(contents).items()} return FileTreeManifest.loads(contents[CONAN_MANIFEST]) diff --git a/conans/client/rest/uploader_downloader.py b/conans/client/rest/uploader_downloader.py index 7f3a7a5a93a..d1d6ffd8780 100644 --- a/conans/client/rest/uploader_downloader.py +++ b/conans/client/rest/uploader_downloader.py @@ -169,9 +169,10 @@ def _download_data(self, response, file_path): if not file_path: ret += response.content else: - total_length = len(response.content) - progress = human_readable_progress(total_length, total_length) - print_progress(self.output, 50, progress) + if self.output: + total_length = len(response.content) + progress = human_readable_progress(total_length, total_length) + print_progress(self.output, 50, progress) save_append(file_path, response.content) else: total_length = int(total_length) @@ -191,13 +192,12 @@ def download_chunks(file_handler=None, ret_buffer=None): ret_buffer.extend(data) if file_handler is not None: file_handler.write(to_file_bytes(data)) - - units = progress_units(download_size, total_length) - progress = human_readable_progress(download_size, total_length) - if last_progress != units: # Avoid screen refresh if nothing has change - if self.output: + if self.output: + units = progress_units(download_size, total_length) + progress = human_readable_progress(download_size, total_length) + if last_progress != units: # Avoid screen refresh if nothing has change print_progress(self.output, units, progress) - last_progress = units + last_progress = units return download_size if file_path: @@ -245,7 +245,8 @@ def call_with_retry(out, retry, retry_wait, method, *args, **kwargs): if counter == (retry - 1): raise else: - msg = exception_message_safe(exc) - out.error(msg) - out.info("Waiting %d seconds to retry..." % retry_wait) + if out: + msg = exception_message_safe(exc) + out.error(msg) + out.info("Waiting %d seconds to retry..." % retry_wait) time.sleep(retry_wait) diff --git a/conans/test/remote/rest_api_test.py b/conans/test/remote/rest_api_test.py index 6a93da50701..2e6cce41040 100644 --- a/conans/test/remote/rest_api_test.py +++ b/conans/test/remote/rest_api_test.py @@ -281,4 +281,4 @@ class MyConan(ConanFile): abs_paths[CONAN_MANIFEST] = os.path.join(tmp_dir, CONAN_MANIFEST) conan_digest.save(tmp_dir) - self.api.upload_recipe(conan_reference, abs_paths, retry, retry_wait, None) + self.api.upload_recipe(conan_reference, abs_paths, retry, retry_wait, None, None)