From 130fe9d03f3793ac682177e3522cf125eb6aeb14 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Mon, 23 Dec 2024 13:21:47 +0100 Subject: [PATCH 1/5] Added untar progress similar to existing unzip --- conan/tools/files/files.py | 59 ++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py index b3a05026b57..835a7b77394 100644 --- a/conan/tools/files/files.py +++ b/conan/tools/files/files.py @@ -262,6 +262,25 @@ def chdir(conanfile, newdir): os.chdir(old_path) +class _ProgressPrinter: + def __init__(self, output, uncompressed_size: int, step: int = 1): + self.enabled = hasattr(sys.stdout, "isatty") and sys.stdout.isatty() + self.uncompressed_size = uncompressed_size + self.percentage = 0 + self.step = step + self.output = output + + def print_progress(self, current_size: int): + if not self.enabled: + return + txt_msg = "Unzipping %d %%" + current_percentage = int(current_size * 100.0 / self.uncompressed_size) if self.uncompressed_size != 0 else 0 + if current_percentage >= self.percentage + self.step: + if 100 - current_percentage == self.step: + current_percentage = 100 + self.output.rewrite_line(txt_msg % current_percentage) + self.percentage = current_percentage + def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern=None, strip_root=False, extract_filter=None): """ @@ -289,7 +308,7 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern= if (filename.endswith(".tar.gz") or filename.endswith(".tgz") or filename.endswith(".tbz2") or filename.endswith(".tar.bz2") or filename.endswith(".tar")): - return untargz(filename, destination, pattern, strip_root, extract_filter) + return untargz(filename, destination, pattern, strip_root, extract_filter, output) if filename.endswith(".gz"): target_name = filename[:-3] if destination == "." else destination target_dir = os.path.dirname(target_name) @@ -300,24 +319,11 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern= shutil.copyfileobj(fin, fout) return if filename.endswith(".tar.xz") or filename.endswith(".txz"): - return untargz(filename, destination, pattern, strip_root, extract_filter) + return untargz(filename, destination, pattern, strip_root, extract_filter, output) import zipfile full_path = os.path.normpath(os.path.join(os.getcwd(), destination)) - if hasattr(sys.stdout, "isatty") and sys.stdout.isatty(): - def print_progress(the_size, uncomp_size): - the_size = (the_size * 100.0 / uncomp_size) if uncomp_size != 0 else 0 - txt_msg = "Unzipping %d %%" - if the_size > print_progress.last_size + 1: - output.rewrite_line(txt_msg % the_size) - print_progress.last_size = the_size - if int(the_size) == 99: - output.rewrite_line(txt_msg % 100) - else: - def print_progress(_, __): - pass - with zipfile.ZipFile(filename, "r") as z: zip_info = z.infolist() if pattern: @@ -343,11 +349,11 @@ def print_progress(_, __): output.info("Unzipping %s" % human_size(uncompress_size)) extracted_size = 0 - print_progress.last_size = -1 + progress_printer = _ProgressPrinter(output, uncompress_size) if platform.system() == "Windows": for file_ in zip_info: extracted_size += file_.file_size - print_progress(extracted_size, uncompress_size) + progress_printer.print_progress(extracted_size) try: z.extract(file_, full_path) except Exception as e: @@ -355,7 +361,7 @@ def print_progress(_, __): else: # duplicated for, to avoid a platform check for each zipped file for file_ in zip_info: extracted_size += file_.file_size - print_progress(extracted_size, uncompress_size) + progress_printer.print_progress(extracted_size) try: z.extract(file_, full_path) if keep_permissions: @@ -367,11 +373,22 @@ def print_progress(_, __): output.error(f"Error extract {file_.filename}\n{str(e)}", error_type="exception") output.writeln("") - -def untargz(filename, destination=".", pattern=None, strip_root=False, extract_filter=None): +def untargz(filename, destination=".", pattern=None, strip_root=False, extract_filter=None, output=None): # NOT EXPOSED at `conan.tools.files` but used in tests import tarfile - with tarfile.TarFile.open(filename, 'r:*') as tarredgzippedFile: + import io + class ProgressFileObject(io.FileIO): + def __init__(self, path, *args, **kwargs): + if output: + self.progress_printer = _ProgressPrinter(output, os.path.getsize(path)) + io.FileIO.__init__(self, path, *args, **kwargs) + + def read(self, size: int = -1, /) -> bytes: + if output: + self.progress_printer.print_progress(self.tell()) + return io.FileIO.read(self, size) + + with tarfile.TarFile.open(fileobj=ProgressFileObject(filename), mode='r:*') as tarredgzippedFile: f = getattr(tarfile, f"{extract_filter}_filter", None) if extract_filter else None tarredgzippedFile.extraction_filter = f or (lambda member_, _: member_) if not pattern and not strip_root: From 375dbf112d12491deec6641dbc0cdae08cd5bd78 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Mon, 23 Dec 2024 13:40:03 +0100 Subject: [PATCH 2/5] Python 3.6 compatible --- conan/tools/files/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py index 835a7b77394..ce4567ed7da 100644 --- a/conan/tools/files/files.py +++ b/conan/tools/files/files.py @@ -383,7 +383,7 @@ def __init__(self, path, *args, **kwargs): self.progress_printer = _ProgressPrinter(output, os.path.getsize(path)) io.FileIO.__init__(self, path, *args, **kwargs) - def read(self, size: int = -1, /) -> bytes: + def read(self, size: int = -1) -> bytes: if output: self.progress_printer.print_progress(self.tell()) return io.FileIO.read(self, size) From 5d1306a1265ff4b0f7bc501e10354f73acd13f41 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Tue, 24 Dec 2024 13:42:10 +0100 Subject: [PATCH 3/5] Refactor FileProgress class and use it on uncompress --- conan/tools/files/files.py | 56 ++++++++++------------------- conans/client/rest/file_uploader.py | 26 ++------------ 2 files changed, 21 insertions(+), 61 deletions(-) diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py index ce4567ed7da..ea41e3c1137 100644 --- a/conan/tools/files/files.py +++ b/conan/tools/files/files.py @@ -3,12 +3,13 @@ import platform import shutil import subprocess -import sys +import io from contextlib import contextmanager from fnmatch import fnmatch from shutil import which +from conan.api.output import TimedOutput from conans.client.downloaders.caching_file_downloader import SourcesCachingDownloader from conan.errors import ConanException from conans.util.files import rmdir as _internal_rmdir, human_size, check_with_algorithm_sum @@ -261,26 +262,6 @@ def chdir(conanfile, newdir): finally: os.chdir(old_path) - -class _ProgressPrinter: - def __init__(self, output, uncompressed_size: int, step: int = 1): - self.enabled = hasattr(sys.stdout, "isatty") and sys.stdout.isatty() - self.uncompressed_size = uncompressed_size - self.percentage = 0 - self.step = step - self.output = output - - def print_progress(self, current_size: int): - if not self.enabled: - return - txt_msg = "Unzipping %d %%" - current_percentage = int(current_size * 100.0 / self.uncompressed_size) if self.uncompressed_size != 0 else 0 - if current_percentage >= self.percentage + self.step: - if 100 - current_percentage == self.step: - current_percentage = 100 - self.output.rewrite_line(txt_msg % current_percentage) - self.percentage = current_percentage - def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern=None, strip_root=False, extract_filter=None): """ @@ -324,7 +305,7 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern= import zipfile full_path = os.path.normpath(os.path.join(os.getcwd(), destination)) - with zipfile.ZipFile(filename, "r") as z: + with zipfile.ZipFile(FileProgress(filename, msg="Unzipping", mode="r")) as z: zip_info = z.infolist() if pattern: zip_info = [zi for zi in zip_info if fnmatch(zi.filename, pattern)] @@ -349,11 +330,9 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern= output.info("Unzipping %s" % human_size(uncompress_size)) extracted_size = 0 - progress_printer = _ProgressPrinter(output, uncompress_size) if platform.system() == "Windows": for file_ in zip_info: extracted_size += file_.file_size - progress_printer.print_progress(extracted_size) try: z.extract(file_, full_path) except Exception as e: @@ -361,7 +340,6 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern= else: # duplicated for, to avoid a platform check for each zipped file for file_ in zip_info: extracted_size += file_.file_size - progress_printer.print_progress(extracted_size) try: z.extract(file_, full_path) if keep_permissions: @@ -376,19 +354,7 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern= def untargz(filename, destination=".", pattern=None, strip_root=False, extract_filter=None, output=None): # NOT EXPOSED at `conan.tools.files` but used in tests import tarfile - import io - class ProgressFileObject(io.FileIO): - def __init__(self, path, *args, **kwargs): - if output: - self.progress_printer = _ProgressPrinter(output, os.path.getsize(path)) - io.FileIO.__init__(self, path, *args, **kwargs) - - def read(self, size: int = -1) -> bytes: - if output: - self.progress_printer.print_progress(self.tell()) - return io.FileIO.read(self, size) - - with tarfile.TarFile.open(fileobj=ProgressFileObject(filename), mode='r:*') as tarredgzippedFile: + with tarfile.TarFile.open(fileobj=FileProgress(filename, msg="Uncompressing"), mode='r:*') as tarredgzippedFile: f = getattr(tarfile, f"{extract_filter}_filter", None) if extract_filter else None tarredgzippedFile.extraction_filter = f or (lambda member_, _: member_) if not pattern and not strip_root: @@ -568,3 +534,17 @@ def move_folder_contents(conanfile, src_folder, dst_folder): os.rmdir(src_folder) except OSError: pass + + +class FileProgress(io.FileIO): + def __init__(self, path: str, msg: str = "Uploading", report_interval: float = 10, *args, **kwargs): + super().__init__(path, *args, **kwargs) + self._total_size = os.path.getsize(path) + self._filename = os.path.basename(path) + self._t = TimedOutput(interval=report_interval) + self.msg = msg + + def read(self, size: int = -1) -> bytes: + current_percentage = int(self.tell() * 100.0 / self._total_size) if self._total_size != 0 else 0 + self._t.info(f"{self.msg} {self._filename}: {current_percentage}%") + return super().read(size) diff --git a/conans/client/rest/file_uploader.py b/conans/client/rest/file_uploader.py index 429150581b0..59c98c47942 100644 --- a/conans/client/rest/file_uploader.py +++ b/conans/client/rest/file_uploader.py @@ -1,7 +1,7 @@ -import os import time -from conan.api.output import ConanOutput, TimedOutput +from conan.api.output import ConanOutput +from conan.tools.files.files import FileProgress from conans.client.rest import response_to_str from conan.internal.errors import InternalErrorException, RequestErrorException, AuthenticationException, \ ForbiddenException, NotFoundException @@ -81,27 +81,7 @@ def upload(self, url, abs_path, auth=None, dedup=False, retry=None, retry_wait=N time.sleep(retry_wait) def _upload_file(self, url, abs_path, headers, auth, ref): - class FileProgress: # Wrapper just to provide an upload progress every 10 seconds - def __init__(self, f, total_size): - self._f = f - self._total = total_size - self._name = os.path.basename(f.name) - self._t = TimedOutput(interval=10) - self._read = 0 - - def __getattr__(self, item): - return getattr(self._f, item) - - def read(self, n=-1): - read_bytes = self._f.read(n) - self._read += len(read_bytes) - self._t.info(f"{ref}: Uploading {self._name}: {int(self._read*100/self._total)}%") - return read_bytes - - filesize = os.path.getsize(abs_path) - with open(abs_path, mode='rb') as file_handler: - big_file = filesize > 100000000 # 100 MB - file_handler = FileProgress(file_handler, filesize) if big_file else file_handler + with FileProgress(abs_path, mode='rb', msg=f"{ref}: Uploading") as file_handler: try: response = self._requester.put(url, data=file_handler, verify=self._verify_ssl, headers=headers, auth=auth, From b7294e6539b8e446fb0cb1252ba4b542ee40f44f Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Tue, 31 Dec 2024 10:23:55 +0100 Subject: [PATCH 4/5] Restored prev FileProgress location and use with --- conan/tools/files/files.py | 27 ++++++--------------------- conans/client/rest/file_uploader.py | 19 +++++++++++++++++-- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py index ea41e3c1137..9e32383eb74 100644 --- a/conan/tools/files/files.py +++ b/conan/tools/files/files.py @@ -3,15 +3,14 @@ import platform import shutil import subprocess -import io from contextlib import contextmanager from fnmatch import fnmatch from shutil import which -from conan.api.output import TimedOutput from conans.client.downloaders.caching_file_downloader import SourcesCachingDownloader from conan.errors import ConanException +from conans.client.rest.file_uploader import FileProgress from conans.util.files import rmdir as _internal_rmdir, human_size, check_with_algorithm_sum @@ -289,7 +288,7 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern= if (filename.endswith(".tar.gz") or filename.endswith(".tgz") or filename.endswith(".tbz2") or filename.endswith(".tar.bz2") or filename.endswith(".tar")): - return untargz(filename, destination, pattern, strip_root, extract_filter, output) + return untargz(filename, destination, pattern, strip_root, extract_filter) if filename.endswith(".gz"): target_name = filename[:-3] if destination == "." else destination target_dir = os.path.dirname(target_name) @@ -300,12 +299,12 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern= shutil.copyfileobj(fin, fout) return if filename.endswith(".tar.xz") or filename.endswith(".txz"): - return untargz(filename, destination, pattern, strip_root, extract_filter, output) + return untargz(filename, destination, pattern, strip_root, extract_filter) import zipfile full_path = os.path.normpath(os.path.join(os.getcwd(), destination)) - with zipfile.ZipFile(FileProgress(filename, msg="Unzipping", mode="r")) as z: + with FileProgress(filename, msg="Unzipping", mode="r") as file, zipfile.ZipFile(file) as z: zip_info = z.infolist() if pattern: zip_info = [zi for zi in zip_info if fnmatch(zi.filename, pattern)] @@ -351,10 +350,10 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern= output.error(f"Error extract {file_.filename}\n{str(e)}", error_type="exception") output.writeln("") -def untargz(filename, destination=".", pattern=None, strip_root=False, extract_filter=None, output=None): +def untargz(filename, destination=".", pattern=None, strip_root=False, extract_filter=None): # NOT EXPOSED at `conan.tools.files` but used in tests import tarfile - with tarfile.TarFile.open(fileobj=FileProgress(filename, msg="Uncompressing"), mode='r:*') as tarredgzippedFile: + with FileProgress(filename, msg="Uncompressing") as fileobj, tarfile.TarFile.open(fileobj=fileobj, mode='r:*') as tarredgzippedFile: f = getattr(tarfile, f"{extract_filter}_filter", None) if extract_filter else None tarredgzippedFile.extraction_filter = f or (lambda member_, _: member_) if not pattern and not strip_root: @@ -534,17 +533,3 @@ def move_folder_contents(conanfile, src_folder, dst_folder): os.rmdir(src_folder) except OSError: pass - - -class FileProgress(io.FileIO): - def __init__(self, path: str, msg: str = "Uploading", report_interval: float = 10, *args, **kwargs): - super().__init__(path, *args, **kwargs) - self._total_size = os.path.getsize(path) - self._filename = os.path.basename(path) - self._t = TimedOutput(interval=report_interval) - self.msg = msg - - def read(self, size: int = -1) -> bytes: - current_percentage = int(self.tell() * 100.0 / self._total_size) if self._total_size != 0 else 0 - self._t.info(f"{self.msg} {self._filename}: {current_percentage}%") - return super().read(size) diff --git a/conans/client/rest/file_uploader.py b/conans/client/rest/file_uploader.py index 59c98c47942..304ddc4e2a6 100644 --- a/conans/client/rest/file_uploader.py +++ b/conans/client/rest/file_uploader.py @@ -1,7 +1,8 @@ +import io +import os import time -from conan.api.output import ConanOutput -from conan.tools.files.files import FileProgress +from conan.api.output import ConanOutput, TimedOutput from conans.client.rest import response_to_str from conan.internal.errors import InternalErrorException, RequestErrorException, AuthenticationException, \ ForbiddenException, NotFoundException @@ -93,3 +94,17 @@ def _upload_file(self, url, abs_path, headers, auth, ref): raise except Exception as exc: raise ConanException(exc) + + +class FileProgress(io.FileIO): + def __init__(self, path: str, msg: str = "Uploading", report_interval: float = 10, *args, **kwargs): + super().__init__(path, *args, **kwargs) + self._total_size = os.path.getsize(path) + self._filename = os.path.basename(path) + self._t = TimedOutput(interval=report_interval) + self.msg = msg + + def read(self, size: int = -1) -> bytes: + current_percentage = int(self.tell() * 100.0 / self._total_size) if self._total_size != 0 else 0 + self._t.info(f"{self.msg} {self._filename}: {current_percentage}%") + return super().read(size) From 05f159bc57efee15011a62b70380cb9f9039e30b Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Fri, 17 Jan 2025 11:54:53 +0100 Subject: [PATCH 5/5] Removed tell() call and added size limitation --- conans/client/rest/file_uploader.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/conans/client/rest/file_uploader.py b/conans/client/rest/file_uploader.py index 304ddc4e2a6..1417cdf92f1 100644 --- a/conans/client/rest/file_uploader.py +++ b/conans/client/rest/file_uploader.py @@ -97,14 +97,19 @@ def _upload_file(self, url, abs_path, headers, auth, ref): class FileProgress(io.FileIO): - def __init__(self, path: str, msg: str = "Uploading", report_interval: float = 10, *args, **kwargs): + def __init__(self, path: str, msg: str = "Uploading", report_interval: float = 1 , *args, **kwargs): super().__init__(path, *args, **kwargs) self._total_size = os.path.getsize(path) self._filename = os.path.basename(path) - self._t = TimedOutput(interval=report_interval) + # Report only on big sizes (>100MB) + self._reporter = TimedOutput(interval=report_interval) if self._total_size > 100_000_000 else None + self._bytes_read = 0 self.msg = msg def read(self, size: int = -1) -> bytes: - current_percentage = int(self.tell() * 100.0 / self._total_size) if self._total_size != 0 else 0 - self._t.info(f"{self.msg} {self._filename}: {current_percentage}%") - return super().read(size) + block = super().read(size) + self._bytes_read += len(block) + if self._reporter: + current_percentage = int(self._bytes_read * 100.0 / self._total_size) if self._total_size != 0 else 0 + self._reporter.info(f"{self.msg} {self._filename}: {current_percentage}%") + return block