Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added untar progress similar to existing unzip #17519

Open
wants to merge 4 commits into
base: develop2
Choose a base branch
from
Open
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 38 additions & 21 deletions conan/tools/files/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,25 @@ def chdir(conanfile, newdir):
os.chdir(old_path)


class _ProgressPrinter:
def __init__(self, output, uncompressed_size: int, step: int = 1):
self.enabled = hasattr(sys.stdout, "isatty") and sys.stdout.isatty()
self.uncompressed_size = uncompressed_size
self.percentage = 0
self.step = step
self.output = output

def print_progress(self, current_size: int):
if not self.enabled:
return
txt_msg = "Unzipping %d %%"
current_percentage = int(current_size * 100.0 / self.uncompressed_size) if self.uncompressed_size != 0 else 0
if current_percentage >= self.percentage + self.step:
if 100 - current_percentage == self.step:
current_percentage = 100
self.output.rewrite_line(txt_msg % current_percentage)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Rewrites are not that great, specially for CI output, which result in endless and annoying logs if enabled, and not a trace at all if not enabled. The "timed-output" seems to be a good balance.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We were already using output.rewrite_line for the unzip progress but I agree it could cause some inconvenience.
With this refactor, we could remove the rewrite by using FileProgress class.

self.percentage = current_percentage

def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern=None,
strip_root=False, extract_filter=None):
"""
Expand Down Expand Up @@ -289,7 +308,7 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern=
if (filename.endswith(".tar.gz") or filename.endswith(".tgz") or
filename.endswith(".tbz2") or filename.endswith(".tar.bz2") or
filename.endswith(".tar")):
return untargz(filename, destination, pattern, strip_root, extract_filter)
return untargz(filename, destination, pattern, strip_root, extract_filter, output)
if filename.endswith(".gz"):
target_name = filename[:-3] if destination == "." else destination
target_dir = os.path.dirname(target_name)
Expand All @@ -300,24 +319,11 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern=
shutil.copyfileobj(fin, fout)
return
if filename.endswith(".tar.xz") or filename.endswith(".txz"):
return untargz(filename, destination, pattern, strip_root, extract_filter)
return untargz(filename, destination, pattern, strip_root, extract_filter, output)

import zipfile
full_path = os.path.normpath(os.path.join(os.getcwd(), destination))

if hasattr(sys.stdout, "isatty") and sys.stdout.isatty():
def print_progress(the_size, uncomp_size):
the_size = (the_size * 100.0 / uncomp_size) if uncomp_size != 0 else 0
txt_msg = "Unzipping %d %%"
if the_size > print_progress.last_size + 1:
output.rewrite_line(txt_msg % the_size)
print_progress.last_size = the_size
if int(the_size) == 99:
output.rewrite_line(txt_msg % 100)
else:
def print_progress(_, __):
pass

with zipfile.ZipFile(filename, "r") as z:
zip_info = z.infolist()
if pattern:
Expand All @@ -343,19 +349,19 @@ def print_progress(_, __):
output.info("Unzipping %s" % human_size(uncompress_size))
extracted_size = 0

print_progress.last_size = -1
progress_printer = _ProgressPrinter(output, uncompress_size)
if platform.system() == "Windows":
for file_ in zip_info:
extracted_size += file_.file_size
print_progress(extracted_size, uncompress_size)
progress_printer.print_progress(extracted_size)
try:
z.extract(file_, full_path)
except Exception as e:
output.error(f"Error extract {file_.filename}\n{str(e)}", error_type="exception")
else: # duplicated for, to avoid a platform check for each zipped file
for file_ in zip_info:
extracted_size += file_.file_size
print_progress(extracted_size, uncompress_size)
progress_printer.print_progress(extracted_size)
try:
z.extract(file_, full_path)
if keep_permissions:
Expand All @@ -367,11 +373,22 @@ def print_progress(_, __):
output.error(f"Error extract {file_.filename}\n{str(e)}", error_type="exception")
output.writeln("")


def untargz(filename, destination=".", pattern=None, strip_root=False, extract_filter=None):
def untargz(filename, destination=".", pattern=None, strip_root=False, extract_filter=None, output=None):
# NOT EXPOSED at `conan.tools.files` but used in tests
import tarfile
with tarfile.TarFile.open(filename, 'r:*') as tarredgzippedFile:
import io
class ProgressFileObject(io.FileIO):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We already have a class FileProgress in the FileUploader, maybe it is the time to extract it and reuse it. Or at least copy it as-is, and leave it for a later refactor. But it would be that at least the initial implementation is the same.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've extracted that FileProgress class and adapted it to different scenarios! See https://github.com/conan-io/conan/pull/17519/files#diff-9afbf22c7cefd3f7b642e9b1b988ba829d3c36e3092bfb6140d01ad0bba4c571R537

def __init__(self, path, *args, **kwargs):
if output:
self.progress_printer = _ProgressPrinter(output, os.path.getsize(path))
io.FileIO.__init__(self, path, *args, **kwargs)

def read(self, size: int = -1) -> bytes:
if output:
self.progress_printer.print_progress(self.tell())
return io.FileIO.read(self, size)

with tarfile.TarFile.open(fileobj=ProgressFileObject(filename), mode='r:*') as tarredgzippedFile:
f = getattr(tarfile, f"{extract_filter}_filter", None) if extract_filter else None
tarredgzippedFile.extraction_filter = f or (lambda member_, _: member_)
if not pattern and not strip_root:
Expand Down
Loading