-
Notifications
You must be signed in to change notification settings - Fork 992
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Added untar progress similar to existing unzip #17519
base: develop2
Are you sure you want to change the base?
Changes from 2 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -262,6 +262,25 @@ def chdir(conanfile, newdir): | |
os.chdir(old_path) | ||
|
||
|
||
class _ProgressPrinter: | ||
def __init__(self, output, uncompressed_size: int, step: int = 1): | ||
self.enabled = hasattr(sys.stdout, "isatty") and sys.stdout.isatty() | ||
self.uncompressed_size = uncompressed_size | ||
self.percentage = 0 | ||
self.step = step | ||
self.output = output | ||
|
||
def print_progress(self, current_size: int): | ||
if not self.enabled: | ||
return | ||
txt_msg = "Unzipping %d %%" | ||
current_percentage = int(current_size * 100.0 / self.uncompressed_size) if self.uncompressed_size != 0 else 0 | ||
if current_percentage >= self.percentage + self.step: | ||
if 100 - current_percentage == self.step: | ||
current_percentage = 100 | ||
self.output.rewrite_line(txt_msg % current_percentage) | ||
self.percentage = current_percentage | ||
|
||
def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern=None, | ||
strip_root=False, extract_filter=None): | ||
""" | ||
|
@@ -289,7 +308,7 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern= | |
if (filename.endswith(".tar.gz") or filename.endswith(".tgz") or | ||
filename.endswith(".tbz2") or filename.endswith(".tar.bz2") or | ||
filename.endswith(".tar")): | ||
return untargz(filename, destination, pattern, strip_root, extract_filter) | ||
return untargz(filename, destination, pattern, strip_root, extract_filter, output) | ||
if filename.endswith(".gz"): | ||
target_name = filename[:-3] if destination == "." else destination | ||
target_dir = os.path.dirname(target_name) | ||
|
@@ -300,24 +319,11 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern= | |
shutil.copyfileobj(fin, fout) | ||
return | ||
if filename.endswith(".tar.xz") or filename.endswith(".txz"): | ||
return untargz(filename, destination, pattern, strip_root, extract_filter) | ||
return untargz(filename, destination, pattern, strip_root, extract_filter, output) | ||
|
||
import zipfile | ||
full_path = os.path.normpath(os.path.join(os.getcwd(), destination)) | ||
|
||
if hasattr(sys.stdout, "isatty") and sys.stdout.isatty(): | ||
def print_progress(the_size, uncomp_size): | ||
the_size = (the_size * 100.0 / uncomp_size) if uncomp_size != 0 else 0 | ||
txt_msg = "Unzipping %d %%" | ||
if the_size > print_progress.last_size + 1: | ||
output.rewrite_line(txt_msg % the_size) | ||
print_progress.last_size = the_size | ||
if int(the_size) == 99: | ||
output.rewrite_line(txt_msg % 100) | ||
else: | ||
def print_progress(_, __): | ||
pass | ||
|
||
with zipfile.ZipFile(filename, "r") as z: | ||
zip_info = z.infolist() | ||
if pattern: | ||
|
@@ -343,19 +349,19 @@ def print_progress(_, __): | |
output.info("Unzipping %s" % human_size(uncompress_size)) | ||
extracted_size = 0 | ||
|
||
print_progress.last_size = -1 | ||
progress_printer = _ProgressPrinter(output, uncompress_size) | ||
if platform.system() == "Windows": | ||
for file_ in zip_info: | ||
extracted_size += file_.file_size | ||
print_progress(extracted_size, uncompress_size) | ||
progress_printer.print_progress(extracted_size) | ||
try: | ||
z.extract(file_, full_path) | ||
except Exception as e: | ||
output.error(f"Error extract {file_.filename}\n{str(e)}", error_type="exception") | ||
else: # duplicated for, to avoid a platform check for each zipped file | ||
for file_ in zip_info: | ||
extracted_size += file_.file_size | ||
print_progress(extracted_size, uncompress_size) | ||
progress_printer.print_progress(extracted_size) | ||
try: | ||
z.extract(file_, full_path) | ||
if keep_permissions: | ||
|
@@ -367,11 +373,22 @@ def print_progress(_, __): | |
output.error(f"Error extract {file_.filename}\n{str(e)}", error_type="exception") | ||
output.writeln("") | ||
|
||
|
||
def untargz(filename, destination=".", pattern=None, strip_root=False, extract_filter=None): | ||
def untargz(filename, destination=".", pattern=None, strip_root=False, extract_filter=None, output=None): | ||
# NOT EXPOSED at `conan.tools.files` but used in tests | ||
import tarfile | ||
with tarfile.TarFile.open(filename, 'r:*') as tarredgzippedFile: | ||
import io | ||
class ProgressFileObject(io.FileIO): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We already have a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I've extracted that |
||
def __init__(self, path, *args, **kwargs): | ||
if output: | ||
self.progress_printer = _ProgressPrinter(output, os.path.getsize(path)) | ||
io.FileIO.__init__(self, path, *args, **kwargs) | ||
|
||
def read(self, size: int = -1) -> bytes: | ||
if output: | ||
self.progress_printer.print_progress(self.tell()) | ||
return io.FileIO.read(self, size) | ||
|
||
with tarfile.TarFile.open(fileobj=ProgressFileObject(filename), mode='r:*') as tarredgzippedFile: | ||
f = getattr(tarfile, f"{extract_filter}_filter", None) if extract_filter else None | ||
tarredgzippedFile.extraction_filter = f or (lambda member_, _: member_) | ||
if not pattern and not strip_root: | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Rewrites are not that great, specially for CI output, which result in endless and annoying logs if enabled, and not a trace at all if not enabled. The "timed-output" seems to be a good balance.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We were already using
output.rewrite_line
for the unzip progress but I agree it could cause some inconvenience.With this refactor, we could remove the
rewrite
by usingFileProgress
class.