From 14e1b81b407064cdbbb05a0c4b3638f0e8cacd41 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 23 Jan 2019 17:43:51 +0100 Subject: [PATCH 1/5] POC of removing tqdm --- conans/util/progress_bar.py | 52 +++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/conans/util/progress_bar.py b/conans/util/progress_bar.py index 51f64f4b205..e39a7507872 100644 --- a/conans/util/progress_bar.py +++ b/conans/util/progress_bar.py @@ -3,11 +3,49 @@ from contextlib import contextmanager from tqdm import tqdm +from conans.client.rest.uploader_downloader import print_progress,\ + progress_units, human_readable_progress TIMEOUT_BEAT_SECONDS = 30 TIMEOUT_BEAT_CHARACTER = '.' +class _FileReaderWithConanProgressBar(object): + + def __init__(self, fileobj, output, desc=None): + self._output = output + self._fileobj = fileobj + self.seek(0, os.SEEK_END) + self.totalsize = self.tell() + self.seek(0) + self._last = 0 + + def seekable(self): + return self._fileobj.seekable() + + def seek(self, *args, **kwargs): + return self._fileobj.seek(*args, **kwargs) + + def tell(self): + return self._fileobj.tell() + + def read(self, size): + ret = self._fileobj.read(size) + diff = self.tell() + if diff - self._last > self.totalsize/200.0: + self._last = diff + if self._output.is_terminal: + units = progress_units(diff, self.totalsize) + progress = human_readable_progress(diff, self.totalsize) + print_progress(self._output, units, progress=progress) + return ret + + def pb_close(self): + progress = human_readable_progress(self.totalsize, self.totalsize) + print_progress(self._output, progress_units(100, 100), progress) + self._output.writeln("") + + class _FileReaderWithProgressBar(object): tqdm_defaults = {'unit': 'B', @@ -68,9 +106,13 @@ def flush(self): @contextmanager def open_binary(path, output, **kwargs): + output.writeln("Extracting %s" % os.path.basename(path)) with open(path, mode='rb') as f: - file_wrapped = _FileReaderWithProgressBar(f, output=output, **kwargs) - yield file_wrapped - file_wrapped.pb_close() - if not output.is_terminal: - output.write("\n") + if output.is_terminal: + file_wrapped = _FileReaderWithConanProgressBar(f, output=output, **kwargs) + yield file_wrapped + file_wrapped.pb_close() + if not output.is_terminal: + output.write("\n") + else: + yield f From 9a694679cf3ad2755c6d8e2ad437417296c51397 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 24 Jan 2019 10:30:36 +0100 Subject: [PATCH 2/5] iteration over IOError for stream output --- conans/client/output.py | 26 ++++++++------- conans/util/progress_bar.py | 65 +++++-------------------------------- 2 files changed, 22 insertions(+), 69 deletions(-) diff --git a/conans/client/output.py b/conans/client/output.py index 9c18a494155..e64d81a8980 100644 --- a/conans/client/output.py +++ b/conans/client/output.py @@ -58,18 +58,20 @@ def write(self, data, front=None, back=None, newline=False): data = decode_text(data) # Keep python 2 compatibility if self._color and (front or back): - color = "%s%s" % (front or '', back or '') - end = (Style.RESET_ALL + "\n") if newline else Style.RESET_ALL # @UndefinedVariable - data = "%s%s%s" % (color, data, end) - else: - if newline: - data = "%s\n" % data - - try: - self._stream.write(data) - except UnicodeError: - data = data.encode("utf8").decode("ascii", "ignore") - self._stream.write(data) + data = "%s%s%s%s" % (front or '', back or '', data, Style.RESET_ALL) + if newline: + data = "%s\n" % data + + for _ in range(3): + try: + self._stream.write(data) + break + except IOError: + import time + time.sleep(0.02) + except UnicodeError: + data = data.encode("utf8").decode("ascii", "ignore") + self._stream.flush() def info(self, data): diff --git a/conans/util/progress_bar.py b/conans/util/progress_bar.py index e39a7507872..295fb2f7c78 100644 --- a/conans/util/progress_bar.py +++ b/conans/util/progress_bar.py @@ -1,51 +1,12 @@ - import os from contextlib import contextmanager from tqdm import tqdm -from conans.client.rest.uploader_downloader import print_progress,\ - progress_units, human_readable_progress TIMEOUT_BEAT_SECONDS = 30 TIMEOUT_BEAT_CHARACTER = '.' -class _FileReaderWithConanProgressBar(object): - - def __init__(self, fileobj, output, desc=None): - self._output = output - self._fileobj = fileobj - self.seek(0, os.SEEK_END) - self.totalsize = self.tell() - self.seek(0) - self._last = 0 - - def seekable(self): - return self._fileobj.seekable() - - def seek(self, *args, **kwargs): - return self._fileobj.seek(*args, **kwargs) - - def tell(self): - return self._fileobj.tell() - - def read(self, size): - ret = self._fileobj.read(size) - diff = self.tell() - if diff - self._last > self.totalsize/200.0: - self._last = diff - if self._output.is_terminal: - units = progress_units(diff, self.totalsize) - progress = human_readable_progress(diff, self.totalsize) - print_progress(self._output, units, progress=progress) - return ret - - def pb_close(self): - progress = human_readable_progress(self.totalsize, self.totalsize) - print_progress(self._output, progress_units(100, 100), progress) - self._output.writeln("") - - class _FileReaderWithProgressBar(object): tqdm_defaults = {'unit': 'B', @@ -56,17 +17,15 @@ class _FileReaderWithProgressBar(object): def __init__(self, fileobj, output, desc=None): pb_kwargs = self.tqdm_defaults.copy() - self._ori_output = output # If there is no terminal, just print a beat every TIMEOUT_BEAT seconds. if not output.is_terminal: output = _NoTerminalOutput(output) pb_kwargs['mininterval'] = TIMEOUT_BEAT_SECONDS - self._output = output self._fileobj = fileobj self.seek(0, os.SEEK_END) - self._pb = tqdm(total=self.tell(), desc=desc, file=output, **pb_kwargs) + self._tqdm_bar = tqdm(total=self.tell(), desc=desc, file=output, **pb_kwargs) self.seek(0) def seekable(self): @@ -81,15 +40,11 @@ def tell(self): def read(self, size): prev = self.tell() ret = self._fileobj.read(size) - self._pb.update(self.tell() - prev) + self._tqdm_bar.update(self.tell() - prev) return ret def pb_close(self): - self._pb.close() - - def pb_write(self, message): - """ Allow to write messages to output without interfering with the progress bar """ - tqdm.write(message, file=self._ori_output) + self._tqdm_bar.close() class _NoTerminalOutput(object): @@ -106,13 +61,9 @@ def flush(self): @contextmanager def open_binary(path, output, **kwargs): - output.writeln("Extracting %s" % os.path.basename(path)) with open(path, mode='rb') as f: - if output.is_terminal: - file_wrapped = _FileReaderWithConanProgressBar(f, output=output, **kwargs) - yield file_wrapped - file_wrapped.pb_close() - if not output.is_terminal: - output.write("\n") - else: - yield f + file_wrapped = _FileReaderWithProgressBar(f, output=output, **kwargs) + yield file_wrapped + file_wrapped.pb_close() + if not output.is_terminal: + output.write("\n") From e347ff26672a70c910a98e0fa6d40030c5e368a2 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 25 Jan 2019 17:03:55 +0100 Subject: [PATCH 3/5] added comments and test --- conans/client/output.py | 2 ++ .../unittests/client/conan_output_test.py | 30 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 conans/test/unittests/client/conan_output_test.py diff --git a/conans/client/output.py b/conans/client/output.py index e64d81a8980..655e6770d46 100644 --- a/conans/client/output.py +++ b/conans/client/output.py @@ -62,6 +62,8 @@ def write(self, data, front=None, back=None, newline=False): if newline: data = "%s\n" % data + # https://github.com/conan-io/conan/issues/4277 + # Windows output locks produce IOErrors for _ in range(3): try: self._stream.write(data) diff --git a/conans/test/unittests/client/conan_output_test.py b/conans/test/unittests/client/conan_output_test.py new file mode 100644 index 00000000000..5ae11008ed4 --- /dev/null +++ b/conans/test/unittests/client/conan_output_test.py @@ -0,0 +1,30 @@ +# coding=utf-8 + +import types +import unittest + +from six import StringIO + +from conans.client.output import ConanOutput +from mock import mock + + +class ConanOutputTest(unittest.TestCase): + + def test_blocked_output(self): + # https://github.com/conan-io/conan/issues/4277 + stream = StringIO() + + def write_raise(self, data): + write_raise.counter = getattr(write_raise, "counter", 0) + 1 + if write_raise.counter < 2: + raise IOError("Stdout locked") + self.super_write(data) + stream.super_write = stream.write + stream.write = types.MethodType(write_raise, stream) + out = ConanOutput(stream) + + with mock.patch("time.sleep") as sleep: + out.write("Hello world") + sleep.assert_any_call(0.02) + self.assertEqual("Hello world", stream.getvalue()) From ac94b5109acdf0d202ae5514aaa76d8569bd5ae1 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 25 Jan 2019 17:35:55 +0100 Subject: [PATCH 4/5] commenting out test --- conans/test/unittests/client/conan_output_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conans/test/unittests/client/conan_output_test.py b/conans/test/unittests/client/conan_output_test.py index 5ae11008ed4..85339be9f30 100644 --- a/conans/test/unittests/client/conan_output_test.py +++ b/conans/test/unittests/client/conan_output_test.py @@ -1,5 +1,5 @@ # coding=utf-8 - +""" import types import unittest @@ -28,3 +28,4 @@ def write_raise(self, data): out.write("Hello world") sleep.assert_any_call(0.02) self.assertEqual("Hello world", stream.getvalue()) +""" \ No newline at end of file From dbec75932b5840df308cdd1968bb59e1b4a650e4 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 25 Jan 2019 17:42:16 +0100 Subject: [PATCH 5/5] trying a different import --- conans/test/unittests/client/conan_output_test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/conans/test/unittests/client/conan_output_test.py b/conans/test/unittests/client/conan_output_test.py index 85339be9f30..77838157ce0 100644 --- a/conans/test/unittests/client/conan_output_test.py +++ b/conans/test/unittests/client/conan_output_test.py @@ -1,7 +1,7 @@ # coding=utf-8 -""" -import types + import unittest +from types import MethodType from six import StringIO @@ -21,11 +21,10 @@ def write_raise(self, data): raise IOError("Stdout locked") self.super_write(data) stream.super_write = stream.write - stream.write = types.MethodType(write_raise, stream) + stream.write = MethodType(write_raise, stream) out = ConanOutput(stream) with mock.patch("time.sleep") as sleep: out.write("Hello world") sleep.assert_any_call(0.02) self.assertEqual("Hello world", stream.getvalue()) -""" \ No newline at end of file