[BUG] warnings/output in parallel processes duplicate the output of Process

[basnijholt]

You may find a solution to your problem in the docs or issues.

Describe the bug

warnings emitted (or other output) in parallel processes duplicate the output of Process.

See the progress bar doubling:

Run the following to reproduce the bug:

from rich.progress import Progress
from concurrent.futures import ProcessPoolExecutor


def warns(x):
    import warnings

    warnings.warn("yo")
    return x

# This also reproduces the error:
# def warns(x):
#     print("yo")
#     return x



if __name__ == "__main__":
    with Progress() as p:
        tasks = [p.add_task(f"task {i}", total=1) for i in range(15)]
        with ProcessPoolExecutor() as ex:
            futs = [ex.submit(warns, i) for i in range(15)]

            for task, fut in zip(tasks, futs):
                fut.result()
                p.advance(task)

Platform

Click to expand

What platform (Win/Linux/Mac) are you running on? What terminal software are you using?
MacOS

I may ask you to copy and paste the output of the following commands. It may save some time if you do it now.

If you're using Rich in a terminal:

python -m rich.diagnose
pip freeze | grep rich

╭───────────────────────── <class 'rich.console.Console'> ─────────────────────────╮
│ A high level console interface.                                                  │
│                                                                                  │
│ ╭──────────────────────────────────────────────────────────────────────────────╮ │
│ │ <console width=94 ColorSystem.TRUECOLOR>                                     │ │
│ ╰──────────────────────────────────────────────────────────────────────────────╯ │
│                                                                                  │
│     color_system = 'truecolor'                                                   │
│         encoding = 'utf-8'                                                       │
│             file = <_io.TextIOWrapper name='<stdout>' mode='w' encoding='utf-8'> │
│           height = 51                                                            │
│    is_alt_screen = False                                                         │
│ is_dumb_terminal = False                                                         │
│   is_interactive = True                                                          │
│       is_jupyter = False                                                         │
│      is_terminal = True                                                          │
│   legacy_windows = False                                                         │
│         no_color = False                                                         │
│          options = ConsoleOptions(                                               │
│                        size=ConsoleDimensions(width=94, height=51),              │
│                        legacy_windows=False,                                     │
│                        min_width=1,                                              │
│                        max_width=94,                                             │
│                        is_terminal=True,                                         │
│                        encoding='utf-8',                                         │
│                        max_height=51,                                            │
│                        justify=None,                                             │
│                        overflow=None,                                            │
│                        no_wrap=False,                                            │
│                        highlight=None,                                           │
│                        markup=None,                                              │
│                        height=None                                               │
│                    )                                                             │
│            quiet = False                                                         │
│           record = False                                                         │
│         safe_box = True                                                          │
│             size = ConsoleDimensions(width=94, height=51)                        │
│        soft_wrap = False                                                         │
│           stderr = False                                                         │
│            style = None                                                          │
│         tab_size = 8                                                             │
│            width = 94                                                            │
╰──────────────────────────────────────────────────────────────────────────────────╯
╭─── <class 'rich._windows.WindowsConsoleFeatures'> ────╮
│ Windows features available.                           │
│                                                       │
│ ╭───────────────────────────────────────────────────╮ │
│ │ WindowsConsoleFeatures(vt=False, truecolor=False) │ │
│ ╰───────────────────────────────────────────────────╯ │
│                                                       │
│ truecolor = False                                     │
│        vt = False                                     │
╰───────────────────────────────────────────────────────╯
╭────── Environment Variables ───────╮
│ {                                  │
│     'TERM': 'xterm-256color',      │
│     'COLORTERM': 'truecolor',      │
│     'CLICOLOR': None,              │
│     'NO_COLOR': None,              │
│     'TERM_PROGRAM': 'iTerm.app',   │
│     'COLUMNS': None,               │
│     'LINES': None,                 │
│     'JPY_PARENT_PID': None,        │
│     'VSCODE_VERBOSE_LOGGING': None │
│ }                                  │
╰────────────────────────────────────╯
platform="Darwin"
rich==12.2.0

[basnijholt]

Seems related to #1052.

[willmcgugan]

Rich can only capture the stdout / stderr of its own process. If you want your subprocesses to write to the console, you will need to capture the output and send them to the main process.

[basnijholt]

Thanks for your quick reply. I don't think it is possible to capture the output of a multiprocessing step, if the to-be-pickled function has output on the import.

Imagine there is a file:
func.py

print("imported")

def f(x):
    return x

then

from rich.progress import Progress
from concurrent.futures import ProcessPoolExecutor
from func import f

if __name__ == "__main__":
    with Progress() as p:
        tasks = [p.add_task(f"task {i}", total=1) for i in range(15)]
        with ProcessPoolExecutor() as ex:  # ⚠️ the import will happen here again! ⚠️
            futs = [ex.submit(f, i) for i in range(15)]

            for task, fut in zip(tasks, futs):
                fut.result()
                p.advance(task)

@willmcgugan, do you have any suggestions on how to do this?

[willmcgugan]

It's possible. But it's not straightforward. You can capture the output in your func.py and send that via a pipe or other IPC mechanism to your main process for printing. But that is a project in itself...

[basnijholt]

OK thanks!

Are you aware of some sort of workaround?

Somehow in Jupyter notebook there is no problem:

Does that teach us anything?

[willmcgugan]

In Jupyter the progress bars are rendered in a different process (the browser) and all output is captured. To do that in your terminal you would essentially have to implement something similar.

No "workarounds" that I know of I'm afraid.

[basnijholt]

I have asked this question on StackOverflow, hopefully, it will help me to solve this issue.

[github-actions]

I hope we solved your problem.

If you like using Rich, you might also enjoy Textual

[basnijholt]

This is marked as "completed", however, the bug still exists:

at 11:00:23 ❯ python test.py
task 0  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 1  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 2  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 3  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 4  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 5  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 6  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 7  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 8  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 9  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 10 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 11 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 12 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 13 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 14 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--yo
task 0  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 1  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 2  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 3  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 4  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 5  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 6  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 7  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 8  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 9  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 10 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 11 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 12 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 13 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--
task 14 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   0% -:--:--

[willmcgugan]

It's not a bug, as I explained above. Rich can only capture output from its own process.