nbconvert raises NotImplementedError on Windows & Python 3.8

[QuLogic]

See for example, this matplotlib build.

[NbConvertApp] Converting notebook C:\projects\matplotlib\lib\matplotlib\tests\test_nbagg_01.ipynb to notebook
Traceback (most recent call last):
  File "c:\miniconda37-x64\envs\test-environment\lib\runpy.py", line 194, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "c:\miniconda37-x64\envs\test-environment\lib\runpy.py", line 87, in _run_code
    exec(code, run_globals)
  File "C:\Miniconda37-x64\envs\test-environment\Scripts\jupyter-nbconvert.EXE\__main__.py", line 7, in <module>
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\jupyter_core\application.py", line 270, in launch_instance
    return super(JupyterApp, cls).launch_instance(argv=argv, **kwargs)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\traitlets\config\application.py", line 837, in launch_instance
    app.start()
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\nbconvert\nbconvertapp.py", line 345, in start
    self.convert_notebooks()
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\nbconvert\nbconvertapp.py", line 519, in convert_notebooks
    self.convert_single_notebook(notebook_filename)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\nbconvert\nbconvertapp.py", line 484, in convert_single_notebook
    output, resources = self.export_single_notebook(notebook_filename, resources, input_buffer=input_buffer)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\nbconvert\nbconvertapp.py", line 413, in export_single_notebook
    output, resources = self.exporter.from_filename(notebook_filename, resources=resources)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\nbconvert\exporters\exporter.py", line 182, in from_filename
    return self.from_file(f, resources=resources, **kw)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\nbconvert\exporters\exporter.py", line 200, in from_file
    return self.from_notebook_node(nbformat.read(file_stream, as_version=4), resources=resources, **kw)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\nbconvert\exporters\notebook.py", line 32, in from_notebook_node
    nb_copy, resources = super().from_notebook_node(nb, resources, **kw)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\nbconvert\exporters\exporter.py", line 144, in from_notebook_node
    nb_copy, resources = self._preprocess(nb_copy, resources)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\nbconvert\exporters\exporter.py", line 319, in _preprocess
    nbc, resc = preprocessor(nbc, resc)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\nbconvert\preprocessors\base.py", line 47, in __call__
    return self.preprocess(nb, resources)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\nbconvert\preprocessors\execute.py", line 65, in preprocess
    self.execute()
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\nbclient\util.py", line 74, in wrapped
    return just_run(coro(*args, **kwargs))
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\nbclient\util.py", line 53, in just_run
    return loop.run_until_complete(coro)
  File "c:\miniconda37-x64\envs\test-environment\lib\asyncio\base_events.py", line 616, in run_until_complete
    return future.result()
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\nbclient\client.py", line 519, in async_execute
    async with self.async_setup_kernel(**kwargs):
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\async_generator\_util.py", line 34, in __aenter__
    return await self._agen.asend(None)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\nbclient\client.py", line 478, in async_setup_kernel
    await self.async_start_new_kernel_client()
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\nbclient\client.py", line 403, in async_start_new_kernel_client
    await ensure_async(self.kc.start_channels())
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\jupyter_client\client.py", line 106, in start_channels
    self.shell_channel.start()
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\jupyter_client\client.py", line 151, in shell_channel
    socket = self.connect_shell(identity=self.session.bsession)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\jupyter_client\connect.py", line 561, in connect_shell
    return self._create_connected_socket('shell', identity=identity)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\jupyter_client\connect.py", line 545, in _create_connected_socket
    sock = self.context.socket(socket_type)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\zmq\sugar\context.py", line 226, in socket
    s = self._socket_class(self, socket_type, **kwargs)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\zmq\_future.py", line 144, in __init__
    self._init_io_state()
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\zmq\asyncio\__init__.py", line 53, in _init_io_state
    self.io_loop.add_reader(self._fd, lambda : self._handle_events(0, 0))
  File "c:\miniconda37-x64\envs\test-environment\lib\asyncio\events.py", line 501, in add_reader
    raise NotImplementedError
NotImplementedError
Error in atexit._run_exitfuncs:
Traceback (most recent call last):
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\nbclient\client.py", line 360, in _async_cleanup_kernel
    await ensure_async(self.kc.stop_channels())
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\jupyter_client\client.py", line 125, in stop_channels
    if self.shell_channel.is_alive():
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\jupyter_client\client.py", line 151, in shell_channel
    socket = self.connect_shell(identity=self.session.bsession)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\jupyter_client\connect.py", line 561, in connect_shell
    return self._create_connected_socket('shell', identity=identity)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\jupyter_client\connect.py", line 545, in _create_connected_socket
    sock = self.context.socket(socket_type)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\zmq\sugar\context.py", line 226, in socket
    s = self._socket_class(self, socket_type, **kwargs)
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\zmq\_future.py", line 144, in __init__
    self._init_io_state()
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\zmq\asyncio\__init__.py", line 53, in _init_io_state
    self.io_loop.add_reader(self._fd, lambda : self._handle_events(0, 0))
  File "c:\miniconda37-x64\envs\test-environment\lib\asyncio\events.py", line 501, in add_reader
    raise NotImplementedError
NotImplementedError
Exception ignored in: <function Socket.__del__ at 0x0000004DEB3CF670>
Traceback (most recent call last):
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\zmq\sugar\socket.py", line 67, in __del__
    self.close()
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\zmq\_future.py", line 160, in close
    self._clear_io_state()
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\zmq\asyncio\__init__.py", line 60, in _clear_io_state
    self.io_loop.remove_reader(self._fd)
  File "c:\miniconda37-x64\envs\test-environment\lib\asyncio\events.py", line 504, in remove_reader
    raise NotImplementedError
NotImplementedError: 
Exception ignored in: <function Socket.__del__ at 0x0000004DEB3CF670>
Traceback (most recent call last):
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\zmq\sugar\socket.py", line 67, in __del__
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\zmq\_future.py", line 160, in close
  File "c:\miniconda37-x64\envs\test-environment\lib\site-packages\zmq\asyncio\__init__.py", line 60, in _clear_io_state
  File "c:\miniconda37-x64\envs\test-environment\lib\asyncio\events.py", line 504, in remove_reader
NotImplementedError: 

This looks related to, e.g., jupyter/notebook#4613, jupyter/notebook#5047, jupyter/notebook#5094, or jupyterhub/jupyterhub#3123.

https://github.com/matplotlib/matplotlib/blob/master/lib/matplotlib/tests/test_nbagg_01.ipynb

Nbconvert version: 6.0.1

[QuLogic]

Or likely related to jupyter/nbclient#85, though since this is the actual application part, you might want to put the workaround here? I'm not too sure about that.

[MSeal]

Yup you beat me to typing that by a couple seconds! And I haven't seen any actual resolution for the async on windows issue, so work-around is the only path forward. I'll look at getting this with a couple other reported bugs into a release.

[MSeal]

@QuLogic I put a fix into 6.0.2 for this issue as it was done for papermill. If you're launching as a main application it should be resolved now. But if you're importing nbconvert as a library instead of executing it from the JupyterApp (usually as a CLI) you have to add the same work-around in your code's entry point, as due to the nature of the issue it's not possible for nbconvert to fix the python async problem once application code is up and running:

# See https://bugs.python.org/issue37373 :(
if sys.version_info[0] == 3 and sys.version_info[1] >= 8 and sys.platform.startswith('win'):
    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())

[QuLogic]

We just shell out, so this has started working correctly on CI already.

[tommed]

Sorry guys, noob question but I'm getting this exact issue on Windows 2019 running Python 3.9. How do I go about using the fixed which closed this ticket pls? I have already run pip install --upgrade jupyterlab --force and I already have the most recent, but get the following:

Traceback (most recent call last):
  File "C:\ProgramData\ViPro\Programs\run_notebook\run.py", line 77, in <module>
    subscribe_to_queue()
  File "C:\ProgramData\ViPro\Programs\run_notebook\run.py", line 23, in subscribe_to_queue
    channel.start_consuming()
  File "C:\Python\Python39\lib\site-packages\pika\adapters\blocking_connection.py", line 1866, in start_consuming
    self._process_data_events(time_limit=None)
  File "C:\Python\Python39\lib\site-packages\pika\adapters\blocking_connection.py", line 2027, in _process_data_events
    self.connection.process_data_events(time_limit=time_limit)
  File "C:\Python\Python39\lib\site-packages\pika\adapters\blocking_connection.py", line 834, in process_data_events
    self._dispatch_channel_events()
  File "C:\Python\Python39\lib\site-packages\pika\adapters\blocking_connection.py", line 566, in _dispatch_channel_events
    impl_channel._get_cookie()._dispatch_events()
  File "C:\Python\Python39\lib\site-packages\pika\adapters\blocking_connection.py", line 1493, in _dispatch_events
    consumer_info.on_message_callback(self, evt.method,
  File "C:\ProgramData\ViPro\Programs\run_notebook\run.py", line 29, in on_message_received
    process_extract_bootloader(instruction['path'], instruction['notebookName'])
  File "C:\ProgramData\ViPro\Programs\run_notebook\run.py", line 40, in process_extract_bootloader
    ep.preprocess(nb, {})
  File "C:\Python\Python39\lib\site-packages\nbconvert\preprocessors\execute.py", line 79, in preprocess
    self.execute()
  File "C:\Python\Python39\lib\site-packages\nbclient\util.py", line 74, in wrapped
    return just_run(coro(*args, **kwargs))
  File "C:\Python\Python39\lib\site-packages\nbclient\util.py", line 53, in just_run
    return loop.run_until_complete(coro)
  File "C:\Python\Python39\lib\asyncio\base_events.py", line 642, in run_until_complete
    return future.result()
  File "C:\Python\Python39\lib\site-packages\nbclient\client.py", line 524, in async_execute
    async with self.async_setup_kernel(**kwargs):
  File "C:\Python\Python39\lib\site-packages\async_generator\_util.py", line 34, in __aenter__
    return await self._agen.asend(None)
  File "C:\Python\Python39\lib\site-packages\nbclient\client.py", line 483, in async_setup_kernel
    await self.async_start_new_kernel_client()
  File "C:\Python\Python39\lib\site-packages\nbclient\client.py", line 408, in async_start_new_kernel_client
    await ensure_async(self.kc.start_channels())
  File "C:\Python\Python39\lib\site-packages\jupyter_client\client.py", line 106, in start_channels
    self.shell_channel.start()
  File "C:\Python\Python39\lib\site-packages\jupyter_client\client.py", line 151, in shell_channel
    socket = self.connect_shell(identity=self.session.bsession)
  File "C:\Python\Python39\lib\site-packages\jupyter_client\connect.py", line 561, in connect_shell
    return self._create_connected_socket('shell', identity=identity)
  File "C:\Python\Python39\lib\site-packages\jupyter_client\connect.py", line 545, in _create_connected_socket
    sock = self.context.socket(socket_type)
  File "C:\Python\Python39\lib\site-packages\zmq\sugar\context.py", line 226, in socket
    s = self._socket_class(self, socket_type, **kwargs)
  File "C:\Python\Python39\lib\site-packages\zmq\_future.py", line 144, in __init__
    self._init_io_state()
  File "C:\Python\Python39\lib\site-packages\zmq\asyncio.py", line 56, in _init_io_state
    self.io_loop.add_reader(self._fd, lambda: self._handle_events(0, 0))
  File "C:\Python\Python39\lib\asyncio\events.py", line 504, in add_reader
    raise NotImplementedError
NotImplementedError
Error in atexit._run_exitfuncs:
Traceback (most recent call last):
  File "C:\Python\Python39\lib\site-packages\nbclient\client.py", line 365, in _async_cleanup_kernel
    await ensure_async(self.kc.stop_channels())
  File "C:\Python\Python39\lib\site-packages\jupyter_client\client.py", line 125, in stop_channels
    if self.shell_channel.is_alive():
  File "C:\Python\Python39\lib\site-packages\jupyter_client\client.py", line 151, in shell_channel
    socket = self.connect_shell(identity=self.session.bsession)
  File "C:\Python\Python39\lib\site-packages\jupyter_client\connect.py", line 561, in connect_shell
    return self._create_connected_socket('shell', identity=identity)
  File "C:\Python\Python39\lib\site-packages\jupyter_client\connect.py", line 545, in _create_connected_socket
    sock = self.context.socket(socket_type)
  File "C:\Python\Python39\lib\site-packages\zmq\sugar\context.py", line 226, in socket
    s = self._socket_class(self, socket_type, **kwargs)
  File "C:\Python\Python39\lib\site-packages\zmq\_future.py", line 144, in __init__
    self._init_io_state()
  File "C:\Python\Python39\lib\site-packages\zmq\asyncio.py", line 56, in _init_io_state
    self.io_loop.add_reader(self._fd, lambda: self._handle_events(0, 0))
  File "C:\Python\Python39\lib\asyncio\events.py", line 504, in add_reader
    raise NotImplementedError
NotImplementedError
Exception ignored in: <function Socket.__del__ at 0x00000185C29C7940>
Traceback (most recent call last):
  File "C:\Python\Python39\lib\site-packages\zmq\sugar\socket.py", line 68, in __del__
    self.close()
  File "C:\Python\Python39\lib\site-packages\zmq\_future.py", line 160, in close
    self._clear_io_state()
  File "C:\Python\Python39\lib\site-packages\zmq\asyncio.py", line 63, in _clear_io_state
    self.io_loop.remove_reader(self._fd)
  File "C:\Python\Python39\lib\asyncio\events.py", line 507, in remove_reader
    raise NotImplementedError
NotImplementedError:
Exception ignored in: <function Socket.__del__ at 0x00000185C29C7940>
Traceback (most recent call last):
  File "C:\Python\Python39\lib\site-packages\zmq\sugar\socket.py", line 68, in __del__
  File "C:\Python\Python39\lib\site-packages\zmq\_future.py", line 160, in close
  File "C:\Python\Python39\lib\site-packages\zmq\asyncio.py", line 63, in _clear_io_state
  File "C:\Python\Python39\lib\asyncio\events.py", line 507, in remove_reader
NotImplementedError:

[MSeal]

You're run.py needs to add the hotfix code listed above in the post to set the event loop policy before calling other library calls. It's a core Python issue unfortunately