From 1726d1fa00969e92411c0d5729fe73a12c1a1343 Mon Sep 17 00:00:00 2001 From: M Pacer Date: Wed, 16 May 2018 17:49:01 -0700 Subject: [PATCH 1/5] Pass config to kernel from ExecutePreprocessor for configurable kernels --- nbconvert/preprocessors/execute.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nbconvert/preprocessors/execute.py b/nbconvert/preprocessors/execute.py index 8768328c1..df594c25e 100644 --- a/nbconvert/preprocessors/execute.py +++ b/nbconvert/preprocessors/execute.py @@ -249,7 +249,8 @@ def preprocess(self, nb, resources): # from jupyter_client.manager import start_new_kernel def start_new_kernel(startup_timeout=60, kernel_name='python', **kwargs): - km = self.kernel_manager_class(kernel_name=kernel_name) + km = self.kernel_manager_class(kernel_name=kernel_name, + config=self.config) km.start_kernel(**kwargs) kc = km.client() kc.start_channels() From c1cb89c33493d01e79d47fdc7c2cbec9b1801b11 Mon Sep 17 00:00:00 2001 From: M Pacer Date: Fri, 18 May 2018 19:17:34 -0700 Subject: [PATCH 2/5] Move preprocess logic into setup_preprocessor and start_new_kernel setup_preprocessor is a contextmanager that handles setting and cleaning up attributes needed by the rest of the execution logic (specifically self.nb, self.km, self.kc). start_new_kernel was previously a callback function and is now a separate method used inside setup_preprocessor. This also introduces a dynamic default for the `kernel_name` traitlet that checks if self.nb is defined to see if it has a kernel name, and otherwise will raise an AttributeError (since that means it is being relied upon without any way to know which kernel is to be used. --- nbconvert/preprocessors/execute.py | 117 ++++++++++++++++++++--------- 1 file changed, 81 insertions(+), 36 deletions(-) diff --git a/nbconvert/preprocessors/execute.py b/nbconvert/preprocessors/execute.py index df594c25e..b0cb85739 100644 --- a/nbconvert/preprocessors/execute.py +++ b/nbconvert/preprocessors/execute.py @@ -5,6 +5,7 @@ # Distributed under the terms of the Modified BSD License. from textwrap import dedent +from contextlib import contextmanager try: from queue import Empty # Py 3 @@ -156,6 +157,17 @@ class ExecutePreprocessor(Preprocessor): """ ) ).tag(config=True) + + @default('kernel_name') + def _kernel_name_default(self): + try: + return self.nb.metadata.get('kernelspec', {}).get('name', 'python') + except AttributeError: + raise AttributeError('You did not specify a kernel_name for ' + 'the ExecutePreprocessor and you have not set ' + 'self.nb to be able to use that to infer the ' + 'kernel_name.') + raise_on_iopub_timeout = Bool(False, help=dedent( @@ -199,7 +211,7 @@ class ExecutePreprocessor(Preprocessor): help='The kernel manager class to use.' ) @default('kernel_manager_class') - def _km_default(self): + def _kernel_manager_class_default(self): """Use a dynamic default to avoid importing jupyter_client at startup""" try: from jupyter_client import KernelManager @@ -217,11 +229,44 @@ def _km_default(self): # } _display_id_map = Dict() - def preprocess(self, nb, resources): + def start_new_kernel(self, **kwargs): + """Creates a new kernel manager and kernel client. + + Parameters + ---------- + kwargs : + Any options for `self.kernel_manager_class.start_kernel()`. Because + that defaults to KernelManager, this will likely include options + accepted by `KernelManager.start_kernel()``, which includes `cwd`. + + Returns + ------- + km : KernelManager + A kernel manager as created by self.kernel_manager_class. + kc : KernelClient + Kernel client as created by the kernel manager `km`. """ - Preprocess notebook executing each code cell. + km = self.kernel_manager_class(kernel_name=self.kernel_name, + config=self.config) + km.start_kernel(extra_arguments=self.extra_arguments, **kwargs) + + kc = km.client() + kc.start_channels() + try: + kc.wait_for_ready(timeout=self.startup_timeout) + except RuntimeError: + kc.stop_channels() + km.shutdown_kernel() + raise + kc.allow_stdin = False + return km, kc + + @contextmanager + def setup_preprocessor(self, nb, resources): + """ + Context manager for setting up the class to execute a notebook. - The input argument `nb` is modified in-place. + This creates The input argument `nb` is modified in-place. Parameters ---------- @@ -242,46 +287,46 @@ def preprocess(self, nb, resources): path = resources.get('metadata', {}).get('path', '') if path == '': path = None - + self.nb = nb # clear display_id map self._display_id_map = {} - # from jupyter_client.manager import start_new_kernel - - def start_new_kernel(startup_timeout=60, kernel_name='python', **kwargs): - km = self.kernel_manager_class(kernel_name=kernel_name, - config=self.config) - km.start_kernel(**kwargs) - kc = km.client() - kc.start_channels() - try: - kc.wait_for_ready(timeout=startup_timeout) - except RuntimeError: - kc.stop_channels() - km.shutdown_kernel() - raise - - return km, kc - - kernel_name = nb.metadata.get('kernelspec', {}).get('name', 'python') - if self.kernel_name: - kernel_name = self.kernel_name - self.log.info("Executing notebook with kernel: %s" % kernel_name) - self.km, self.kc = start_new_kernel( - startup_timeout=self.startup_timeout, - kernel_name=kernel_name, - extra_arguments=self.extra_arguments, - cwd=path) - self.kc.allow_stdin = False - self.nb = nb - + self.km, self.kc = self.start_new_kernel(cwd=path) try: - nb, resources = super(ExecutePreprocessor, self).preprocess(nb, resources) + yield finally: self.kc.stop_channels() self.km.shutdown_kernel(now=self.shutdown_kernel == 'immediate') + + for attr in ['nb', 'km', 'kc']: + delattr(self, attr) + + def preprocess(self, nb, resources): + """ + Preprocess notebook executing each code cell. + + The input argument `nb` is modified in-place. - delattr(self, 'nb') + Parameters + ---------- + nb : NotebookNode + Notebook being executed. + resources : dictionary + Additional resources used in the conversion process. For example, + passing ``{'metadata': {'path': run_path}}`` sets the + execution path to ``run_path``. + + Returns + ------- + nb : NotebookNode + The executed notebook. + resources : dictionary + Additional resources used in the conversion process. + """ + + with self.setup_preprocessor(nb, resources): + self.log.info("Executing notebook with kernel: %s" % self.kernel_name) + nb, resources = super(ExecutePreprocessor, self).preprocess(nb, resources) return nb, resources From 0de8f30404a99e63d5cc22e31cb4a984cc201beb Mon Sep 17 00:00:00 2001 From: M Pacer Date: Fri, 18 May 2018 19:18:50 -0700 Subject: [PATCH 3/5] cleans up traitlet stuff moved _display_id_map help inside the declaration instead of comment unified named imports from traitlets --- nbconvert/preprocessors/execute.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/nbconvert/preprocessors/execute.py b/nbconvert/preprocessors/execute.py index b0cb85739..b93cfdc80 100644 --- a/nbconvert/preprocessors/execute.py +++ b/nbconvert/preprocessors/execute.py @@ -12,12 +12,12 @@ except ImportError: from Queue import Empty # Py 2 -from traitlets import List, Unicode, Bool, Enum, Any, Type, Dict, default +from traitlets import List, Unicode, Bool, Enum, Any, Type, Dict, Integer, default from nbformat.v4 import output_from_msg + from .base import Preprocessor from ..utils.exceptions import ConversionException -from traitlets import Integer class CellExecutionError(ConversionException): @@ -219,15 +219,18 @@ def _kernel_manager_class_default(self): raise ImportError("`nbconvert --execute` requires the jupyter_client package: `pip install jupyter_client`") return KernelManager - # mapping of locations of outputs with a given display_id - # tracks cell index and output index within cell.outputs for - # each appearance of the display_id - # { - # 'display_id': { - # cell_idx: [output_idx,] - # } - # } - _display_id_map = Dict() + _display_id_map = Dict( + help=dedent( + """ + mapping of locations of outputs with a given display_id + tracks cell index and output index within cell.outputs for + each appearance of the display_id + { + 'display_id': { + cell_idx: [output_idx,] + } + } + """)) def start_new_kernel(self, **kwargs): """Creates a new kernel manager and kernel client. From d7939027ea6b14e806ca0997fd14b05c613a6424 Mon Sep 17 00:00:00 2001 From: M Pacer Date: Tue, 5 Jun 2018 11:10:36 -0700 Subject: [PATCH 4/5] put attr deletion in finally block, clearer docstring, shorter path def --- nbconvert/preprocessors/execute.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/nbconvert/preprocessors/execute.py b/nbconvert/preprocessors/execute.py index b93cfdc80..d7de2e3d5 100644 --- a/nbconvert/preprocessors/execute.py +++ b/nbconvert/preprocessors/execute.py @@ -269,7 +269,13 @@ def setup_preprocessor(self, nb, resources): """ Context manager for setting up the class to execute a notebook. - This creates The input argument `nb` is modified in-place. + The assigns `nb` to `self.nb` where it will be modified in-place. It also creates + and assigns the Kernel Manager (`self.km`) and Kernel Client(`self.kc`). + + It is intended to yield to a block that will execute codeself. + + When control returns from the yield it stops the client's zmq channels, shuts + down the kernel, and removes the now unused attributes. Parameters ---------- @@ -287,9 +293,7 @@ def setup_preprocessor(self, nb, resources): resources : dictionary Additional resources used in the conversion process. """ - path = resources.get('metadata', {}).get('path', '') - if path == '': - path = None + path = resources.get('metadata', {}).get('path', '') or None self.nb = nb # clear display_id map self._display_id_map = {} @@ -301,8 +305,8 @@ def setup_preprocessor(self, nb, resources): self.kc.stop_channels() self.km.shutdown_kernel(now=self.shutdown_kernel == 'immediate') - for attr in ['nb', 'km', 'kc']: - delattr(self, attr) + for attr in ['nb', 'km', 'kc']: + delattr(self, attr) def preprocess(self, nb, resources): """ From 9611e952877a0053fc81e9eda0d868fbe2a33e65 Mon Sep 17 00:00:00 2001 From: M Pacer Date: Tue, 5 Jun 2018 11:14:33 -0700 Subject: [PATCH 5/5] surface variables from yield as requested in review --- nbconvert/preprocessors/execute.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nbconvert/preprocessors/execute.py b/nbconvert/preprocessors/execute.py index d7de2e3d5..ba7c1d8ff 100644 --- a/nbconvert/preprocessors/execute.py +++ b/nbconvert/preprocessors/execute.py @@ -300,7 +300,8 @@ def setup_preprocessor(self, nb, resources): self.km, self.kc = self.start_new_kernel(cwd=path) try: - yield + # Yielding unbound args for more easier understanding and downstream consumption + yield nb, self.km, self.kc finally: self.kc.stop_channels() self.km.shutdown_kernel(now=self.shutdown_kernel == 'immediate')